owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add user-agent #39

Merged
owner merged 1 commits from AddUserAgent into main 2024-12-21 21:19:29 +00:00
Conversation 0 Commits 1 Files Changed 4 +16 -10
4 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Backend/Handler/ContentFilter.cs
View File

@ -171,9 +171,9 @@ public class ContentFilter
{ {
html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult(); html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
} }
catch (Exception e) catch
{ {
Console.WriteLine(e); //
} }
} }
else else
@ -184,9 +184,9 @@ public class ContentFilter
{ {
html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult(); html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
} }
catch (Exception e) catch
{ {
Console.WriteLine(e); //
} }
} }

2
Backend/Handler/ThreadHandler.cs
View File

@ -59,7 +59,7 @@ public class ThreadHandler
{ {
Thread.Sleep(5000); // Let the database handler instantiate and warm up first. Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
List<WaitHandle[]> wait = _ipScanner.Start(128); List<WaitHandle[]> wait = _ipScanner.Start(32);
for (int i = 0; i < wait.Count; i++) for (int i = 0; i < wait.Count; i++)
{ {

12
Backend/Helper/HttpClientHelper.cs
View File

@ -1,9 +1,9 @@
using System.Diagnostics;
namespace Backend.Helper; namespace Backend.Helper;
public static class HttpClientHelper public static class HttpClientHelper
{ {
private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
public static async Task<string> GetHtml(string url, int port) public static async Task<string> GetHtml(string url, int port)
{ {
using HttpClient client = new(); using HttpClient client = new();
@ -18,7 +18,8 @@ public static class HttpClientHelper
} }
client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Clear();
client.Timeout = TimeSpan.FromSeconds(1); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
client.Timeout = TimeSpan.FromSeconds(30);
HttpResponseMessage? response; HttpResponseMessage? response;
@ -53,12 +54,13 @@ public static class HttpClientHelper
} }
client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Clear();
client.Timeout = TimeSpan.FromSeconds(1); client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
client.Timeout = TimeSpan.FromSeconds(30);
HttpResponseMessage? response = null; HttpResponseMessage? response = null;
try try
{// {
response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt")); response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
} }
catch catch

4
frontend/pages/progress.vue
View File

@ -7,6 +7,7 @@ const loading = ref<boolean>(true);
const error = ref<string | null>(null); const error = ref<string | null>(null);
let dict = ref<ProgressDictionary[] | null>(null); let dict = ref<ProgressDictionary[] | null>(null);
const fetchMyData = async () => { const fetchMyData = async () => {
try { try {
loading.value = true; loading.value = true;
@ -21,6 +22,7 @@ const fetchMyData = async () => {
}, },
5 // Cache max age in seconds 5 // Cache max age in seconds
); );
} catch (err) { } catch (err) {
error.value = (err as Error).message; error.value = (err as Error).message;
} finally { } finally {
@ -34,6 +36,8 @@ const fetchMyData = async () => {
fetchMyData(); fetchMyData();
</script> </script>
<template> <template>