diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 676bb47..8d78691 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -171,9 +171,9 @@ public class ContentFilter { html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult(); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } } else @@ -184,9 +184,9 @@ public class ContentFilter { html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult(); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } } diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 784c270..f92d076 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -59,7 +59,7 @@ public class ThreadHandler { Thread.Sleep(5000); // Let the database handler instantiate and warm up first. - List wait = _ipScanner.Start(128); + List wait = _ipScanner.Start(32); for (int i = 0; i < wait.Count; i++) { diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs index 0dfe66b..25c597a 100644 --- a/Backend/Helper/HttpClientHelper.cs +++ b/Backend/Helper/HttpClientHelper.cs @@ -1,9 +1,9 @@ -using System.Diagnostics; - namespace Backend.Helper; public static class HttpClientHelper { + private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + public static async Task GetHtml(string url, int port) { using HttpClient client = new(); @@ -18,7 +18,8 @@ public static class HttpClientHelper } client.DefaultRequestHeaders.Accept.Clear(); - client.Timeout = TimeSpan.FromSeconds(1); + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader); + client.Timeout = TimeSpan.FromSeconds(30); HttpResponseMessage? response; @@ -53,12 +54,13 @@ public static class HttpClientHelper } client.DefaultRequestHeaders.Accept.Clear(); - client.Timeout = TimeSpan.FromSeconds(1); + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader); + client.Timeout = TimeSpan.FromSeconds(30); HttpResponseMessage? response = null; try - {// + { response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt")); } catch diff --git a/frontend/pages/progress.vue b/frontend/pages/progress.vue index 6de49d6..e079bb2 100644 --- a/frontend/pages/progress.vue +++ b/frontend/pages/progress.vue @@ -7,6 +7,7 @@ const loading = ref(true); const error = ref(null); let dict = ref(null); + const fetchMyData = async () => { try { loading.value = true; @@ -21,6 +22,7 @@ const fetchMyData = async () => { }, 5 // Cache max age in seconds ); + } catch (err) { error.value = (err as Error).message; } finally { @@ -34,6 +36,8 @@ const fetchMyData = async () => { fetchMyData(); + +