Change favicon.ico to favicon.jpg

Backend/Handler/ContentFilter.cs

@@ -171,9 +171,9 @@ public class ContentFilter
                 {
                     html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
                 }
-                catch
+                catch (Exception e)
                 {
-                    //
+                    Console.WriteLine(e);
                 }
             }
             else
@@ -184,9 +184,9 @@ public class ContentFilter
                 {
                     html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
                 }
-                catch
+                catch (Exception e)
                 {
-                    //
+                    Console.WriteLine(e);
                 }
             }
             

Backend/Handler/ThreadHandler.cs

@@ -59,7 +59,7 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(32);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
 
         for (int i = 0; i < wait.Count; i++)
         {

Backend/Helper/HttpClientHelper.cs

@@ -1,9 +1,9 @@
+using System.Diagnostics;
+
 namespace Backend.Helper;
 
 public static class HttpClientHelper
 {
-    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-    
     public static async Task<string> GetHtml(string url, int port)
     {
         using HttpClient client = new();
@@ -18,8 +18,7 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
-        client.Timeout = TimeSpan.FromSeconds(30);
+        client.Timeout = TimeSpan.FromSeconds(1);
         
         HttpResponseMessage? response;
         
@@ -54,13 +53,12 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
-        client.Timeout = TimeSpan.FromSeconds(30);
+        client.Timeout = TimeSpan.FromSeconds(1);
         
         HttpResponseMessage? response = null;
         
         try
-        {
+        {//
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch

Proxy/Program.cs

@@ -2,7 +2,6 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using AspNetCoreRateLimit;
 using Microsoft.AspNetCore.HttpOverrides;
-using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
@@ -20,7 +19,23 @@ builder.Services.AddCors(options =>
     options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
-builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+// Add memory cache and rate limiting services
+builder.Services.AddMemoryCache();
+builder.Services.Configure<IpRateLimitOptions>(options =>
+{
+    options.GeneralRules =
+    [
+        new()
+        {
+            Endpoint = "*", // Apply to all endpoints
+            Period = "10s", // Rate limiting window of 10 second
+            Limit = 5 // Maximum 5 requests per 10 seconds
+        }
+    ];
+});
+
+builder.Services.AddInMemoryRateLimiting();
+builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
 
 WebApplication app = builder.Build();
 
@@ -29,17 +44,14 @@ app.UseForwardedHeaders(new()
     ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
 });
 
+
+app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
+
 app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", (IMemoryCache memoryCache) =>
+progressApi.MapGet("/", () =>
 {
-    const string cacheKey = "progress_status";
-    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
-    {
-        return scanningStatus;
-    }
-    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -53,11 +65,7 @@ progressApi.MapGet("/", (IMemoryCache memoryCache) =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
-    
-    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
-    
-    return scanningStatus;
+    return JsonSerializer.Deserialize<ScanningStatus>(msg);
 });
 
 /*

RSE.sln.DotSettings.user

@@ -1,9 +1,7 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

frontend/app.vue

@@ -4,7 +4,11 @@ useHead({
   meta: [
     { name: 'description', content: 'Just a search engine for fun' },
     { name: 'lang', content: 'en'}
-  ]
+  ],
+
+  link: [
+    { rel: 'icon', type: 'image/jpeg', href: '/favicon.jpg' }
+  ],
 })
 </script>
 

frontend/pages/progress.vue

@@ -7,7 +7,6 @@ const loading = ref<boolean>(true);
 const error = ref<string | null>(null);
 let dict = ref<ProgressDictionary[] | null>(null);
 
-
 const fetchMyData = async () => {
     try {
         loading.value = true;
@@ -22,7 +21,6 @@ const fetchMyData = async () => {
             },
             5 // Cache max age in seconds
         );
-
     } catch (err) {
         error.value = (err as Error).message;
     } finally {
@@ -36,8 +34,6 @@ const fetchMyData = async () => {
 
 fetchMyData();
 
-
-
 </script>
 
 <template>