Add user-agent

Reviewed-on: #36

Backend/Handler/ContentFilter.cs

@@ -171,9 +171,9 @@ public class ContentFilter
                 {
                     html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
                 }
-                catch (Exception e)
+                catch
                 {
-                    Console.WriteLine(e);
+                    //
                 }
             }
             else
@@ -184,9 +184,9 @@ public class ContentFilter
                 {
                     html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
                 }
-                catch (Exception e)
+                catch
                 {
-                    Console.WriteLine(e);
+                    //
                 }
             }
             

Backend/Handler/ThreadHandler.cs

@@ -59,7 +59,7 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(128);
+        List<WaitHandle[]> wait = _ipScanner.Start(32);
 
         for (int i = 0; i < wait.Count; i++)
         {

Backend/Helper/HttpClientHelper.cs

@@ -1,9 +1,9 @@
-using System.Diagnostics;
-
 namespace Backend.Helper;
 
 public static class HttpClientHelper
 {
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    
     public static async Task<string> GetHtml(string url, int port)
     {
         using HttpClient client = new();
@@ -18,7 +18,8 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response;
         
@@ -53,12 +54,13 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {//
+        {
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch

Proxy/Program.cs

@@ -2,6 +2,7 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using AspNetCoreRateLimit;
 using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
@@ -19,23 +20,7 @@ builder.Services.AddCors(options =>
     options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
-// Add memory cache and rate limiting services
-builder.Services.AddMemoryCache();
-builder.Services.Configure<IpRateLimitOptions>(options =>
-{
-    options.GeneralRules =
-    [
-        new()
-        {
-            Endpoint = "*", // Apply to all endpoints
-            Period = "10s", // Rate limiting window of 10 second
-            Limit = 5 // Maximum 5 requests per 10 seconds
-        }
-    ];
-});
-
-builder.Services.AddInMemoryRateLimiting();
-builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
 
 WebApplication app = builder.Build();
 
@@ -44,14 +29,17 @@ app.UseForwardedHeaders(new()
     ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
 });
 
-
-app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
-
 app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", () =>
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -65,7 +53,11 @@ progressApi.MapGet("/", () =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });
 
 /*

RSE.sln.DotSettings.user

@@ -1,7 +1,9 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

frontend/app.vue

@@ -1,3 +1,13 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
 <template>
   <NuxtLayout>
     <NuxtPage page-key="static" />

frontend/nuxt.config.ts

@@ -14,5 +14,4 @@ export default defineNuxtConfig({
   },
 
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
-
 });

frontend/pages/progress.vue

@@ -7,6 +7,7 @@ const loading = ref<boolean>(true);
 const error = ref<string | null>(null);
 let dict = ref<ProgressDictionary[] | null>(null);
 
+
 const fetchMyData = async () => {
     try {
         loading.value = true;
@@ -21,6 +22,7 @@ const fetchMyData = async () => {
             },
             5 // Cache max age in seconds
         );
+
     } catch (err) {
         error.value = (err as Error).message;
     } finally {
@@ -34,6 +36,8 @@ const fetchMyData = async () => {
 
 fetchMyData();
 
+
+
 </script>
 
 <template>