Change favicon.ico to favicon.jpg

2024-12-18 20:07:33 +01:00
12 changed files with 72 additions and 93 deletions
@@ -16,7 +16,6 @@
    <ItemGroup>
      <PackageReference Include="FuzzySharp" Version="2.0.2" />
      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
      <PackageReference Include="NetMQ" Version="4.0.1.13" />
    </ItemGroup>
@@ -161,37 +161,41 @@ public class ContentFilter
        for (int i = 0; i < ports.Length; i++)
        {
            string? html = "";
            if (ports[i] == 80)
            {
                if (string.IsNullOrWhiteSpace(url1)) continue;
-                
+
                try
                {
-                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
+                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
                    title1 = temp.Item1;
                    description1 = temp.Item2;
                }
-                catch
+                catch (Exception e)
                {
-                    //
+                    Console.WriteLine(e);
                }
            }
            else
            {
                if (string.IsNullOrWhiteSpace(url2)) continue;
-                
+
                try
                {
-                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
+                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
                    title2 = temp.Item1;
                    description2 = temp.Item2;
                }
-                catch
+                catch (Exception e)
                {
-                    //
+                    Console.WriteLine(e);
                }
            }
            if (string.IsNullOrWhiteSpace(html)) continue;
            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
        }
@@ -1,18 +1,10 @@
-using System.Text.RegularExpressions;
+using System.Diagnostics;
 using HtmlAgilityPack;
 namespace Backend.Helper;
-public static partial class HttpClientHelper
+public static class HttpClientHelper
 {
-    // Reddit, for example, will block the GET request if you don't have a user agent.
+    public static async Task<string> GetHtml(string url, int port)
    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
    private const string TitlePattern = "<title>(.*)</title>";
    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
    private const string StartHeadTag = "<head>";
    private const string EndHeadTag = "</head>";
    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
    {
        using HttpClient client = new();
@@ -26,8 +18,7 @@ public static partial class HttpClientHelper
        }
        client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(1);
        client.Timeout = TimeSpan.FromSeconds(30);
        HttpResponseMessage? response;
@@ -37,48 +28,15 @@ public static partial class HttpClientHelper
        }
        catch
        {
-            return ("", "");
+            return "";
        }
        if (!response.IsSuccessStatusCode)
        {
-            return ("", "");
+            return "";
        }
        string html = await response.Content.ReadAsStringAsync();
        int firstIndex = 0;
        int lastIndex = 0;
        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
        {
            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
        }
-        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        return await response.Content.ReadAsStringAsync();
        html = "";
        string title = "";
        string description = "";
        Regex titleRegex = TitleRegEx();
        Match titleMatch = titleRegex.Match(head);
        if (titleMatch.Success)
        {
            title = titleMatch.Groups[1].Value;
        }
        Regex descriptionRegex = DexcriptionRegEx();
        Match descriptionMatch = descriptionRegex.Match(head);
        if (descriptionMatch.Success)
        {
            description = descriptionMatch.Groups[1].Value;
        }
        return (title, description);
    }
    public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -95,13 +53,12 @@ public static partial class HttpClientHelper
        }
        client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(1);
        client.Timeout = TimeSpan.FromSeconds(30);
        HttpResponseMessage? response = null;
        try
-        {
+        {//
            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
@@ -111,9 +68,4 @@ public static partial class HttpClientHelper
        return response is not null && response.IsSuccessStatusCode;
    }
    [GeneratedRegex(TitlePattern)]
    private static partial Regex TitleRegEx();
    [GeneratedRegex(DescriptionPattern)]
    private static partial Regex DexcriptionRegEx();
 }
@@ -2,7 +2,6 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using AspNetCoreRateLimit;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
@@ -20,7 +19,23 @@ builder.Services.AddCors(options =>
    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
-builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+// Add memory cache and rate limiting services
 builder.Services.AddMemoryCache();
 builder.Services.Configure<IpRateLimitOptions>(options =>
 {
    options.GeneralRules =
    [
        new()
        {
            Endpoint = "*", // Apply to all endpoints
            Period = "10s", // Rate limiting window of 10 second
            Limit = 5 // Maximum 5 requests per 10 seconds
        }
    ];
 });
 builder.Services.AddInMemoryRateLimiting();
 builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
 WebApplication app = builder.Build();
@@ -29,17 +44,14 @@ app.UseForwardedHeaders(new()
    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
 });
 app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
 app.UseCors(myAllowSpecificOrigins);
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", (IMemoryCache memoryCache) =>
+progressApi.MapGet("/", () =>
 {
    const string cacheKey = "progress_status";
    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
    {
        return scanningStatus;
    }
    CommunicationObject communicationObject = new()
    {
        Command = CommunicationCommand.GetScanningProgress
@@ -53,11 +65,7 @@ progressApi.MapGet("/", (IMemoryCache memoryCache) =>
    byte[] msg = client.ReceiveFrameBytes();
    client.Close();
-    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    return JsonSerializer.Deserialize<ScanningStatus>(msg);
    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
    return scanningStatus;
 });
 /*
@@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -30,5 +32,9 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -1,11 +1,7 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
@@ -0,0 +1,5 @@
 namespace Shared;
 public class Class1
 {
 }
@@ -0,0 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
    <PropertyGroup>
        <TargetFramework>net8.0</TargetFramework>
        <ImplicitUsings>enable</ImplicitUsings>
        <Nullable>enable</Nullable>
    </PropertyGroup>
 </Project>
@@ -4,7 +4,11 @@ useHead({
  meta: [
    { name: 'description', content: 'Just a search engine for fun' },
    { name: 'lang', content: 'en'}
-  ]
+  ],
  link: [
    { rel: 'icon', type: 'image/jpeg', href: '/favicon.jpg' }
  ],
 })
 </script>
@@ -7,7 +7,6 @@ const loading = ref<boolean>(true);
 const error = ref<string | null>(null);
 let dict = ref<ProgressDictionary[] | null>(null);
 const fetchMyData = async () => {
    try {
        loading.value = true;
@@ -22,7 +21,6 @@ const fetchMyData = async () => {
            },
            5 // Cache max age in seconds
        );
    } catch (err) {
        error.value = (err as Error).message;
    } finally {
@@ -36,8 +34,6 @@ const fetchMyData = async () => {
 fetchMyData();
 </script>
 <template>