Merge pull request 'Add search page' (#41 ) from CreateSearchPage into main

Reviewed-on: #41
Add search page
2024-12-24 10:11:15 +00:00 · 2024-12-24 11:10:38 +01:00 · 2024-12-23 13:10:49 +00:00 · 2024-12-23 14:09:51 +01:00 · 2024-12-21 21:19:28 +00:00 · 2024-12-21 21:57:01 +01:00
17 changed files with 451 additions and 119 deletions
@@ -16,6 +16,7 @@
    <ItemGroup>
      <PackageReference Include="FuzzySharp" Version="2.0.2" />
      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
      <PackageReference Include="NetMQ" Version="4.0.1.13" />
    </ItemGroup>
@@ -161,19 +161,19 @@ public class ContentFilter
        for (int i = 0; i < ports.Length; i++)
        {
            string? html = "";
            if (ports[i] == 80)
            {
                if (string.IsNullOrWhiteSpace(url1)) continue;
                try
                {
-                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
                    title1 = temp.Item1;
                    description1 = temp.Item2;
                }
-                catch (Exception e)
+                catch
                {
-                    Console.WriteLine(e);
+                    //
                }
            }
            else
@@ -182,20 +182,16 @@ public class ContentFilter
                try
                {
-                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
                    title2 = temp.Item1;
                    description2 = temp.Item2;
                }
-                catch (Exception e)
+                catch
                {
-                    Console.WriteLine(e);
+                    //
                }
            }
            if (string.IsNullOrWhiteSpace(html)) continue;
            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
        }
@@ -1,10 +1,18 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
 using HtmlAgilityPack;
 namespace Backend.Helper;
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
    private const string TitlePattern = "<title>(.*)</title>";
    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
    private const string StartHeadTag = "<head>";
    private const string EndHeadTag = "</head>";
    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
    {
        using HttpClient client = new();
@@ -18,7 +26,8 @@ public static class HttpClientHelper
        }
        client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
        client.Timeout = TimeSpan.FromSeconds(30);
        HttpResponseMessage? response;
@@ -28,15 +37,48 @@ public static class HttpClientHelper
        }
        catch
        {
-            return "";
+            return ("", "");
        }
        if (!response.IsSuccessStatusCode)
        {
-            return "";
+            return ("", "");
        }
-        return await response.Content.ReadAsStringAsync();
+        string html = await response.Content.ReadAsStringAsync();
        int firstIndex = 0;
        int lastIndex = 0;
        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
        {
            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
        }
        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
        html = "";
        string title = "";
        string description = "";
        Regex titleRegex = TitleRegEx();
        Match titleMatch = titleRegex.Match(head);
        if (titleMatch.Success)
        {
            title = titleMatch.Groups[1].Value;
        }
        Regex descriptionRegex = DexcriptionRegEx();
        Match descriptionMatch = descriptionRegex.Match(head);
        if (descriptionMatch.Success)
        {
            description = descriptionMatch.Groups[1].Value;
        }
        return (title, description);
    }
    public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -53,12 +95,13 @@ public static class HttpClientHelper
        }
        client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
        client.Timeout = TimeSpan.FromSeconds(30);
        HttpResponseMessage? response = null;
        try
-        {//
+        {
            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
@@ -68,4 +111,9 @@ public static class HttpClientHelper
        return response is not null && response.IsSuccessStatusCode;
    }
    [GeneratedRegex(TitlePattern)]
    private static partial Regex TitleRegEx();
    [GeneratedRegex(DescriptionPattern)]
    private static partial Regex DexcriptionRegEx();
 }
@@ -2,6 +2,7 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using AspNetCoreRateLimit;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
@@ -19,23 +20,7 @@ builder.Services.AddCors(options =>
    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
-// Add memory cache and rate limiting services
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
 builder.Services.AddMemoryCache();
 builder.Services.Configure<IpRateLimitOptions>(options =>
 {
    options.GeneralRules =
    [
        new()
        {
            Endpoint = "*", // Apply to all endpoints
            Period = "10s", // Rate limiting window of 10 second
            Limit = 5 // Maximum 5 requests per 10 seconds
        }
    ];
 });
 builder.Services.AddInMemoryRateLimiting();
 builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
 WebApplication app = builder.Build();
@@ -44,14 +29,17 @@ app.UseForwardedHeaders(new()
    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
 });
 app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
 app.UseCors(myAllowSpecificOrigins);
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", () =>
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
 {
    const string cacheKey = "progress_status";
    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
    {
        return scanningStatus;
    }
    CommunicationObject communicationObject = new()
    {
        Command = CommunicationCommand.GetScanningProgress
@@ -65,7 +53,11 @@ progressApi.MapGet("/", () =>
    byte[] msg = client.ReceiveFrameBytes();
    client.Close();
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
    return scanningStatus;
 });
 /*
@@ -8,8 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -32,9 +30,5 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -1,7 +1,11 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
@@ -1,5 +0,0 @@
 namespace Shared;
 public class Class1
 {
 }
@@ -1,9 +0,0 @@
 <Project Sdk="Microsoft.NET.Sdk">
    <PropertyGroup>
        <TargetFramework>net8.0</TargetFramework>
        <ImplicitUsings>enable</ImplicitUsings>
        <Nullable>enable</Nullable>
    </PropertyGroup>
 </Project>
@@ -1,7 +1,16 @@
-<template>
+<script setup lang="ts">
-  <div>
+useHead({
-    <NuxtRouteAnnouncer />
+  title: 'Rasmus Search Engine',
  meta: [
    { name: 'description', content: 'Just a search engine for fun' },
    { name: 'lang', content: 'en'}
  ]
 })
 </script>
 <template>
  <NuxtLayout>
    <NuxtPage page-key="static" />
  </NuxtLayout>
  </div>
  <NuxtPage />
 </template>
@@ -13,10 +13,5 @@ export default defineNuxtConfig({
    },
  },
-  routeRules: {
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
-    "/": {isr: true},
+});
    "/progress": {isr: true},
  },
  modules: ['@nuxtjs/tailwindcss'],
 })
@@ -15,6 +15,7 @@
      "devDependencies": {
        "@nuxtjs/tailwindcss": "^6.12.2",
        "autoprefixer": "^10.4.20",
        "nuxt-purgecss": "^2.0.0",
        "postcss": "^8.4.49",
        "tailwindcss": "^3.4.16"
      }
@@ -930,6 +931,19 @@
        "node": ">=18"
      }
    },
    "node_modules/@fullhuman/postcss-purgecss": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/@fullhuman/postcss-purgecss/-/postcss-purgecss-5.0.0.tgz",
      "integrity": "sha512-onDS/b/2pMRzqSoj4qOs2tYFmOpaspjTAgvACIHMPiicu1ptajiBruTrjBzTKdxWdX0ldaBb7wj8nEaTLyFkJw==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "purgecss": "^5.0.0"
      },
      "peerDependencies": {
        "postcss": "^8.0.0"
      }
    },
    "node_modules/@ioredis/commands": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
@@ -3423,6 +3437,16 @@
        "node": ">= 6.0.0"
      }
    },
    "node_modules/camelcase": {
      "version": "5.3.1",
      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
      "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">=6"
      }
    },
    "node_modules/camelcase-css": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz",
@@ -4161,6 +4185,16 @@
        }
      }
    },
    "node_modules/decamelize": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
      "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">=0.10.0"
      }
    },
    "node_modules/deep-equal": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
@@ -4680,6 +4714,20 @@
        "node": ">=8"
      }
    },
    "node_modules/find-up": {
      "version": "4.1.0",
      "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
      "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "locate-path": "^5.0.0",
        "path-exists": "^4.0.0"
      },
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/flatted": {
      "version": "3.3.2",
      "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz",
@@ -4934,6 +4982,91 @@
        "url": "https://github.com/sponsors/isaacs"
      }
    },
    "node_modules/glob-all": {
      "version": "3.3.1",
      "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.3.1.tgz",
      "integrity": "sha512-Y+ESjdI7ZgMwfzanHZYQ87C59jOO0i+Hd+QYtVt9PhLi6d8wlOpzQnfBxWUlaTuAoR3TkybLqqbIoWveU4Ji7Q==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "glob": "^7.2.3",
        "yargs": "^15.3.1"
      },
      "bin": {
        "glob-all": "bin/glob-all"
      }
    },
    "node_modules/glob-all/node_modules/cliui": {
      "version": "6.0.0",
      "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz",
      "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==",
      "dev": true,
      "license": "ISC",
      "dependencies": {
        "string-width": "^4.2.0",
        "strip-ansi": "^6.0.0",
        "wrap-ansi": "^6.2.0"
      }
    },
    "node_modules/glob-all/node_modules/wrap-ansi": {
      "version": "6.2.0",
      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz",
      "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "ansi-styles": "^4.0.0",
        "string-width": "^4.1.0",
        "strip-ansi": "^6.0.0"
      },
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/glob-all/node_modules/y18n": {
      "version": "4.0.3",
      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",
      "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==",
      "dev": true,
      "license": "ISC"
    },
    "node_modules/glob-all/node_modules/yargs": {
      "version": "15.4.1",
      "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz",
      "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "cliui": "^6.0.0",
        "decamelize": "^1.2.0",
        "find-up": "^4.1.0",
        "get-caller-file": "^2.0.1",
        "require-directory": "^2.1.1",
        "require-main-filename": "^2.0.0",
        "set-blocking": "^2.0.0",
        "string-width": "^4.2.0",
        "which-module": "^2.0.0",
        "y18n": "^4.0.0",
        "yargs-parser": "^18.1.2"
      },
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/glob-all/node_modules/yargs-parser": {
      "version": "18.1.3",
      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz",
      "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==",
      "dev": true,
      "license": "ISC",
      "dependencies": {
        "camelcase": "^5.0.0",
        "decamelize": "^1.2.0"
      },
      "engines": {
        "node": ">=6"
      }
    },
    "node_modules/glob-parent": {
      "version": "5.1.2",
      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
@@ -6007,6 +6140,19 @@
        "url": "https://github.com/sponsors/antfu"
      }
    },
    "node_modules/locate-path": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
      "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "p-locate": "^4.1.0"
      },
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/lodash": {
      "version": "4.17.21",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -6774,6 +6920,27 @@
        }
      }
    },
    "node_modules/nuxt-purgecss": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/nuxt-purgecss/-/nuxt-purgecss-2.0.0.tgz",
      "integrity": "sha512-ajTVNZ7CoHwgmR0le0mqNg+CHXuOxVAmUmvnUeBpdurHmJ2Q4fJ6xbK2vbd99Vi0COpmz/TPMNz2gGmA478kcQ==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "@fullhuman/postcss-purgecss": "^5.0.0",
        "@nuxt/kit": "^3.0.0-rc.11",
        "consola": "^2.15.3",
        "glob-all": "^3.2.1",
        "purgecss": "^5.0.0"
      }
    },
    "node_modules/nuxt-purgecss/node_modules/consola": {
      "version": "2.15.3",
      "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz",
      "integrity": "sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==",
      "dev": true,
      "license": "MIT"
    },
    "node_modules/nypm": {
      "version": "0.3.12",
      "resolved": "https://registry.npmjs.org/nypm/-/nypm-0.3.12.tgz",
@@ -6992,6 +7159,45 @@
        "typescript": "^5.x"
      }
    },
    "node_modules/p-limit": {
      "version": "2.3.0",
      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
      "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "p-try": "^2.0.0"
      },
      "engines": {
        "node": ">=6"
      },
      "funding": {
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
    "node_modules/p-locate": {
      "version": "4.1.0",
      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
      "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "p-limit": "^2.2.0"
      },
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/p-try": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">=6"
      }
    },
    "node_modules/package-json-from-dist": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
@@ -7067,6 +7273,16 @@
        "node": ">= 0.8"
      }
    },
    "node_modules/path-exists": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
      "dev": true,
      "license": "MIT",
      "engines": {
        "node": ">=8"
      }
    },
    "node_modules/path-is-absolute": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
@@ -7960,6 +8176,76 @@
      "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==",
      "license": "MIT"
    },
    "node_modules/purgecss": {
      "version": "5.0.0",
      "resolved": "https://registry.npmjs.org/purgecss/-/purgecss-5.0.0.tgz",
      "integrity": "sha512-RAnuxrGuVyLLTr8uMbKaxDRGWMgK5CCYDfRyUNNcaz5P3kGgD2b7ymQGYEyo2ST7Tl/ScwFgf5l3slKMxHSbrw==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "commander": "^9.0.0",
        "glob": "^8.0.3",
        "postcss": "^8.4.4",
        "postcss-selector-parser": "^6.0.7"
      },
      "bin": {
        "purgecss": "bin/purgecss.js"
      }
    },
    "node_modules/purgecss/node_modules/brace-expansion": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
        "balanced-match": "^1.0.0"
      }
    },
    "node_modules/purgecss/node_modules/commander": {
      "version": "9.5.0",
      "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz",
      "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==",
      "dev": true,
      "license": "MIT",
      "engines": {
        "node": "^12.20.0 || >=14"
      }
    },
    "node_modules/purgecss/node_modules/glob": {
      "version": "8.1.0",
      "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
      "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
      "deprecated": "Glob versions prior to v9 are no longer supported",
      "dev": true,
      "license": "ISC",
      "dependencies": {
        "fs.realpath": "^1.0.0",
        "inflight": "^1.0.4",
        "inherits": "2",
        "minimatch": "^5.0.1",
        "once": "^1.3.0"
      },
      "engines": {
        "node": ">=12"
      },
      "funding": {
        "url": "https://github.com/sponsors/isaacs"
      }
    },
    "node_modules/purgecss/node_modules/minimatch": {
      "version": "5.1.6",
      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
      "dev": true,
      "license": "ISC",
      "dependencies": {
        "brace-expansion": "^2.0.1"
      },
      "engines": {
        "node": ">=10"
      }
    },
    "node_modules/queue-microtask": {
      "version": "1.2.3",
      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -8146,6 +8432,13 @@
        "node": ">=0.10.0"
      }
    },
    "node_modules/require-main-filename": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
      "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
      "dev": true,
      "license": "ISC"
    },
    "node_modules/resolve": {
      "version": "1.22.8",
      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
@@ -10736,6 +11029,13 @@
        "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
      }
    },
    "node_modules/which-module": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.1.tgz",
      "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==",
      "dev": true,
      "license": "ISC"
    },
    "node_modules/wide-align": {
      "version": "1.1.5",
      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz",
@@ -18,6 +18,7 @@
  "devDependencies": {
    "@nuxtjs/tailwindcss": "^6.12.2",
    "autoprefixer": "^10.4.20",
    "nuxt-purgecss": "^2.0.0",
    "postcss": "^8.4.49",
    "tailwindcss": "^3.4.16"
  }
@@ -3,9 +3,10 @@ const route = useRoute()
 </script>
 <template>
-  <div>
+  <div class="h-screen flex items-center justify-center">
-    <h1>Nuxt Routing set up successfully!</h1>
+    <div class="flex space-x-2">
-    <p>Current route: {{ route.path }}</p>
+      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
-    <a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
+      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
    </div>
  </div>
 </template>
@@ -1,9 +1,11 @@
 <script setup lang="ts">
 import { fetchWithCache } from '~/utils/cacheUtil';
 import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
 const data = ref<ProgressType | null>(null);
 const loading = ref<boolean>(true);
 const error = ref<string | null>(null);
 let dict = ref<ProgressDictionary[] | null>(null);
 const fetchMyData = async () => {
    try {
@@ -19,10 +21,15 @@ const fetchMyData = async () => {
            },
            5 // Cache max age in seconds
        );
    } catch (err) {
        error.value = (err as Error).message;
    } finally {
        loading.value = false;
      if (data.value !== null) {
        dict.value = progressTypeToDictionary(data.value);
      }
    }
 };
@@ -32,7 +39,7 @@ fetchMyData();
 <template>
  <div class="h-screen grid place-items-center">
-    <div v-if="data">
+    <div v-if="dict">
      <table class="table-auto border-gray-300">
        <thead>
          <tr>
@@ -42,33 +49,9 @@ fetchMyData();
        </thead>
        <tbody>
-        <tr>
+          <tr v-for="d in dict">
-          <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+            <td class="px-4 py-2">{{d.description}}</td>
-          <td class="px-4 py-2">{{data.percentageOfIpv4Scanned}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Total filtered</td>
          <td class="px-4 py-2">{{data.totalFiltered}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Total Discarded</td>
          <td class="px-4 py-2">{{data.totalDiscarded}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Amount of Ipv4 left</td>
          <td class="px-4 py-2">{{data.amountOfIpv4Left}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Discarded db size</td>
          <td class="px-4 py-2">{{data.discardedDbSize}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Filtered db size</td>
          <td class="px-4 py-2">{{data.filteredDbSize}}</td>
        </tr>
        <tr>
          <td class="px-4 py-2">Unfiltered db size</td>
          <td class="px-4 py-2">{{data.myDbSize}}</td>
          </tr>
        </tbody>
      </table>
@@ -9,7 +9,13 @@ export default {
    "./error.vue",
  ],
  theme: {
-    extend: {},
+    extend: {
      width: {
        '128': '32rem',
        '192' : '48rem',
        '256': '64rem',
      }
    },
  },
  plugins: [],
 }
@@ -8,6 +8,11 @@ interface ProgressType {
    myDbSize: bigint;
 }
 interface ProgressDictionary {
    description: string;
    value: string;
 }
 type CacheEntry<T> = {
    data: T;
    timestamp: number;
@@ -0,0 +1,11 @@
 export function progressTypeToDictionary(progress: ProgressType) {
    return [
    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
    { description: "Total filtered", value: progress.totalFiltered.toString() },
    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
    ]
 }
Author	SHA1	Message	Date
owner	3f647a3e68	Merge pull request 'Add search page' (#41 ) from CreateSearchPage into main Reviewed-on: #41	2024-12-24 10:11:15 +00:00
owner	53289db102	Add search page	2024-12-24 11:10:38 +01:00
owner	f4d94a3336	Merge pull request 'Title and description extraction now only look in the head section of the HTML document' (#40 ) from OptimizeTitleAndDescriptionExtraction into main Reviewed-on: #40	2024-12-23 13:10:49 +00:00
owner	e9fd738f1f	Title and description extraction now only look in the head section of the HTML document	2024-12-23 14:09:51 +01:00
owner	e581d947f8	Merge pull request 'Add user-agent' (#39 ) from AddUserAgent into main Reviewed-on: #39	2024-12-21 21:19:28 +00:00
owner	580d687f62	Add user-agent	2024-12-21 21:57:01 +01:00
owner	af7b385641	Merge pull request 'Add cached progress response' (#36 ) from ImplementAPICache into main Reviewed-on: #36	2024-12-19 17:37:01 +00:00
owner	7562bbf7d1	Add cached progress response	2024-12-19 18:36:13 +01:00
owner	ac645b01b9	Merge pull request 'OptimiseFrontend' (#35 ) from OptimiseFrontend into main Reviewed-on: #35	2024-12-18 13:28:03 +00:00
owner	97321492ba	Add headers	2024-12-18 14:27:31 +01:00
owner	8628d31bec	Add nuxt-purgecss to further minimize bundle size.	2024-12-18 12:02:35 +01:00
owner	23d0a8b978	Merge pull request 'UseDictionary' (#34 ) from UseDictionary into main Reviewed-on: #34	2024-12-17 20:33:08 +00:00
owner	8b55cf7bb6	Reworked the /progress page	2024-12-17 13:15:47 +01:00