Add search page

Merge pull request 'Title and description extraction now only look in the head section of the HTML document' (#40 ) from OptimizeTitleAndDescriptionExtraction into main
Reviewed-on: #40
2024-12-24 11:10:38 +01:00 · 2024-12-23 13:10:49 +00:00 · 2024-12-23 14:09:51 +01:00 · 2024-12-21 21:19:28 +00:00 · 2024-12-21 21:57:01 +01:00 · 2024-12-19 17:37:01 +00:00
27 changed files with 11639 additions and 57 deletions
@@ -16,6 +16,7 @@
    
    <ItemGroup>
      <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
      <PackageReference Include="NetMQ" Version="4.0.1.13" />
    </ItemGroup>
    
@@ -161,19 +161,19 @@ public class ContentFilter
        
        for (int i = 0; i < ports.Length; i++)
        {
-            string? html = "";
-            
            if (ports[i] == 80)
            {
                if (string.IsNullOrWhiteSpace(url1)) continue;
                
                try
                {
-                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
+                    title1 = temp.Item1;
+                    description1 = temp.Item2;
                }
-                catch (Exception e)
+                catch
                {
-                    Console.WriteLine(e);
+                    //
                }
            }
            else
@@ -182,20 +182,16 @@ public class ContentFilter
                
                try
                {
-                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
+                    title2 = temp.Item1;
+                    description2 = temp.Item2;
                }
-                catch (Exception e)
+                catch
                {
-                    Console.WriteLine(e);
+                    //
                }
            }
            
-            if (string.IsNullOrWhiteSpace(html)) continue;
-            
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
        }
@@ -1,10 +1,18 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;

 namespace Backend.Helper;

-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
    {
        using HttpClient client = new();
        
@@ -18,7 +26,8 @@ public static class HttpClientHelper
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response;
        
@@ -28,15 +37,48 @@ public static class HttpClientHelper
        }
        catch
        {
-            return "";
+            return ("", "");
        }
        
        if (!response.IsSuccessStatusCode)
        {
-            return "";
+            return ("", "");
        }

-        return await response.Content.ReadAsStringAsync();
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
+        }
+        
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
    }
    
    public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -53,12 +95,13 @@ public static class HttpClientHelper
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response = null;
        
        try
-        {//
+        {
            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
@@ -68,4 +111,9 @@ public static class HttpClientHelper
        
        return response is not null && response.IsSuccessStatusCode;
    }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }
@@ -1,10 +1,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using MessagePack;
-using Microsoft.AspNetCore.Cors.Infrastructure;
+using AspNetCoreRateLimit;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";

 WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);

@@ -15,18 +17,29 @@ builder.Services.ConfigureHttpJsonOptions(options =>

 builder.Services.AddCors(options =>
 {
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });

+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
 WebApplication app = builder.Build();

-app.UseCors();
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);

 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
    CommunicationObject communicationObject = new()
    {
        Command = CommunicationCommand.GetScanningProgress
@@ -40,11 +53,15 @@ progressApi.MapGet("/", () =>
    byte[] msg = client.ReceiveFrameBytes();
    client.Close();
    
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });

+/*
 RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
 searchApi.MapGet("/{term}", (string term) =>
 {
    CommunicationObject communicationObject = new();
@@ -61,11 +78,11 @@ searchApi.MapGet("/{term}", (string term) =>
    
    return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-
+*/
 app.Run();

 [JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
+//[JsonSerializable(typeof(SearchResults))]
 [JsonSerializable(typeof(CommunicationObject))]
 internal partial class AppJsonSerializerContext : JsonSerializerContext
 {
@@ -9,6 +9,7 @@
    </PropertyGroup>

    <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
        <PackageReference Include="NetMQ" Version="4.0.1.13" />
    </ItemGroup>

@@ -1,7 +1,7 @@
 {
  "Logging": {
    "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
      "Microsoft.AspNetCore": "Warning"
    }
  },
@@ -8,8 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -32,9 +30,5 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal
@@ -1,6 +1,11 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}
@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-</Project>
@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example
@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.
@@ -0,0 +1,16 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <NuxtLayout>
+    <NuxtPage page-key="static" />
+  </NuxtLayout>
+
+</template>
@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
@@ -0,0 +1,17 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});
@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}
@@ -0,0 +1,12 @@
+<script setup lang="ts">
+const route = useRoute()
+</script>
+
+<template>
+  <div class="h-screen flex items-center justify-center">
+    <div class="flex space-x-2">
+      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
+    </div>
+  </div>
+</template>
@@ -0,0 +1,64 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <div v-if="dict">
+      <table class="table-auto border-gray-300">
+        <thead>
+          <tr>
+            <th class="px-4 py-2">Metric</th>
+            <th class="px-4 py-2">Value</th>
+          </tr>
+        </thead>
+
+        <tbody>
+          <tr v-for="d in dict">
+            <td class="px-4 py-2">{{d.description}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+
+</style>
@@ -0,0 +1 @@
+
@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}
@@ -0,0 +1,22 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {
+      width: {
+        '128': '32rem',
+        '192' : '48rem',
+        '256': '64rem',
+      }
+    },
+  },
+  plugins: [],
+}
+
@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}
@@ -0,0 +1,19 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+}
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+}
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};
@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}
@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}
Author	SHA1	Message	Date
owner	53289db102	Add search page	2024-12-24 11:10:38 +01:00
owner	f4d94a3336	Merge pull request 'Title and description extraction now only look in the head section of the HTML document' (#40 ) from OptimizeTitleAndDescriptionExtraction into main Reviewed-on: #40	2024-12-23 13:10:49 +00:00
owner	e9fd738f1f	Title and description extraction now only look in the head section of the HTML document	2024-12-23 14:09:51 +01:00
owner	e581d947f8	Merge pull request 'Add user-agent' (#39 ) from AddUserAgent into main Reviewed-on: #39	2024-12-21 21:19:28 +00:00
owner	580d687f62	Add user-agent	2024-12-21 21:57:01 +01:00
owner	af7b385641	Merge pull request 'Add cached progress response' (#36 ) from ImplementAPICache into main Reviewed-on: #36	2024-12-19 17:37:01 +00:00
owner	7562bbf7d1	Add cached progress response	2024-12-19 18:36:13 +01:00
owner	ac645b01b9	Merge pull request 'OptimiseFrontend' (#35 ) from OptimiseFrontend into main Reviewed-on: #35	2024-12-18 13:28:03 +00:00
owner	97321492ba	Add headers	2024-12-18 14:27:31 +01:00
owner	8628d31bec	Add nuxt-purgecss to further minimize bundle size.	2024-12-18 12:02:35 +01:00
owner	23d0a8b978	Merge pull request 'UseDictionary' (#34 ) from UseDictionary into main Reviewed-on: #34	2024-12-17 20:33:08 +00:00
owner	8b55cf7bb6	Reworked the /progress page	2024-12-17 13:15:47 +01:00
owner	3822339dd9	Add a time-based cache for the progress result from the API	2024-12-17 10:46:10 +01:00
owner	1c74ae9de5	Added frontend and working fetching.	2024-12-14 13:26:21 +01:00
owner	bab8fa6c25	Added frontend project with some basic api fetching implemented.	2024-12-06 12:33:48 +01:00