Title and description extraction now only look in the head section of the HTML document

Reviewed-on: #36

Backend/Backend.csproj

@@ -6,8 +6,8 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
 
-        <Platform>x64</Platform>
+        <!--<Platform>x64</Platform>-->
-        <Optimize>true</Optimize>
+        <Optimize>false</Optimize>
     </PropertyGroup>
     
     <ItemGroup>
@@ -16,6 +16,7 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/ContentFilter.cs

@@ -161,29 +161,39 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html;
-            
             if (ports[i] == 80)
             {
                 if (string.IsNullOrWhiteSpace(url1)) continue;
                 
-                html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result;
+                try
+                {
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
+                    title1 = temp.Item1;
+                    description1 = temp.Item2;
+                }
+                catch
+                {
+                    //
+                }
             }
             else
             {
                 if (string.IsNullOrWhiteSpace(url2)) continue;
                 
-                html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result;
+                try
+                {
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
+                    title2 = temp.Item1;
+                    description2 = temp.Item2;
+                }
+                catch
+                {
+                    //
+                }
             }
             
-            if (string.IsNullOrWhiteSpace(html)) continue;
+            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
-            
+            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
-            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; }
-            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; }
         }
         
         Filtered siteData = new()

Backend/Handler/IpScanner.cs

@@ -41,15 +41,30 @@ public class IpScanner
         _timeout = milliseconds;
     }
     
-    public WaitHandle[] Start(int threads)
+    public List<WaitHandle[]> Start(int threads)
     {
         int threadsAmount = 0;
         if (threads % 2 == 0)
         {
             threadsAmount = 256 / threads;
         }
+
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
         
-        WaitHandle[] waitHandles = new WaitHandle[threads];
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
+        
+        int counter = 0;
+        int counter2 = 0;
         
         for (int i = 0; i < threads; i++)
         {
@@ -62,16 +77,33 @@ public class IpScanner
                 ThreadNumber = i,
                 Handle = handle
             };
-            
+
-            waitHandles[i] = handle;
+            if (i < 64)
+            {
+                waitHandle1[counter] = handle;
+                counter++;
+            }
+            else
+            {
+                waitHandle2[counter2] = handle;
+                counter2++;
+            }
             
             Thread f = new (Scan!);
             f.Start(scanSettings);
             
             Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(1000);
+            Thread.Sleep(100);
         }
         
+        List<WaitHandle[]> waitHandles = new();
+        
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
+        
+        waitHandles.Add(waitHandle1);
+        waitHandles.Add(waitHandle2);
+        
         return waitHandles;
     }
     
@@ -193,7 +225,8 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
+            
+            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
         _resumeQueue.Enqueue(resumeObject);

Backend/Handler/ThreadHandler.cs

@@ -59,9 +59,12 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        WaitHandle[] wait = _ipScanner.Start(64);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
-        
+
-        WaitHandle.WaitAll(wait);
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
         
         Console.WriteLine("Scanner finished");
         
@@ -70,6 +73,8 @@ public class ThreadHandler
     
     private void StartContentFilter()
     {
+        Thread.Sleep(5000);
+        
         WaitHandle[] wait = _contentFilter.Start();
         
         WaitHandle.WaitAll(wait);
@@ -83,12 +88,12 @@ public class ThreadHandler
     {
         _dbHandler.UnfilteredDbHandler();
     }
-
+    
     private void StartFilteredDbHandler()
     {
         _dbHandler.FilteredDbHandler();
     }
-
+    
     private void StartResumeDbHandler()
     {
         _dbHandler.ResumeDbHandler();
@@ -96,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(2);
+        WaitHandle[] wait = _dbHandler.Start(4);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -1,10 +1,18 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
     {
         using HttpClient client = new();
         
@@ -18,9 +26,10 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
         client.Timeout = TimeSpan.FromSeconds(30);
         
-        HttpResponseMessage? response = null;
+        HttpResponseMessage? response;
         
         try
         {
@@ -28,15 +37,48 @@ public static class HttpClientHelper
         }
         catch
         {
-            //
+            return ("", "");
         }
         
-        if (response is null || !response.IsSuccessStatusCode)
+        if (!response.IsSuccessStatusCode)
         {
-            return "";
+            return ("", "");
+        }
+
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
         }
         
-        return await response.Content.ReadAsStringAsync();
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
     }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -53,11 +95,13 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {//
+        {
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
@@ -67,4 +111,9 @@ public static class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }

Models/Handler/DbHandler.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using Microsoft.Data.Sqlite;
 using Models.Model.Backend;
 using Models.Model.External;
@@ -24,18 +25,45 @@ public class DbHandler
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
                                               " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
                                               " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
                                               " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
                                               " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
                                               " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
-                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2," +
+                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
-                                              " @title1, @title2, @description1, @description2, @url1, @url2, @serverType1," +
+                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
-                                              " @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2," +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
-                                              " @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion," +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
-                                              " @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2," +
+                                              " @robotsTXT1, @robotsTXT2," +
-                                              " @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion1)," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion2)," +
+                                              " (SELECT CertificateIssuerId FROM CertificateIssuerCountry WHERE Country = @certificateIssuerCountry)," +
+                                              " (SELECT CertificateOrganizationId FROM CertificateOrganizationName WHERE Name = @certificateOrganizationName), " +
+                                              " @ipV6, " +
+                                              " (SELECT TlsId FROM TlsVersion WHERE Version = @tlsVersion)," +
+                                              " (SELECT CipherId FROM CipherSuite WHERE Suite = @cipherSuite)," +
+                                              " (SELECT KeyExchangeId FROM KeyExchangeAlgorithm WHERE Algorithm = @keyExchangeAlgorithm)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType1)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType2)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType3)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding1)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding2)," +
+                                              " (SELECT ALPNId FROM ALPN WHERE ALPNValue = @aLPN)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection1)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection2))";
+
+    private const string InsertIntoFilteredServerType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ServerType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredHttpVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO HttpVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCertificateIssuerCountry = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateIssuerCountry (Country) VALUES (@country)";
+    private const string InsertIntoFilteredCertificateOrganizationName = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateOrganizationName (Name) VALUES (@name)";
+    private const string InsertIntoFilteredTlsVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO TlsVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCipherSuite = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CipherSuite (Suite) VALUES (@suite)";
+    private const string InsertIntoFilteredKeyExchangeAlgorithm = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO KeyExchangeAlgorithm (Algorithm) VALUES (@algorithm)";
+    private const string InsertIntoFilteredPublicKeyType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO PublicKeyType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredAcceptEncoding = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO AcceptEncoding (Encoding) VALUES (@encoding)";
+    private const string InsertIntoFilteredALPN = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ALPN (ALPNValue) VALUES (@alpnValue)";
+    private const string InsertIntoFilteredConnection = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO Connection (ConnectionValue) VALUES (@connectionValue)";
     
     private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -270,8 +298,75 @@ public class DbHandler
         using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
-        using SqliteCommand command = new(InsertIntoFiltered, connection);
+        SqliteCommand command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType1);
+        _ = command.ExecuteNonQuery();
         
+        command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateIssuerCountry, connection);
+        command.Parameters.AddWithValue("@country", filtered.CertificateIssuerCountry);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateOrganizationName, connection);
+        command.Parameters.AddWithValue("@name", filtered.CertificateOrganizationName);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredTlsVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.TlsVersion);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCipherSuite, connection);
+        command.Parameters.AddWithValue("@suite", filtered.CipherSuite);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredKeyExchangeAlgorithm, connection);
+        command.Parameters.AddWithValue("@algorithm", filtered.KeyExchangeAlgorithm);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType3);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredALPN, connection);
+        command.Parameters.AddWithValue("@alpnValue", filtered.ALPN);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFiltered, connection);
         command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
         command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
         command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
@@ -304,8 +399,9 @@ public class DbHandler
         command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
         command.Parameters.AddWithValue("@connection1", filtered.Connection1);
         command.Parameters.AddWithValue("@connection2", filtered.Connection2);
-        
         _ = command.ExecuteNonQuery();
+        command.Dispose();
+        
         connection.Close();
     }
     
@@ -640,8 +736,8 @@ public class DbHandler
     {
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 TEXT NOT NULL, Ip2 TEXT NOT NULL, Ip3 TEXT NOT NULL, Ip4 TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-
+        
         _discardedConnectionStrings.Add(databaseName);
         
         using SqliteConnection connection = new(databaseName);

Models/Model/Backend/IP.cs

@@ -9,7 +9,7 @@ public struct Ip
     public int Ip3 { get; set; }
     
     public int Ip4 { get; set; }
-
+    
     public override string ToString()
     {
         return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";

Models/Models.csproj

@@ -8,6 +8,5 @@
 
     <ItemGroup>
       <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
-      <PackageReference Include="SQLite" Version="3.13.0" />
     </ItemGroup>
 </Project>

Models/Sql/DropAndCreateFiltered.sql

@@ -1,10 +1,68 @@
 VACUUM;
 
+CREATE TABLE IF NOT EXISTS "ServerType" (
+    "ServerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "HttpVersion" (
+    "HttpId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateIssuerCountry" (
+	"CertificateIssuerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Country" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateOrganizationName" (
+	"CertificateOrganizationId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Name" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "TlsVersion" (
+	"TlsId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CipherSuite" (
+	"CipherId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Suite" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "KeyExchangeAlgorithm" (
+	"KeyExchangeId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Algorithm" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "PublicKeyType" (
+	"PublicKeyId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "AcceptEncoding" (
+	"AcceptId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Encoding" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "ALPN" (
+	"ALPNId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ALPNValue" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "Connection" (
+	"ConnectionId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ConnectionValue" TEXT NOT NULL UNIQUE
+);
+
 DROP TABLE Filtered;
 
 CREATE TABLE "Filtered" (
 	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"	INTEGER NOT NULL,
+	"Ip3"	INTEGER NOT NULL,
+	"Ip4"	INTEGER NOT NULL,
 	"Port1"	INTEGER NOT NULL,
 	"Port2"	INTEGER NOT NULL,
 	"Title1"	TEXT NOT NULL,
@@ -13,25 +71,42 @@ CREATE TABLE "Filtered" (
 	"Description2"	TEXT NOT NULL,
 	"Url1"	TEXT NOT NULL,
 	"Url2"	TEXT NOT NULL,
-	"ServerType1"	TEXT NOT NULL,
+	"ServerType1"	INTEGER NOT NULL,
-	"ServerType2"	TEXT NOT NULL,
+	"ServerType2"	INTEGER NOT NULL,
 	"RobotsTXT1"	TEXT NOT NULL,
 	"RobotsTXT2"	TEXT NOT NULL,
-	"HttpVersion1"	TEXT NOT NULL,
+	"HttpVersion1"	INTEGER NOT NULL,
-	"HttpVersion2"	TEXT NOT NULL,
+	"HttpVersion2"	INTEGER NOT NULL,
-	"CertificateIssuerCountry"	TEXT NOT NULL,
+	"CertificateIssuerCountry"	INTEGER NOT NULL,
-	"CertificateOrganizationName"	TEXT NOT NULL,
+	"CertificateOrganizationName"	INTEGER NOT NULL,
 	"IpV6"	TEXT NOT NULL,
-	"TlsVersion"	TEXT NOT NULL,
+	"TlsVersion"	INTEGER NOT NULL,
-	"CipherSuite"	TEXT NOT NULL,
+	"CipherSuite"	INTEGER NOT NULL,
-	"KeyExchangeAlgorithm"	TEXT NOT NULL,
+	"KeyExchangeAlgorithm"	INTEGER NOT NULL,
-	"PublicKeyType1"	TEXT NOT NULL,
+	"PublicKeyType1"	INTEGER NOT NULL,
-	"PublicKeyType2"	TEXT NOT NULL,
+	"PublicKeyType2"	INTEGER NOT NULL,
-	"PublicKeyType3"	TEXT NOT NULL,
+	"PublicKeyType3"	INTEGER NOT NULL,
-	"AcceptEncoding1"	TEXT NOT NULL,
+	"AcceptEncoding1"	INTEGER NOT NULL,
-	"AcceptEncoding2"	TEXT NOT NULL,
+	"AcceptEncoding2"	INTEGER NOT NULL,
-	"ALPN"	TEXT NOT NULL,
+	"ALPN"	INTEGER NOT NULL,
-	"Connection1" TEXT NOT NULL,
+	"Connection1"	INTEGER NOT NULL,
-	"Connection2" TEXT NOT NULL,
+	"Connection2"	INTEGER NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
+	PRIMARY KEY("Id" AUTOINCREMENT),
+	FOREIGN KEY("ALPN") REFERENCES "ALPN"("ALPNId"),
+	FOREIGN KEY("AcceptEncoding1") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("AcceptEncoding2") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("CertificateIssuerCountry") REFERENCES "CertificateIssuerCountry"("CertificateIssuerId"),
+	FOREIGN KEY("CertificateOrganizationName") REFERENCES "CertificateOrganizationName"("CertificateOrganizationId"),
+	FOREIGN KEY("CipherSuite") REFERENCES "CipherSuite"("CipherId"),
+	FOREIGN KEY("Connection1") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("Connection2") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("HttpVersion1") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("HttpVersion2") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("KeyExchangeAlgorithm") REFERENCES "KeyExchangeAlgorithm"("KeyExchangeId"),
+	FOREIGN KEY("PublicKeyType1") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType2") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType3") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("ServerType1") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("ServerType2") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("TlsVersion") REFERENCES "TlsVersion"("TlsId")
 )

Proxy/Program.cs

@@ -1,10 +1,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using MessagePack;
+using AspNetCoreRateLimit;
-using Microsoft.AspNetCore.Cors.Infrastructure;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
 
 WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
 
@@ -15,18 +17,29 @@ builder.Services.ConfigureHttpJsonOptions(options =>
 
 builder.Services.AddCors(options =>
 {
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
 WebApplication app = builder.Build();
 
-app.UseCors();
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -40,11 +53,15 @@ progressApi.MapGet("/", () =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });
 
+/*
 RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
 searchApi.MapGet("/{term}", (string term) =>
 {
     CommunicationObject communicationObject = new();
@@ -61,11 +78,11 @@ searchApi.MapGet("/{term}", (string term) =>
     
     return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-
+*/
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
+//[JsonSerializable(typeof(SearchResults))]
 [JsonSerializable(typeof(CommunicationObject))]
 internal partial class AppJsonSerializerContext : JsonSerializerContext
 {

Proxy/Proxy.csproj

@@ -9,6 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
         <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
 

Proxy/appsettings.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
       "Microsoft.AspNetCore": "Warning"
     }
   },

RSE.sln

@@ -8,8 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -32,9 +30,5 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,5 +1,11 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}

Shared/Shared.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-</Project>

frontend/.gitignore

@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example

frontend/README.md

@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.

frontend/app.vue

@@ -0,0 +1,16 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <NuxtLayout>
+    <NuxtPage page-key="static" />
+  </NuxtLayout>
+
+</template>

frontend/assets/css/main.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/nuxt.config.ts

@@ -0,0 +1,17 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});

frontend/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}

frontend/pages/index.vue

@@ -0,0 +1,11 @@
+<script setup lang="ts">
+const route = useRoute()
+</script>
+
+<template>
+  <div>
+    <h1>Nuxt Routing set up successfully!</h1>
+    <p>Current route: {{ route.path }}</p>
+    <a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
+  </div>
+</template>

frontend/pages/progress.vue

@@ -0,0 +1,67 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <div v-if="dict">
+      <table class="table-auto border-gray-300">
+        <thead>
+          <tr>
+            <th class="px-4 py-2">Metric</th>
+            <th class="px-4 py-2">Value</th>
+          </tr>
+        </thead>
+
+        <tbody>
+          <tr v-for="d in dict">
+            <td class="px-4 py-2">{{d.description}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+
+</style>

frontend/public/robots.txt

@@ -0,0 +1 @@
+

frontend/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}

frontend/tailwind.config.js

@@ -0,0 +1,16 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {},
+  },
+  plugins: [],
+}
+

frontend/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}

frontend/types/global.d.ts

@@ -0,0 +1,19 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+}
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+}
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};

frontend/utils/cacheUtil.ts

@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}

frontend/utils/convertProgressTypeToDictionary.ts

@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}