Merge pull request 'CompressDatabase' (#43) from CompressDatabase into main

Reviewed-on: #43

Backend/Backend.csproj

@@ -6,8 +6,8 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
 
-        <Platform>x64</Platform>
+        <!--<Platform>x64</Platform>-->
-        <Optimize>true</Optimize>
+        <Optimize>false</Optimize>
     </PropertyGroup>
     
     <ItemGroup>
@@ -16,6 +16,7 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/Communication.cs

@@ -87,6 +87,7 @@ public class Communication
                 }
                 else
                 {
+                    // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
                     status.PercentageOfIpv4Scanned = 0.0000000001f;
                 }
                 
@@ -96,7 +97,7 @@ public class Communication
                 status.MyDbSize = databaseSizes.MyDbSize;
                 status.FilteredDbSize = databaseSizes.FilteredDbSize;
                 status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
-
+                
                 byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
                 
                 rep.SendFrame(serializedResult);

Backend/Handler/ContentFilter.cs

@@ -8,7 +8,8 @@ namespace Backend.Handler;
 
 public class ContentFilter
 {
-    private readonly ConcurrentQueue<QueueItem> _queue;
+    private readonly ConcurrentQueue<Filtered> _queue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly DbHandler _dbHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
@@ -16,11 +17,12 @@ public class ContentFilter
     private int _timeOut;
     private readonly string _basePath;
     
-    public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, string basePath)
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
+        _unfilteredQueue = unfilteredQueue;
         
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
@@ -49,43 +51,39 @@ public class ContentFilter
     {
         while (!_stop)
         {
-            long indexes = _dbHandler.GetUnfilteredIndexes();
+            List<long> indexes = _dbHandler.GetUnfilteredIndexes();
             
-            for (long i = 0; i < indexes; i++)
+            for (int i = 0; i < indexes.Count; i++)
             {
                 if (_stop) break;
                 
-                Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i);
+                Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                 
-                if (unfiltered is null || unfiltered.Filtered == 1) continue;
+                if (unfiltered.Filtered) continue;
                 
-                unfiltered.Filtered = 1;
+                Ip ip = unfiltered.Ip;
                 
-                QueueItem superUnfilteredObject = new()
+                unfiltered.Filtered = true;
+                
+                UnfilteredQueueItem superUnfilteredObject = new()
                 {
                     Unfiltered = unfiltered,
                     Operations = Operations.Update
                 };
                 
-                _queue.Enqueue(superUnfilteredObject);
+                _unfilteredQueue.Enqueue(superUnfilteredObject);
                 
-                if (_dbHandler.GetFilteredIp(unfiltered.Ip))
+                if (_dbHandler.FilteredIpExists(unfiltered.Ip))
                 {
                     continue;
                 }
                 
-                Filtered filtered = GetSiteData(unfiltered.Ip);
+                Filtered filtered = GetSiteData(ip);
                 
                 filtered.Port1 = unfiltered.Port1;
                 filtered.Port2 = unfiltered.Port2;
                 
-                QueueItem superFilteredObject = new()
+                _queue.Enqueue(filtered);
-                {
-                    Filtered = filtered,
-                    Operations = Operations.Insert
-                };
-                
-                _queue.Enqueue(superFilteredObject);
             }
             
             Thread.Sleep(_timeOut);
@@ -94,7 +92,7 @@ public class ContentFilter
         ((EventWaitHandle) obj).Set();
     }
     
-    private Filtered GetSiteData(string ip)
+    private Filtered GetSiteData(Ip ip)
     {
         StartProcess(ip, 80);
         StartProcess(ip, 443);
@@ -163,29 +161,39 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html;
-            
             if (ports[i] == 80)
             {
                 if (string.IsNullOrWhiteSpace(url1)) continue;
                 
-                html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result;
+                try
+                {
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
+                    title1 = temp.Item1;
+                    description1 = temp.Item2;
+                }
+                catch
+                {
+                    //
+                }
             }
             else
             {
                 if (string.IsNullOrWhiteSpace(url2)) continue;
                 
-                html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result;
+                try
+                {
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
+                    title2 = temp.Item1;
+                    description2 = temp.Item2;
+                }
+                catch
+                {
+                    //
+                }
             }
             
-            if (string.IsNullOrWhiteSpace(html)) continue;
+            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
-            
+            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
-            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; }
-            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; }
         }
         
         Filtered siteData = new()
@@ -222,7 +230,7 @@ public class ContentFilter
         return siteData;
     }
     
-    private void StartProcess(string ip, int port)
+    private void StartProcess(Ip ip, int port)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
         
@@ -230,7 +238,7 @@ public class ContentFilter
         proc.StartInfo = new()
         {
             FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt",
+            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,

Backend/Handler/IpScanner.cs

@@ -17,19 +17,23 @@ public class ScanSettings
 
 public class IpScanner
 {
-    private readonly ConcurrentQueue<QueueItem> _queue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
     private int _timeout;
     
-    public IpScanner(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, ConcurrentQueue<Discarded> discardedQueue)
+    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
+        )
     {
-        _queue = queue;
         _dbHandler = dbHandler;
         _discardedQueue = discardedQueue;
-        
+        _unfilteredQueue = unfilteredQueue;
-        SetTimeout(128);
+        _resumeQueue = resumeQueue;
+
+        SetTimeout(64);
     }
     
     public void SetTimeout(int milliseconds)
@@ -37,7 +41,7 @@ public class IpScanner
         _timeout = milliseconds;
     }
     
-    public WaitHandle[] Start(int threads)
+    public List<WaitHandle[]> Start(int threads)
     {
         int threadsAmount = 0;
         if (threads % 2 == 0)
@@ -45,7 +49,23 @@ public class IpScanner
             threadsAmount = 256 / threads;
         }
         
-        WaitHandle[] waitHandles = new WaitHandle[threads];
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
+        
+        // This is jank. But it was to get it production ready.
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
+        
+        int counter = 0;
+        int counter2 = 0;
         
         for (int i = 0; i < threads; i++)
         {
@@ -59,15 +79,32 @@ public class IpScanner
                 Handle = handle
             };
             
-            waitHandles[i] = handle;
+            if (i < 64)
+            {
+                waitHandle1[counter] = handle;
+                counter++;
+            }
+            else
+            {
+                waitHandle2[counter2] = handle;
+                counter2++;
+            }
             
             Thread f = new (Scan!);
             f.Start(scanSettings);
             
             Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(1000);
+            Thread.Sleep(100);
         }
         
+        List<WaitHandle[]> waitHandles = new();
+        
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
+        
+        waitHandles.Add(waitHandle1);
+        waitHandles.Add(waitHandle2);
+        
         return waitHandles;
     }
     
@@ -95,6 +132,7 @@ public class IpScanner
             fourthByte = resumeNow.FourthByte;
         }
         
+        // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
         
@@ -127,16 +165,22 @@ public class IpScanner
                             resumeObject.FourthByte = l;
                             break;
                         }
-                        
+
-                        string ip = $"{i}.{j}.{k}.{l}";
+                        Ip ip = new()
-                        
+                        {
+                            Ip1 = i,
+                            Ip2 = j,
+                            Ip3 = k,
+                            Ip4 = l
+                        };
+
                         IPStatus responseCode = IPStatus.Unknown;
                         
                         try
                         {
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
-                            _ = IPAddress.TryParse(ip, out IPAddress? address);
+                            _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             if (address is not null)
                             {
                                 responseCode = ping.Send(address, _timeout, buf, null).Status;
@@ -153,7 +197,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        (int, int) ports = TcpClientHelper.CheckPort(ip, 80, 443);
+                        (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
                         
                         if (ports is { Item1: 0, Item2: 0 })
                         {
@@ -161,7 +205,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        _queue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode, ports));
+                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
                     }
                     
                     if (_stop)
@@ -183,49 +227,39 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
+            
+            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
-        QueueItem resume = new()
+        _resumeQueue.Enqueue(resumeObject);
-        {
-            ResumeObject = resumeObject,
-            Operations = Operations.Insert
-        };
-        
-        _queue.Enqueue(resume);
         
         scanSettings.Handle!.Set();
     }
     
-    private static Discarded CreateDiscardedQueueItem(string ip, int responseCode)
+    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
-        Discarded discarded = new()
+        return new()
         {
             Ip = ip,
             ResponseCode = responseCode
         };
-        
-        return discarded;
     }
     
-    private static QueueItem CreateUnfilteredQueueItem(string ip, int responseCode, (int, int) ports)
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
     {
         Unfiltered unfiltered = new()
         {
             Ip = ip,
-            ResponseCode = responseCode,
             Port1 = ports.Item1,
             Port2 = ports.Item2,
-            Filtered = 0
+            Filtered = false
         };
         
-        QueueItem superUnfilteredObject = new()
+        return new()
         {
             Unfiltered = unfiltered,
             Operations = Operations.Insert
         };
-        
-        return superUnfilteredObject;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -17,12 +17,14 @@ public class ThreadHandler
     
     public ThreadHandler(string path)
     {
-        ConcurrentQueue<QueueItem> contentQueue = new();
+        ConcurrentQueue<Filtered> filteredQueue = new();
         ConcurrentQueue<Discarded> discardedQueue = new();
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
+        ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
         
-        _dbHandler = new(contentQueue, discardedQueue, path);
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-        _ipScanner = new(contentQueue, _dbHandler, discardedQueue);
+        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
-        _contentFilter = new(contentQueue, _dbHandler, path);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
         _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
     }
     
@@ -32,28 +34,37 @@ public class ThreadHandler
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
+        Thread filtered = new(StartFilteredDbHandler);
+        Thread resume = new(StartResumeDbHandler);
         Thread communication = new(StartCommunicationHandler);
         
         scanner.Start();
         indexer.Start();
         database.Start();
         discarded.Start();
+        filtered.Start();
+        resume.Start();
         communication.Start();
         
         scanner.Join();
         indexer.Join();
         database.Join();
         discarded.Join();
+        filtered.Join();
+        resume.Join();
         communication.Join();
     }
     
     private void StartScanner()
     {
-        Thread.Sleep(10000); // Let the database handler instantiate and warm up first.
+        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        WaitHandle[] wait = _ipScanner.Start(4);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
-        
+
-        WaitHandle.WaitAll(wait);
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
         
         Console.WriteLine("Scanner finished");
         
@@ -62,6 +73,8 @@ public class ThreadHandler
     
     private void StartContentFilter()
     {
+        Thread.Sleep(5000);
+        
         WaitHandle[] wait = _contentFilter.Start();
         
         WaitHandle.WaitAll(wait);
@@ -73,12 +86,22 @@ public class ThreadHandler
     
     private void StartDbHandler()
     {
-        _dbHandler.StartContent();
+        _dbHandler.UnfilteredDbHandler();
+    }
+    
+    private void StartFilteredDbHandler()
+    {
+        _dbHandler.FilteredDbHandler();
+    }
+    
+    private void StartResumeDbHandler()
+    {
+        _dbHandler.ResumeDbHandler();
     }
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(2);
+        WaitHandle[] wait = _dbHandler.Start(4);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -1,8 +1,18 @@
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
+
 namespace Backend.Helper;
 
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
     {
         using HttpClient client = new();
         
@@ -16,8 +26,10 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
-        HttpResponseMessage? response = null;
+        HttpResponseMessage? response;
         
         try
         {
@@ -25,15 +37,48 @@ public static class HttpClientHelper
         }
         catch
         {
-            //
+            return ("", "");
         }
         
-        if (response is null || !response.IsSuccessStatusCode)
+        if (!response.IsSuccessStatusCode)
         {
-            return "";
+            return ("", "");
+        }
+
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
         }
         
-        return await response.Content.ReadAsStringAsync();
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
     }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -50,12 +95,14 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
         {
-            response = await client.GetAsync("/robots.txt");
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
         {
@@ -64,4 +111,9 @@ public static class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }

Backend/Sql/DropAndCreateFiltered.sql

@@ -1,37 +0,0 @@
-VACUUM;
-
-DROP TABLE Filtered;
-
-CREATE TABLE "Filtered" (
-	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
-	"Port1"	INTEGER NOT NULL,
-	"Port2"	INTEGER NOT NULL,
-	"Title1"	TEXT NOT NULL,
-	"Title2"	TEXT NOT NULL,
-	"Description1"	TEXT NOT NULL,
-	"Description2"	TEXT NOT NULL,
-	"Url1"	TEXT NOT NULL,
-	"Url2"	TEXT NOT NULL,
-	"ServerType1"	TEXT NOT NULL,
-	"ServerType2"	TEXT NOT NULL,
-	"RobotsTXT1"	TEXT NOT NULL,
-	"RobotsTXT2"	TEXT NOT NULL,
-	"HttpVersion1"	TEXT NOT NULL,
-	"HttpVersion2"	TEXT NOT NULL,
-	"CertificateIssuerCountry"	TEXT NOT NULL,
-	"CertificateOrganizationName"	TEXT NOT NULL,
-	"IpV6"	TEXT NOT NULL,
-	"TlsVersion"	TEXT NOT NULL,
-	"CipherSuite"	TEXT NOT NULL,
-	"KeyExchangeAlgorithm"	TEXT NOT NULL,
-	"PublicKeyType1"	TEXT NOT NULL,
-	"PublicKeyType2"	TEXT NOT NULL,
-	"PublicKeyType3"	TEXT NOT NULL,
-	"AcceptEncoding1"	TEXT NOT NULL,
-	"AcceptEncoding2"	TEXT NOT NULL,
-	"ALPN"	TEXT NOT NULL,
-	"Connection1" TEXT NOT NULL,
-	"Connection2" TEXT NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
-)

Backend/Sql/DropAndCreateMyDB.sql

@@ -1,19 +0,0 @@
-DROP TABLE Unfiltered;
-DROP TABLE Runtimes;
-
-CREATE TABLE "Unfiltered" (
-	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
-	"ResponseCode"	INTEGER NOT NULL,
-	"Port1"	INTEGER NOT NULL,
-	"Port2"	INTEGER NOT NULL,
-	"Filtered"	INTEGER NOT NULL,
-	CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT)
-);
-
-CREATE TABLE "Runtimes" (
-    "Id" INTEGER NOT NULL CONSTRAINT "PK_Runtimes" PRIMARY KEY AUTOINCREMENT,
-    "StartTime" TEXT NOT NULL,
-    "EndTime" TEXT NOT NULL,
-    "ThreadNumber" INTEGER NOT NULL
-);

Models/Handler/DbHandler.cs

@@ -1,5 +1,7 @@
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using Microsoft.Data.Sqlite;
+using Models.Helper;
 using Models.Model.Backend;
 using Models.Model.External;
 
@@ -7,27 +9,85 @@ namespace Models.Handler;
 
 public class DbHandler
 {
-    private readonly ConcurrentQueue<QueueItem> _contentQueue;
+    private readonly ConcurrentQueue<Filtered> _filteredQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
+    private readonly string _compressedConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
-    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)";
+    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
+                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)";
+                                           " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
-    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
+    
+    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
+                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
+                                              " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
+                                              " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
+                                              " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
+                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
+                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
+                                              " @robotsTXT1, @robotsTXT2," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion1)," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion2)," +
+                                              " (SELECT CertificateIssuerId FROM CertificateIssuerCountry WHERE Country = @certificateIssuerCountry)," +
+                                              " (SELECT CertificateOrganizationId FROM CertificateOrganizationName WHERE Name = @certificateOrganizationName), " +
+                                              " @ipV6, " +
+                                              " (SELECT TlsId FROM TlsVersion WHERE Version = @tlsVersion)," +
+                                              " (SELECT CipherId FROM CipherSuite WHERE Suite = @cipherSuite)," +
+                                              " (SELECT KeyExchangeId FROM KeyExchangeAlgorithm WHERE Algorithm = @keyExchangeAlgorithm)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType1)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType2)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType3)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding1)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding2)," +
+                                              " (SELECT ALPNId FROM ALPN WHERE ALPNValue = @aLPN)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection1)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection2))";
+
+    private const string InsertIntoFilteredServerType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ServerType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredHttpVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO HttpVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCertificateIssuerCountry = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateIssuerCountry (Country) VALUES (@country)";
+    private const string InsertIntoFilteredCertificateOrganizationName = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateOrganizationName (Name) VALUES (@name)";
+    private const string InsertIntoFilteredTlsVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO TlsVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCipherSuite = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CipherSuite (Suite) VALUES (@suite)";
+    private const string InsertIntoFilteredKeyExchangeAlgorithm = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO KeyExchangeAlgorithm (Algorithm) VALUES (@algorithm)";
+    private const string InsertIntoFilteredPublicKeyType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO PublicKeyType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredAcceptEncoding = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO AcceptEncoding (Encoding) VALUES (@encoding)";
+    private const string InsertIntoFilteredALPN = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ALPN (ALPNValue) VALUES (@alpnValue)";
+    private const string InsertIntoFilteredConnection = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO Connection (ConnectionValue) VALUES (@connectionValue)";
+    
+    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
+                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+    
+    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+
+    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
-    private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
+    private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
     private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;";
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
-    private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;";
+    private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
@@ -41,26 +101,32 @@ public class DbHandler
     private bool _stop;
     private bool _pause;
     private bool _paused;
+    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
 
     private readonly string _basePath;
     
-    public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
+    public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
+        ConcurrentQueue<Discarded> discardedQueue, 
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
     {
-        _contentQueue = contentQueue;
+        _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
-        
+        _unfilteredQueue = unfilteredQueue;
-        SetContentWaitTime(10);
+        _resumeQueue = resumeQueue;
+
+        SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
         
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
-        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
+        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -73,41 +139,78 @@ public class DbHandler
         _discardedWaitTime = waitTime;
     }
     
-    public void StartContent()
+    public void UnfilteredDbHandler()
     {
-        Console.WriteLine("Content DbHandler started");
+        Console.WriteLine("Unfiltered DbHandler started");
         
         while (!_stop)
         {
-            if (_contentQueue.IsEmpty || _pause)
+            if (_unfilteredQueue.IsEmpty || _pause)
             {
                 Thread.Sleep(_contentWaitTime);
                 _paused = true;
                 continue;
             }
             
-            _contentQueue.TryDequeue(out QueueItem? queueItem);
+            _unfilteredQueue.TryDequeue(out UnfilteredQueueItem queueItem);
             
-            if (queueItem is null) { continue; }
+            if (queueItem.Operations == Operations.Insert)
-            
-            switch (queueItem.Operations)
             {
-                case Operations.Insert when queueItem.Unfiltered is not null:
+                InsertUnfiltered(queueItem.Unfiltered);
-                    InsertUnfiltered(queueItem.Unfiltered);
+            }
-                    break;
+            
-                case Operations.Insert when queueItem.Filtered is not null:
+            else if (queueItem.Operations == Operations.Update)
-                    InsertFiltered(queueItem.Filtered);
+            {
-                    break;
+                UpdateUnfiltered(queueItem.Unfiltered);
-                case Operations.Insert when queueItem.ResumeObject is not null:
-                    InsertResumeObject(queueItem.ResumeObject);
-                    break;
-                case Operations.Update when queueItem.Unfiltered is not null:
-                    UpdateUnfiltered(queueItem.Unfiltered);
-                    break;
             }
         }
         
-        Console.WriteLine("Content DbHandler stopped.");
+        Console.WriteLine("Unfiltered DbHandler stopped.");
+    }
+    
+    public void FilteredDbHandler()
+    {
+        Console.WriteLine("Filtered DB handler started");
+        
+        while (!_stop)
+        {
+            if (_filteredQueue.IsEmpty || _pause)
+            {
+                Thread.Sleep(_contentWaitTime);
+                _paused = true;
+                continue;
+            }
+            
+            _filteredQueue.TryDequeue(out Filtered? queueItem);
+            
+            InsertFiltered(queueItem!);
+        }
+        
+        Console.WriteLine("Filtered DbHandler stopped.");
+    }
+    
+    public void ResumeDbHandler()
+    {
+        Console.WriteLine("Resume DB handler started");
+        
+        while (!_stop)
+        {
+            if (_resumeQueue.IsEmpty || _pause)
+            {
+                Thread.Sleep(_contentWaitTime);
+                _paused = true;
+                continue;
+            }
+            
+            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
+            
+            if (queueItem is not null)
+            {
+                InsertResumeObject(queueItem);
+            }
+        }
+        
+        Console.WriteLine("Resume DbHandler stopped.");
     }
     
     public WaitHandle[] Start(int threads)
@@ -117,7 +220,7 @@ public class DbHandler
         for (int i = 0; i < threads; i++)
         {
             EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-
+            
             DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
             {
                 Handle = handle,
@@ -126,21 +229,23 @@ public class DbHandler
             
             waitHandles[i] = handle;
             
-            Thread f = new (RunDiscarded!);
+            Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(1000);
+            Thread.Sleep(5000);
         }
         
         return waitHandles;
     }
     
-    private void RunDiscarded(object obj)
+    private void DiscardedDbHandler(object obj)
     {
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        
+        int i = 0;
         
         while (!_stop)
         {
@@ -151,16 +256,57 @@ public class DbHandler
                 continue;
             }
             
-            _discardedQueue.TryDequeue(out Discarded? queueItem);
+            if (i >= 50_000_000 && !_compressing)
+            {
+                _compressing = true;
+                
+                i = 0;
+                
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                
+                _compressing = false;
+            }
             
-            if (queueItem is null) { continue; }
+            _discardedQueue.TryDequeue(out Discarded queueItem);
             
             InsertDiscarded(queueItem, connectionString);
+            
+            i++;
         }
         
         discardedDbHandlerSetting.Handle!.Set();
         
-        Console.WriteLine("Content DbHandler stopped.");
+        Console.WriteLine("Discarded DbHandler stopped.");
+    }
+    
+    private void DropAndCreateDiscarded(int threadNumber)
+    {
+        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
+        
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string dropStatement = "DROP TABLE Discarded;";
+        const string vacuum = "VACUUM;";
+        
+        using SqliteConnection connection = new(databaseName);
+        connection.Open();
+        
+        SqliteCommand command = new(dropStatement, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(vacuum, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(createStatement, connection);
+        command.ExecuteNonQuery();
+        command.Dispose();
+        
+        connection.Close();
     }
     
     private void InsertUnfiltered(Unfiltered unfiltered)
@@ -170,8 +316,10 @@ public class DbHandler
         
         using SqliteCommand command = new(InsertStatement, connection);
         
-        command.Parameters.AddWithValue("@ip", unfiltered.Ip);
+        command.Parameters.AddWithValue("@ip1", unfiltered.Ip.Ip1);
-        command.Parameters.AddWithValue("@responseCode", unfiltered.ResponseCode);
+        command.Parameters.AddWithValue("@ip2", unfiltered.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", unfiltered.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", unfiltered.Ip.Ip4);
         command.Parameters.AddWithValue("@port1", unfiltered.Port1);
         command.Parameters.AddWithValue("@port2", unfiltered.Port2);
         command.Parameters.AddWithValue("@filtered", unfiltered.Filtered);
@@ -187,7 +335,10 @@ public class DbHandler
         
         using SqliteCommand command = new(InsertIntoDiscarded, connection);
         
-        command.Parameters.AddWithValue("@ip", discarded.Ip);
+        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
         command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
         _ = command.ExecuteNonQuery();
         
@@ -199,9 +350,79 @@ public class DbHandler
         using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
-        using SqliteCommand command = new(InsertIntoFiltered, connection);
+        SqliteCommand command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType1);
+        _ = command.ExecuteNonQuery();
         
-        command.Parameters.AddWithValue("@ip", filtered.Ip);
+        command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateIssuerCountry, connection);
+        command.Parameters.AddWithValue("@country", filtered.CertificateIssuerCountry);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateOrganizationName, connection);
+        command.Parameters.AddWithValue("@name", filtered.CertificateOrganizationName);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredTlsVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.TlsVersion);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCipherSuite, connection);
+        command.Parameters.AddWithValue("@suite", filtered.CipherSuite);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredKeyExchangeAlgorithm, connection);
+        command.Parameters.AddWithValue("@algorithm", filtered.KeyExchangeAlgorithm);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType3);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredALPN, connection);
+        command.Parameters.AddWithValue("@alpnValue", filtered.ALPN);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFiltered, connection);
+        command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", filtered.Ip.Ip4);
         command.Parameters.AddWithValue("@port1", filtered.Port1);
         command.Parameters.AddWithValue("@port2", filtered.Port2);
         command.Parameters.AddWithValue("@url1", filtered.Url1);
@@ -230,8 +451,9 @@ public class DbHandler
         command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
         command.Parameters.AddWithValue("@connection1", filtered.Connection1);
         command.Parameters.AddWithValue("@connection2", filtered.Connection2);
-        
         _ = command.ExecuteNonQuery();
+        command.Dispose();
+        
         connection.Close();
     }
     
@@ -253,6 +475,20 @@ public class DbHandler
         _ = command.ExecuteNonQuery();
         connection.Close();
     }
+
+    private void InsertCompressedDatabase(int threadNumber, long rows)
+    {
+        using SqliteConnection connection = new(_compressedConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
+        
+        command.Parameters.AddWithValue("@dbNumber", threadNumber);
+        command.Parameters.AddWithValue("@rows", rows);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+    }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
@@ -267,7 +503,7 @@ public class DbHandler
         connection.Close();
     }
     
-    public Unfiltered? ReadUnfilteredWithId(long id)
+    public Unfiltered ReadUnfilteredWithId(long id)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
         connection.Open();
@@ -276,26 +512,30 @@ public class DbHandler
         command.Parameters.AddWithValue("@id", id);
         
         using SqliteDataReader reader = command.ExecuteReader();
-        if (!reader.HasRows) return null;
+        if (!reader.HasRows) return new();
         
         Unfiltered unfiltered = new();
+        Ip ip = new();
         
         while (reader.Read())
         {
             unfiltered.Id = reader.GetInt32(0);
-            unfiltered.Ip = reader.GetString(1);
+            ip.Ip1 = reader.GetInt32(1);
-            unfiltered.Port1 = reader.GetInt32(3);
+            ip.Ip2 = reader.GetInt32(2);
-            unfiltered.Port2 = reader.GetInt32(4);
+            ip.Ip3 = reader.GetInt32(3);
-            unfiltered.Filtered = reader.GetInt32(5);
+            ip.Ip4 = reader.GetInt32(4);
+            unfiltered.Port1 = reader.GetInt32(5);
+            unfiltered.Port2 = reader.GetInt32(6);
+            unfiltered.Filtered = reader.GetBoolean(7);
         }
         
+        unfiltered.Ip = ip;
+        
         return unfiltered;
     }
     
-    public long GetUnfilteredIndexes()
+    public List<long> GetUnfilteredIndexes()
     {
-        long rowId = 0;
-        
         using SqliteConnection connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -304,15 +544,17 @@ public class DbHandler
         
         if (!reader.HasRows)
         {
-            return 0;
+            return [];
         }
+
+        List<long> ids = [];
         
         while (reader.Read())
         {
-            rowId = reader.GetInt64(0);
+            ids.Add(reader.GetInt64(0));
         }
         
-        return rowId;
+        return ids;
     }
     
     public long GetFilteredIndexes()
@@ -341,36 +583,88 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
+
+        SqliteConnection connection;
+        SqliteCommand command;
+        SqliteDataReader reader;
         
         for (int i = 0; i < _discardedConnectionStrings.Count; i++)
         {
-            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
+            connection = new(_discardedConnectionStrings[i]);
             connection.Open();
             
-            using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+            command = new(ReadDiscardedSeqIdsStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
+            reader = command.ExecuteReader();
             
             if (!reader.HasRows)
             {
-                return 0;
+                return rowId;
             }
             
             while (reader.Read())
             {
                 rowId += reader.GetInt64(0);
             }
+            
+            connection.Close();
         }
         
+        connection = new(_compressedConnectionString);
+        connection.Open();
+        command = new(ReadCompressedDbRowsStatement, connection);
+        reader = command.ExecuteReader();
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        reader.Dispose();
+        
         return rowId;
     }
 
-    public bool GetFilteredIp(string ip)
+    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    {
+        using SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+        using SqliteDataReader reader = command.ExecuteReader();
+        
+        long rowId = 0;
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+
+        return rowId;
+    }
+
+    public bool FilteredIpExists(Ip ip)
     {
         using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         using SqliteCommand command = new(ReadFilteredIpStatement, connection);
-        command.Parameters.AddWithValue("@ip", ip);
+        command.Parameters.AddWithValue("@ip1", ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", ip.Ip4);
         
         using SqliteDataReader reader = command.ExecuteReader();
         
@@ -379,14 +673,37 @@ public class DbHandler
             return false;
         }
         
-        string ipAddress = "";
+        Ip tempIp = new();
         
         while (reader.Read())
         {
-            ipAddress = reader.GetString(0);
+            tempIp.Ip1 = reader.GetInt32(0);
+            tempIp.Ip2 = reader.GetInt32(1);
+            tempIp.Ip3 = reader.GetInt32(2);
+            tempIp.Ip4 = reader.GetInt32(3);
         }
         
-        return ipAddress == ip;
+        if (tempIp.Ip1 != ip.Ip1)
+        {
+            return false;
+        }
+        
+        if (tempIp.Ip2 != ip.Ip2)
+        {
+            return false;
+        }
+        
+        if (tempIp.Ip3 != ip.Ip3)
+        {
+            return false;
+        }
+        
+        if (tempIp.Ip4 != ip.Ip4)
+        {
+            return false;
+        }
+        
+        return true;
     }
     
     public List<SearchResult?> GetSearchResults()
@@ -463,20 +780,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(ReIndexDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -502,20 +812,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(VacuumDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -530,12 +833,13 @@ public class DbHandler
         _paused = false;
     }
 
-    private string CreateDiscardedDb(int threadNumber)
+    private (string, string) CreateDiscardedDb(int threadNumber)
     {
+        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-
+        
         _discardedConnectionStrings.Add(databaseName);
         
         using SqliteConnection connection = new(databaseName);
@@ -544,7 +848,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return databaseName;
+        return (absolutePath, databaseName);
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -0,0 +1,22 @@
+using System.IO.Compression;
+
+namespace Models.Helper;
+
+public static class CompressionHelper
+{
+    public static void CompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
+        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
+        originalFileStream.CopyTo(compressor);
+    }
+    
+    public static void DecompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
+        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
+        decompressor.CopyTo(decompressedFileStream);
+    }
+}

Models/Helper/GetDatabasesHelper.cs

@@ -0,0 +1,9 @@
+namespace Models.Helper;
+
+public static class GetDatabasesHelper
+{
+    public static int GetTotalCompressedDatabases(string path)
+    {
+        return Directory.GetFiles(path, "*.gz").Length;
+    }
+}

Models/Model/Backend/Discarded.cs

@@ -1,7 +1,8 @@
 namespace Models.Model.Backend;
 
-public class Discarded
+public struct Discarded
 {
-    public string Ip { get; set; } = "";
+    public Ip Ip { get; set; }
+    
     public int ResponseCode { get; set; }
 }

Models/Model/Backend/Filtered.cs

@@ -2,7 +2,7 @@ namespace Models.Model.Backend;
 
 public class Filtered
 {
-    public string Ip { get; set; } = "";
+    public Ip Ip { get; set; }
     public string Title1 { get; set; } = "";
     public string Title2 { get; set; } = "";
     public string Description1 { get; set; } = "";

Models/Model/Backend/IP.cs

@@ -0,0 +1,17 @@
+namespace Models.Model.Backend;
+
+public struct Ip
+{
+    public int Ip1 { get; set; }
+    
+    public int Ip2 { get; set; }
+    
+    public int Ip3 { get; set; }
+    
+    public int Ip4 { get; set; }
+    
+    public override string ToString()
+    {
+        return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
+    }
+}

Models/Model/Backend/Operations.cs

@@ -4,5 +4,4 @@ public enum Operations
 {
     Insert,
     Update,
-    Optimize,
 }

Models/Model/Backend/QueueItem.cs

@@ -1,9 +0,0 @@
-namespace Models.Model.Backend;
-
-public class QueueItem
-{
-    public Unfiltered? Unfiltered { get; init; }
-    public Filtered? Filtered { get; init; }
-    public ScannerResumeObject? ResumeObject { get; init; }
-    public Operations Operations { get; init; }
-}

Models/Model/Backend/Unfiltered.cs

@@ -1,16 +1,14 @@
 namespace Models.Model.Backend;
 
-public class Unfiltered
+public struct Unfiltered
 {
     public int Id { get; set; }
-
+    
-    public string Ip { get; set; } = "";
+    public Ip Ip { get; set; }
-
+    
-    public int ResponseCode { get; init; }
-
     public int Port1 { get; set; }
-
+    
     public int Port2 { get; set; }
-
+    
-    public int Filtered { get; set; }
+    public bool Filtered { get; set; }
 }

Models/Model/Backend/UnfilteredQueueItem.cs

@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public struct UnfilteredQueueItem
+{
+    public Unfiltered Unfiltered { get; init; }
+    public Operations Operations { get; init; }
+}

Models/Models.csproj

@@ -8,6 +8,5 @@
 
     <ItemGroup>
       <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
-      <PackageReference Include="SQLite" Version="3.13.0" />
     </ItemGroup>
 </Project>

Models/Models.csproj.DotSettings

@@ -1,3 +0,0 @@
-<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cbackend/@EntryIndexedValue">False</s:Boolean>
-	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cinternal/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

Models/Sql/DropAndCreateFiltered.sql

@@ -0,0 +1,112 @@
+VACUUM;
+
+CREATE TABLE IF NOT EXISTS "ServerType" (
+    "ServerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "HttpVersion" (
+    "HttpId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateIssuerCountry" (
+	"CertificateIssuerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Country" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateOrganizationName" (
+	"CertificateOrganizationId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Name" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "TlsVersion" (
+	"TlsId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CipherSuite" (
+	"CipherId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Suite" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "KeyExchangeAlgorithm" (
+	"KeyExchangeId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Algorithm" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "PublicKeyType" (
+	"PublicKeyId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "AcceptEncoding" (
+	"AcceptId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Encoding" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "ALPN" (
+	"ALPNId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ALPNValue" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "Connection" (
+	"ConnectionId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ConnectionValue" TEXT NOT NULL UNIQUE
+);
+
+DROP TABLE Filtered;
+
+CREATE TABLE "Filtered" (
+	"Id"	INTEGER NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"	INTEGER NOT NULL,
+	"Ip3"	INTEGER NOT NULL,
+	"Ip4"	INTEGER NOT NULL,
+	"Port1"	INTEGER NOT NULL,
+	"Port2"	INTEGER NOT NULL,
+	"Title1"	TEXT NOT NULL,
+	"Title2"	TEXT NOT NULL,
+	"Description1"	TEXT NOT NULL,
+	"Description2"	TEXT NOT NULL,
+	"Url1"	TEXT NOT NULL,
+	"Url2"	TEXT NOT NULL,
+	"ServerType1"	INTEGER NOT NULL,
+	"ServerType2"	INTEGER NOT NULL,
+	"RobotsTXT1"	TEXT NOT NULL,
+	"RobotsTXT2"	TEXT NOT NULL,
+	"HttpVersion1"	INTEGER NOT NULL,
+	"HttpVersion2"	INTEGER NOT NULL,
+	"CertificateIssuerCountry"	INTEGER NOT NULL,
+	"CertificateOrganizationName"	INTEGER NOT NULL,
+	"IpV6"	TEXT NOT NULL,
+	"TlsVersion"	INTEGER NOT NULL,
+	"CipherSuite"	INTEGER NOT NULL,
+	"KeyExchangeAlgorithm"	INTEGER NOT NULL,
+	"PublicKeyType1"	INTEGER NOT NULL,
+	"PublicKeyType2"	INTEGER NOT NULL,
+	"PublicKeyType3"	INTEGER NOT NULL,
+	"AcceptEncoding1"	INTEGER NOT NULL,
+	"AcceptEncoding2"	INTEGER NOT NULL,
+	"ALPN"	INTEGER NOT NULL,
+	"Connection1"	INTEGER NOT NULL,
+	"Connection2"	INTEGER NOT NULL,
+	PRIMARY KEY("Id" AUTOINCREMENT),
+	FOREIGN KEY("ALPN") REFERENCES "ALPN"("ALPNId"),
+	FOREIGN KEY("AcceptEncoding1") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("AcceptEncoding2") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("CertificateIssuerCountry") REFERENCES "CertificateIssuerCountry"("CertificateIssuerId"),
+	FOREIGN KEY("CertificateOrganizationName") REFERENCES "CertificateOrganizationName"("CertificateOrganizationId"),
+	FOREIGN KEY("CipherSuite") REFERENCES "CipherSuite"("CipherId"),
+	FOREIGN KEY("Connection1") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("Connection2") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("HttpVersion1") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("HttpVersion2") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("KeyExchangeAlgorithm") REFERENCES "KeyExchangeAlgorithm"("KeyExchangeId"),
+	FOREIGN KEY("PublicKeyType1") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType2") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType3") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("ServerType1") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("ServerType2") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("TlsVersion") REFERENCES "TlsVersion"("TlsId")
+)

Models/Sql/DropAndCreateMyDB.sql

@@ -0,0 +1,13 @@
+DROP TABLE Unfiltered;
+
+CREATE TABLE "Unfiltered" (
+	"Id"	INTEGER NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"   INTEGER NOT NULL,
+	"Ip3"   INTEGER NOT NULL,
+	"Ip4"   INTEGER NOT NULL,
+	"Port1"	INTEGER NOT NULL,
+	"Port2"	INTEGER NOT NULL,
+	"Filtered"	INTEGER NOT NULL,
+	CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT)
+)

Proxy/Program.cs

@@ -1,10 +1,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using MessagePack;
+using AspNetCoreRateLimit;
-using Microsoft.AspNetCore.Cors.Infrastructure;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
 
 WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
 
@@ -15,18 +17,29 @@ builder.Services.ConfigureHttpJsonOptions(options =>
 
 builder.Services.AddCors(options =>
 {
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
 WebApplication app = builder.Build();
 
-app.UseCors();
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -40,11 +53,15 @@ progressApi.MapGet("/", () =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });
 
+/*
 RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
 searchApi.MapGet("/{term}", (string term) =>
 {
     CommunicationObject communicationObject = new();
@@ -61,11 +78,11 @@ searchApi.MapGet("/{term}", (string term) =>
     
     return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-
+*/
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
+//[JsonSerializable(typeof(SearchResults))]
 [JsonSerializable(typeof(CommunicationObject))]
 internal partial class AppJsonSerializerContext : JsonSerializerContext
 {

Proxy/Proxy.csproj

@@ -9,6 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
         <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
 

Proxy/appsettings.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
       "Microsoft.AspNetCore": "Warning"
     }
   },

RSE.sln

@@ -8,8 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -32,9 +30,5 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,29 +1,15 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000006pdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fd17d4b69379a42eb90f15b17ef6c846a5400_003F39_003F1eeed291_003F02000006pdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F096da63313a8436fb75089601a728c765200_003Fbb_003Ff6383783_003F02000007pdb1Low_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F357247e747614c8297021dd08d79da3d5200_003Ff2_003Fc4ae8c87_003F02000007pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000010pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F80cf8253b0c2409096d0c0e368300af28200_003Fdd_003F8098fc19_003F02000010pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb4Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F1f6cfdd1e3a14f6390237f6ab98b06af8000_003F0d_003Febdca535_003F02000011pdb4Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttributes_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9a4cb3a98df697f884fa5f45bc79e4291fa3e948c7f42560e14678e12055e_003FAttributes_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttribute_002ECoreCLR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbb9be2cb8efb72a7d2286fbdd10a293b55b3e2a912f49fac6a7719673268e4b_003FAttribute_002ECoreCLR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConsole_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2ccd2056ec55b7b67558d52e32a887ed5ac7e346fc429b218c188d0a99cb5be_003FConsole_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5029ca69753a60cb407b1053f5834320dadee066_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F99b5d033f8bfb58a6a753ca88a3d956f2228bd48_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F13cbfe6856867dd1ed4e39575e46d816dae2a146a8ceec8f76b5897f5e0fe_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpContent_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a55e5df9328b51021ba85b8f12ff49eca8772dba9772c0d61d5e28edf255c_003FHttpContent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpMethodAttribute_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2e4732e712a98f9ac0d080ac68669b3db6d98781e0d5ad89886cb6bcc890b18_003FHttpMethodAttribute_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIFormatterResolver_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc7d856f45ddf907f9face6c2bdba7836cc70fed4bb2b8ebd83ee6917584af3_003FIFormatterResolver_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIOThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F125fae6d49819da7be4fb21fbb4a936a7ec325971cf9264a24af55bf7111ad8_003FIOThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8fe420eaf60c4dfca87ce1d5f1cdfa4816200_003Fbd_003F71bcce16_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5cffb8c94881849b6aa8abe7ba5e7cafe05d991f859903020362d6aac371_003FMessagePackReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe5f8b0187e62d74a4b9fefd657c192dab367bb342ef5ffbd90a4ed2ea428ec7a_003FMessagePackSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AOutgoingSocketExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc4824d483bf85fa615ad2125f13f6d75788d45d1510ec1cfb8226831da54f_003FOutgoingSocketExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F77bf2fbffc5491eadd62f1dbebc233798cd7fcb9b39ab7ee3bb35519f4d94ecc_003FPingOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002ERawSocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2f645da43e51b8be94c9217511b45c23384f041ffa9aad041f0ddc158d732f0_003FPing_002ERawSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8d57f5f5fd3290d6a89a5b767ad89988dd893c988eba430cd461b8b88d7ad9d_003FPing_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARep_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Feea8e921c916164ccc376e162da148b8215450dfae96e08bdd29119165c67f_003FRep_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AReq_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F33fafcc2bb6e91ae8abb5a52936d39cd93951ad9208861e758ca93efa619eaf3_003FReq_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATask_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F35f3a54f5acb408a3e219b2de039f1a39557b7e4515f11238cba07b60c0ce_003FTask_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATextWriter_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdda89b2ed0975050b6847325357a756a5866a5435e3ddb4feff535ba36facb7_003FTextWriter_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}

Shared/Shared.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-</Project>

frontend/.gitignore

@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example

frontend/README.md

@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.

frontend/app.vue

@@ -0,0 +1,16 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <NuxtLayout>
+    <NuxtPage page-key="static" />
+  </NuxtLayout>
+
+</template>

frontend/assets/css/main.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/nuxt.config.ts

@@ -0,0 +1,17 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});

frontend/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}

frontend/pages/index.vue

@@ -0,0 +1,12 @@
+<script setup lang="ts">
+const route = useRoute()
+</script>
+
+<template>
+  <div class="h-screen flex items-center justify-center">
+    <div class="flex space-x-2">
+      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
+    </div>
+  </div>
+</template>

frontend/pages/progress.vue

@@ -0,0 +1,64 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <div v-if="dict">
+      <table class="table-auto border-gray-300">
+        <thead>
+          <tr>
+            <th class="px-4 py-2">Metric</th>
+            <th class="px-4 py-2">Value</th>
+          </tr>
+        </thead>
+
+        <tbody>
+          <tr v-for="d in dict">
+            <td class="px-4 py-2">{{d.description}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+
+</style>

frontend/public/robots.txt

@@ -0,0 +1 @@
+

frontend/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}

frontend/tailwind.config.js

@@ -0,0 +1,22 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {
+      width: {
+        '128': '32rem',
+        '192' : '48rem',
+        '256': '64rem',
+      }
+    },
+  },
+  plugins: [],
+}
+

frontend/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}

frontend/types/global.d.ts

@@ -0,0 +1,19 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+}
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+}
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};

frontend/utils/cacheUtil.ts

@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}

frontend/utils/convertProgressTypeToDictionary.ts

@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}