Add user-agent

Merge pull request 'Add cached progress response' (#36 ) from ImplementAPICache into main
Reviewed-on: #36
2024-12-21 21:57:01 +01:00 · 2024-12-19 17:37:01 +00:00 · 2024-12-19 18:36:13 +01:00 · 2024-12-18 13:28:03 +00:00 · 2024-12-18 14:27:31 +01:00 · 2024-12-18 12:02:35 +01:00
43 changed files with 12021 additions and 214 deletions
@@ -6,8 +6,8 @@
        <ImplicitUsings>enable</ImplicitUsings>
        <Nullable>enable</Nullable>

-        <Platform>x64</Platform>
-        <Optimize>true</Optimize>
+        <!--<Platform>x64</Platform>-->
+        <Optimize>false</Optimize>
    </PropertyGroup>
    
    <ItemGroup>
@@ -8,7 +8,8 @@ namespace Backend.Handler;

 public class ContentFilter
 {
-    private readonly ConcurrentQueue<QueueItem> _queue;
+    private readonly ConcurrentQueue<Filtered> _queue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
    private readonly DbHandler _dbHandler;
    private readonly string _getDomainPort80;
    private readonly string _getDomainPort443;
@@ -16,11 +17,12 @@ public class ContentFilter
    private int _timeOut;
    private readonly string _basePath;
    
-    public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, string basePath)
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
    {
        _queue = queue;
        _dbHandler = dbHandler;
        _basePath = basePath;
+        _unfilteredQueue = unfilteredQueue;
        
        _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
        _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
@@ -49,13 +51,13 @@ public class ContentFilter
    {
        while (!_stop)
        {
-            long indexes = _dbHandler.GetUnfilteredIndexes();
+            List<long> indexes = _dbHandler.GetUnfilteredIndexes();
            
-            for (long i = 0; i < indexes; i++)
+            for (int i = 0; i < indexes.Count; i++)
            {
                if (_stop) break;
                
-                Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(i);
+                Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                
                if (unfiltered.Filtered) continue;
                
@@ -63,14 +65,13 @@ public class ContentFilter
                
                unfiltered.Filtered = true;
                
-                QueueItem superUnfilteredObject = new()
+                UnfilteredQueueItem superUnfilteredObject = new()
                {
                    Unfiltered = unfiltered,
-                    Operations = Operations.Update,
-                    DbType = DbType.Unfiltered
+                    Operations = Operations.Update
                };
                
-                _queue.Enqueue(superUnfilteredObject);
+                _unfilteredQueue.Enqueue(superUnfilteredObject);
                
                if (_dbHandler.FilteredIpExists(unfiltered.Ip))
                {
@@ -82,14 +83,7 @@ public class ContentFilter
                filtered.Port1 = unfiltered.Port1;
                filtered.Port2 = unfiltered.Port2;
                
-                QueueItem superFilteredObject = new()
-                {
-                    Filtered = filtered,
-                    Operations = Operations.Insert,
-                    DbType = DbType.Filtered
-                };
-                
-                _queue.Enqueue(superFilteredObject);
+                _queue.Enqueue(filtered);
            }
            
            Thread.Sleep(_timeOut);
@@ -167,19 +161,33 @@ public class ContentFilter
        
        for (int i = 0; i < ports.Length; i++)
        {
-            string? html;
+            string? html = "";
            
            if (ports[i] == 80)
            {
                if (string.IsNullOrWhiteSpace(url1)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                }
+                catch
+                {
+                    //
+                }
            }
            else
            {
                if (string.IsNullOrWhiteSpace(url2)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                }
+                catch
+                {
+                    //
+                }
            }
            
            if (string.IsNullOrWhiteSpace(html)) continue;
@@ -188,8 +196,8 @@ public class ContentFilter
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
-            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; }
-            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; }
+            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
+            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
        }
        
        Filtered siteData = new()
@@ -229,7 +237,7 @@ public class ContentFilter
    private void StartProcess(Ip ip, int port)
    {
        string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
-        //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}");
+        
        Process proc = new();
        proc.StartInfo = new()
        {
@@ -17,19 +17,23 @@ public class ScanSettings

 public class IpScanner
 {
-    private readonly ConcurrentQueue<QueueItem> _queue;
    private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
    private readonly DbHandler _dbHandler;
    private bool _stop;
    private int _timeout;
    
-    public IpScanner(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, ConcurrentQueue<Discarded> discardedQueue)
+    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
+        )
    {
-        _queue = queue;
        _dbHandler = dbHandler;
        _discardedQueue = discardedQueue;
-        
-        SetTimeout(128);
+        _unfilteredQueue = unfilteredQueue;
+        _resumeQueue = resumeQueue;
+
+        SetTimeout(64);
    }
    
    public void SetTimeout(int milliseconds)
@@ -37,15 +41,30 @@ public class IpScanner
        _timeout = milliseconds;
    }
    
-    public WaitHandle[] Start(int threads)
+    public List<WaitHandle[]> Start(int threads)
    {
        int threadsAmount = 0;
        if (threads % 2 == 0)
        {
            threadsAmount = 256 / threads;
        }
+
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
        
-        WaitHandle[] waitHandles = new WaitHandle[threads];
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
+        
+        int counter = 0;
+        int counter2 = 0;
        
        for (int i = 0; i < threads; i++)
        {
@@ -58,16 +77,33 @@ public class IpScanner
                ThreadNumber = i,
                Handle = handle
            };
-            
-            waitHandles[i] = handle;
+
+            if (i < 64)
+            {
+                waitHandle1[counter] = handle;
+                counter++;
+            }
+            else
+            {
+                waitHandle2[counter2] = handle;
+                counter2++;
+            }
            
            Thread f = new (Scan!);
            f.Start(scanSettings);
            
            Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(1000);
+            Thread.Sleep(100);
        }
        
+        List<WaitHandle[]> waitHandles = new();
+        
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
+        
+        waitHandles.Add(waitHandle1);
+        waitHandles.Add(waitHandle2);
+        
        return waitHandles;
    }
    
@@ -167,7 +203,7 @@ public class IpScanner
                            continue;
                        }
                        
-                        _queue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
+                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
                    }
                    
                    if (_stop)
@@ -189,32 +225,25 @@ public class IpScanner
                resumeObject.FirstByte = i;
                break;
            }
-            //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
+            
+            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
        }
        
-        QueueItem resume = new()
-        {
-            ResumeObject = resumeObject,
-            Operations = Operations.Insert
-        };
-        
-        _queue.Enqueue(resume);
+        _resumeQueue.Enqueue(resumeObject);
        
        scanSettings.Handle!.Set();
    }
    
    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
    {
-        Discarded discarded = new()
+        return new()
        {
            Ip = ip,
            ResponseCode = responseCode
        };
-        
-        return discarded;
    }
    
-    private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
    {
        Unfiltered unfiltered = new()
        {
@@ -224,14 +253,11 @@ public class IpScanner
            Filtered = false
        };
        
-        QueueItem superUnfilteredObject = new()
+        return new()
        {
            Unfiltered = unfiltered,
-            Operations = Operations.Insert,
-            DbType = DbType.Unfiltered
+            Operations = Operations.Insert
        };
-        
-        return superUnfilteredObject;
    }
    
    public void Stop()
@@ -17,12 +17,14 @@ public class ThreadHandler
    
    public ThreadHandler(string path)
    {
-        ConcurrentQueue<QueueItem> contentQueue = new();
+        ConcurrentQueue<Filtered> filteredQueue = new();
        ConcurrentQueue<Discarded> discardedQueue = new();
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
+        ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
        
-        _dbHandler = new(contentQueue, discardedQueue, path);
-        _ipScanner = new(contentQueue, _dbHandler, discardedQueue);
-        _contentFilter = new(contentQueue, _dbHandler, path);
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
+        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
        _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
    }
    
@@ -32,18 +34,24 @@ public class ThreadHandler
        Thread indexer = new(StartContentFilter);
        Thread database = new(StartDbHandler);
        Thread discarded = new(StartDiscardedDbHandler);
+        Thread filtered = new(StartFilteredDbHandler);
+        Thread resume = new(StartResumeDbHandler);
        Thread communication = new(StartCommunicationHandler);
        
        scanner.Start();
        indexer.Start();
        database.Start();
        discarded.Start();
+        filtered.Start();
+        resume.Start();
        communication.Start();
        
        scanner.Join();
        indexer.Join();
        database.Join();
        discarded.Join();
+        filtered.Join();
+        resume.Join();
        communication.Join();
    }
    
@@ -51,9 +59,12 @@ public class ThreadHandler
    {
        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
        
-        WaitHandle[] wait = _ipScanner.Start(64);
-        
-        WaitHandle.WaitAll(wait);
+        List<WaitHandle[]> wait = _ipScanner.Start(32);
+
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
        
        Console.WriteLine("Scanner finished");
        
@@ -62,6 +73,8 @@ public class ThreadHandler
    
    private void StartContentFilter()
    {
+        Thread.Sleep(5000);
+        
        WaitHandle[] wait = _contentFilter.Start();
        
        WaitHandle.WaitAll(wait);
@@ -73,12 +86,22 @@ public class ThreadHandler
    
    private void StartDbHandler()
    {
-        _dbHandler.StartContent();
+        _dbHandler.UnfilteredDbHandler();
+    }
+    
+    private void StartFilteredDbHandler()
+    {
+        _dbHandler.FilteredDbHandler();
+    }
+    
+    private void StartResumeDbHandler()
+    {
+        _dbHandler.ResumeDbHandler();
    }
    
    private void StartDiscardedDbHandler()
    {
-        WaitHandle[] wait = _dbHandler.Start(2);
+        WaitHandle[] wait = _dbHandler.Start(4);
        
        WaitHandle.WaitAll(wait);
        
@@ -2,6 +2,8 @@ namespace Backend.Helper;

 public static class HttpClientHelper
 {
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    
    public static async Task<string> GetHtml(string url, int port)
    {
        using HttpClient client = new();
@@ -16,8 +18,10 @@ public static class HttpClientHelper
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
        
-        HttpResponseMessage? response = null;
+        HttpResponseMessage? response;
        
        try
        {
@@ -25,10 +29,10 @@ public static class HttpClientHelper
        }
        catch
        {
-            //
+            return "";
        }
        
-        if (response is null || !response.IsSuccessStatusCode)
+        if (!response.IsSuccessStatusCode)
        {
            return "";
        }
@@ -50,12 +54,14 @@ public static class HttpClientHelper
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response = null;
        
        try
        {
-            response = await client.GetAsync("/robots.txt");
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
        {
@@ -1,37 +0,0 @@
-VACUUM;
-
-DROP TABLE Filtered;
-
-CREATE TABLE "Filtered" (
-	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
-	"Port1"	INTEGER NOT NULL,
-	"Port2"	INTEGER NOT NULL,
-	"Title1"	TEXT NOT NULL,
-	"Title2"	TEXT NOT NULL,
-	"Description1"	TEXT NOT NULL,
-	"Description2"	TEXT NOT NULL,
-	"Url1"	TEXT NOT NULL,
-	"Url2"	TEXT NOT NULL,
-	"ServerType1"	TEXT NOT NULL,
-	"ServerType2"	TEXT NOT NULL,
-	"RobotsTXT1"	TEXT NOT NULL,
-	"RobotsTXT2"	TEXT NOT NULL,
-	"HttpVersion1"	TEXT NOT NULL,
-	"HttpVersion2"	TEXT NOT NULL,
-	"CertificateIssuerCountry"	TEXT NOT NULL,
-	"CertificateOrganizationName"	TEXT NOT NULL,
-	"IpV6"	TEXT NOT NULL,
-	"TlsVersion"	TEXT NOT NULL,
-	"CipherSuite"	TEXT NOT NULL,
-	"KeyExchangeAlgorithm"	TEXT NOT NULL,
-	"PublicKeyType1"	TEXT NOT NULL,
-	"PublicKeyType2"	TEXT NOT NULL,
-	"PublicKeyType3"	TEXT NOT NULL,
-	"AcceptEncoding1"	TEXT NOT NULL,
-	"AcceptEncoding2"	TEXT NOT NULL,
-	"ALPN"	TEXT NOT NULL,
-	"Connection1" TEXT NOT NULL,
-	"Connection2" TEXT NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
-)
@@ -1,5 +1,5 @@
-using System.Collections;
 using System.Collections.Concurrent;
+using System.Diagnostics;
 using Microsoft.Data.Sqlite;
 using Models.Model.Backend;
 using Models.Model.External;
@@ -8,8 +8,10 @@ namespace Models.Handler;

 public class DbHandler
 {
-    private readonly ConcurrentQueue<QueueItem> _contentQueue;
+    private readonly ConcurrentQueue<Filtered> _filteredQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
    private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
    
    private readonly string _unfilteredConnectionString;
    private readonly string _discardedConnectionString;
@@ -17,13 +19,64 @@ public class DbHandler
    private readonly string _resumeConnectionString;
    private readonly List<string> _discardedConnectionStrings = [];
    
-    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
-    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
-    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode) VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
-    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                           " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
+                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
+    
+    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
+                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
+                                              " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
+                                              " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
+                                              " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
+                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
+                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
+                                              " @robotsTXT1, @robotsTXT2," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion1)," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion2)," +
+                                              " (SELECT CertificateIssuerId FROM CertificateIssuerCountry WHERE Country = @certificateIssuerCountry)," +
+                                              " (SELECT CertificateOrganizationId FROM CertificateOrganizationName WHERE Name = @certificateOrganizationName), " +
+                                              " @ipV6, " +
+                                              " (SELECT TlsId FROM TlsVersion WHERE Version = @tlsVersion)," +
+                                              " (SELECT CipherId FROM CipherSuite WHERE Suite = @cipherSuite)," +
+                                              " (SELECT KeyExchangeId FROM KeyExchangeAlgorithm WHERE Algorithm = @keyExchangeAlgorithm)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType1)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType2)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType3)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding1)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding2)," +
+                                              " (SELECT ALPNId FROM ALPN WHERE ALPNValue = @aLPN)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection1)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection2))";
+
+    private const string InsertIntoFilteredServerType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ServerType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredHttpVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO HttpVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCertificateIssuerCountry = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateIssuerCountry (Country) VALUES (@country)";
+    private const string InsertIntoFilteredCertificateOrganizationName = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateOrganizationName (Name) VALUES (@name)";
+    private const string InsertIntoFilteredTlsVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO TlsVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCipherSuite = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CipherSuite (Suite) VALUES (@suite)";
+    private const string InsertIntoFilteredKeyExchangeAlgorithm = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO KeyExchangeAlgorithm (Algorithm) VALUES (@algorithm)";
+    private const string InsertIntoFilteredPublicKeyType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO PublicKeyType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredAcceptEncoding = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO AcceptEncoding (Encoding) VALUES (@encoding)";
+    private const string InsertIntoFilteredALPN = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ALPN (ALPNValue) VALUES (@alpnValue)";
+    private const string InsertIntoFilteredConnection = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO Connection (ConnectionValue) VALUES (@connectionValue)";
+    
+    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
+                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+    
+    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
    
    private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
-    private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
+    private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
    private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;";
    private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
    private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
@@ -48,12 +101,17 @@ public class DbHandler

    private readonly string _basePath;
    
-    public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
+    public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
+        ConcurrentQueue<Discarded> discardedQueue, 
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
    {
-        _contentQueue = contentQueue;
+        _filteredQueue = filteredQueue;
        _discardedQueue = discardedQueue;
-        
-        SetContentWaitTime(10);
+        _unfilteredQueue = unfilteredQueue;
+        _resumeQueue = resumeQueue;
+
+        SetContentWaitTime(100);
        SetDiscardedWaitTime(10);
        
        _basePath = basePath;
@@ -74,45 +132,78 @@ public class DbHandler
        _discardedWaitTime = waitTime;
    }
    
-    public void StartContent()
+    public void UnfilteredDbHandler()
    {
-        Console.WriteLine("Content DbHandler started");
+        Console.WriteLine("Unfiltered DbHandler started");
        
        while (!_stop)
        {
-            if (_contentQueue.IsEmpty || _pause)
+            if (_unfilteredQueue.IsEmpty || _pause)
            {
                Thread.Sleep(_contentWaitTime);
                _paused = true;
                continue;
            }
            
-            _contentQueue.TryDequeue(out QueueItem? queueItem);
+            _unfilteredQueue.TryDequeue(out UnfilteredQueueItem queueItem);
            
-            if (queueItem is null) { continue; }
-            
-            if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Unfiltered)
+            if (queueItem.Operations == Operations.Insert)
            {
                InsertUnfiltered(queueItem.Unfiltered);
            }
            
-            else if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Filtered)
-            {
-                InsertFiltered(queueItem.Filtered!);
-            }
-            
-            else if (queueItem.Operations == Operations.Insert && queueItem.ResumeObject is not null)
-            {
-                InsertResumeObject(queueItem.ResumeObject);
-            }
-            
-            else if (queueItem.Operations == Operations.Update && queueItem.DbType == DbType.Unfiltered)
+            else if (queueItem.Operations == Operations.Update)
            {
                UpdateUnfiltered(queueItem.Unfiltered);
            }
        }
        
-        Console.WriteLine("Content DbHandler stopped.");
+        Console.WriteLine("Unfiltered DbHandler stopped.");
+    }
+    
+    public void FilteredDbHandler()
+    {
+        Console.WriteLine("Filtered DB handler started");
+        
+        while (!_stop)
+        {
+            if (_filteredQueue.IsEmpty || _pause)
+            {
+                Thread.Sleep(_contentWaitTime);
+                _paused = true;
+                continue;
+            }
+            
+            _filteredQueue.TryDequeue(out Filtered? queueItem);
+            
+            InsertFiltered(queueItem!);
+        }
+        
+        Console.WriteLine("Filtered DbHandler stopped.");
+    }
+    
+    public void ResumeDbHandler()
+    {
+        Console.WriteLine("Resume DB handler started");
+        
+        while (!_stop)
+        {
+            if (_resumeQueue.IsEmpty || _pause)
+            {
+                Thread.Sleep(_contentWaitTime);
+                _paused = true;
+                continue;
+            }
+            
+            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
+            
+            if (queueItem is not null)
+            {
+                InsertResumeObject(queueItem);
+            }
+        }
+        
+        Console.WriteLine("Resume DbHandler stopped.");
    }
    
    public WaitHandle[] Start(int threads)
@@ -131,7 +222,7 @@ public class DbHandler
            
            waitHandles[i] = handle;
            
-            Thread f = new (RunDiscarded!);
+            Thread f = new (DiscardedDbHandler!);
            f.Start(discardedDbHandlerSetting);
            
            Thread.Sleep(1000);
@@ -140,7 +231,7 @@ public class DbHandler
        return waitHandles;
    }
    
-    private void RunDiscarded(object obj)
+    private void DiscardedDbHandler(object obj)
    {
        DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
        Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
@@ -163,7 +254,7 @@ public class DbHandler
        
        discardedDbHandlerSetting.Handle!.Set();
        
-        Console.WriteLine("Content DbHandler stopped.");
+        Console.WriteLine("Discarded DbHandler stopped.");
    }
    
    private void InsertUnfiltered(Unfiltered unfiltered)
@@ -207,8 +298,75 @@ public class DbHandler
        using SqliteConnection connection = new(_filteredConnectionString);
        connection.Open();
        
-        using SqliteCommand command = new(InsertIntoFiltered, connection);
+        SqliteCommand command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType1);
+        _ = command.ExecuteNonQuery();
        
+        command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateIssuerCountry, connection);
+        command.Parameters.AddWithValue("@country", filtered.CertificateIssuerCountry);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateOrganizationName, connection);
+        command.Parameters.AddWithValue("@name", filtered.CertificateOrganizationName);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredTlsVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.TlsVersion);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCipherSuite, connection);
+        command.Parameters.AddWithValue("@suite", filtered.CipherSuite);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredKeyExchangeAlgorithm, connection);
+        command.Parameters.AddWithValue("@algorithm", filtered.KeyExchangeAlgorithm);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType3);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredALPN, connection);
+        command.Parameters.AddWithValue("@alpnValue", filtered.ALPN);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFiltered, connection);
        command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
        command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
        command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
@@ -241,8 +399,9 @@ public class DbHandler
        command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
        command.Parameters.AddWithValue("@connection1", filtered.Connection1);
        command.Parameters.AddWithValue("@connection2", filtered.Connection2);
-        
        _ = command.ExecuteNonQuery();
+        command.Dispose();
+        
        connection.Close();
    }
    
@@ -299,23 +458,18 @@ public class DbHandler
            ip.Ip2 = reader.GetInt32(2);
            ip.Ip3 = reader.GetInt32(3);
            ip.Ip4 = reader.GetInt32(4);
-            //Console.WriteLine(ip + "lmo");
            unfiltered.Port1 = reader.GetInt32(5);
            unfiltered.Port2 = reader.GetInt32(6);
            unfiltered.Filtered = reader.GetBoolean(7);
        }
        
-        //Console.WriteLine(ip + "aaaaa");
        unfiltered.Ip = ip;
-        //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs");
        
        return unfiltered;
    }
    
-    public long GetUnfilteredIndexes()
+    public List<long> GetUnfilteredIndexes()
    {
-        long rowId = 0;
-        
        using SqliteConnection connection = new(_unfilteredConnectionString);
        connection.Open();
        
@@ -324,15 +478,17 @@ public class DbHandler
        
        if (!reader.HasRows)
        {
-            return 0;
+            return [];
        }
+
+        List<long> ids = [];
        
        while (reader.Read())
        {
-            rowId = reader.GetInt64(0);
+            ids.Add(reader.GetInt64(0));
        }
        
-        return rowId;
+        return ids;
    }
    
    public long GetFilteredIndexes()
@@ -580,8 +736,8 @@ public class DbHandler
    {
        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
        
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 TEXT NOT NULL, Ip2 TEXT NOT NULL, Ip3 TEXT NOT NULL, Ip4 TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        
        _discardedConnectionStrings.Add(databaseName);
        
        using SqliteConnection connection = new(databaseName);
@@ -1,7 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum DbType
-{
-    Unfiltered,
-    Filtered,
-}
@@ -9,7 +9,7 @@ public struct Ip
    public int Ip3 { get; set; }
    
    public int Ip4 { get; set; }
-
+    
    public override string ToString()
    {
        return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
@@ -4,5 +4,4 @@ public enum Operations
 {
    Insert,
    Update,
-    Optimize,
 }
@@ -1,10 +0,0 @@
-namespace Models.Model.Backend;
-
-public class QueueItem
-{
-    public Unfiltered Unfiltered { get; init; }
-    public Filtered? Filtered { get; init; }
-    public ScannerResumeObject? ResumeObject { get; init; }
-    public Operations Operations { get; init; }
-    public DbType DbType { get; init; }
-}
@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public struct UnfilteredQueueItem
+{
+    public Unfiltered Unfiltered { get; init; }
+    public Operations Operations { get; init; }
+}
@@ -8,6 +8,5 @@

    <ItemGroup>
      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
-      <PackageReference Include="SQLite" Version="3.13.0" />
    </ItemGroup>
 </Project>
@@ -1,3 +0,0 @@
-<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cbackend/@EntryIndexedValue">False</s:Boolean>
-	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models_005Cinternal/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
@@ -0,0 +1,112 @@
+VACUUM;
+
+CREATE TABLE IF NOT EXISTS "ServerType" (
+    "ServerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "HttpVersion" (
+    "HttpId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateIssuerCountry" (
+	"CertificateIssuerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Country" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateOrganizationName" (
+	"CertificateOrganizationId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Name" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "TlsVersion" (
+	"TlsId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CipherSuite" (
+	"CipherId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Suite" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "KeyExchangeAlgorithm" (
+	"KeyExchangeId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Algorithm" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "PublicKeyType" (
+	"PublicKeyId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "AcceptEncoding" (
+	"AcceptId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Encoding" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "ALPN" (
+	"ALPNId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ALPNValue" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "Connection" (
+	"ConnectionId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ConnectionValue" TEXT NOT NULL UNIQUE
+);
+
+DROP TABLE Filtered;
+
+CREATE TABLE "Filtered" (
+	"Id"	INTEGER NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"	INTEGER NOT NULL,
+	"Ip3"	INTEGER NOT NULL,
+	"Ip4"	INTEGER NOT NULL,
+	"Port1"	INTEGER NOT NULL,
+	"Port2"	INTEGER NOT NULL,
+	"Title1"	TEXT NOT NULL,
+	"Title2"	TEXT NOT NULL,
+	"Description1"	TEXT NOT NULL,
+	"Description2"	TEXT NOT NULL,
+	"Url1"	TEXT NOT NULL,
+	"Url2"	TEXT NOT NULL,
+	"ServerType1"	INTEGER NOT NULL,
+	"ServerType2"	INTEGER NOT NULL,
+	"RobotsTXT1"	TEXT NOT NULL,
+	"RobotsTXT2"	TEXT NOT NULL,
+	"HttpVersion1"	INTEGER NOT NULL,
+	"HttpVersion2"	INTEGER NOT NULL,
+	"CertificateIssuerCountry"	INTEGER NOT NULL,
+	"CertificateOrganizationName"	INTEGER NOT NULL,
+	"IpV6"	TEXT NOT NULL,
+	"TlsVersion"	INTEGER NOT NULL,
+	"CipherSuite"	INTEGER NOT NULL,
+	"KeyExchangeAlgorithm"	INTEGER NOT NULL,
+	"PublicKeyType1"	INTEGER NOT NULL,
+	"PublicKeyType2"	INTEGER NOT NULL,
+	"PublicKeyType3"	INTEGER NOT NULL,
+	"AcceptEncoding1"	INTEGER NOT NULL,
+	"AcceptEncoding2"	INTEGER NOT NULL,
+	"ALPN"	INTEGER NOT NULL,
+	"Connection1"	INTEGER NOT NULL,
+	"Connection2"	INTEGER NOT NULL,
+	PRIMARY KEY("Id" AUTOINCREMENT),
+	FOREIGN KEY("ALPN") REFERENCES "ALPN"("ALPNId"),
+	FOREIGN KEY("AcceptEncoding1") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("AcceptEncoding2") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("CertificateIssuerCountry") REFERENCES "CertificateIssuerCountry"("CertificateIssuerId"),
+	FOREIGN KEY("CertificateOrganizationName") REFERENCES "CertificateOrganizationName"("CertificateOrganizationId"),
+	FOREIGN KEY("CipherSuite") REFERENCES "CipherSuite"("CipherId"),
+	FOREIGN KEY("Connection1") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("Connection2") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("HttpVersion1") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("HttpVersion2") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("KeyExchangeAlgorithm") REFERENCES "KeyExchangeAlgorithm"("KeyExchangeId"),
+	FOREIGN KEY("PublicKeyType1") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType2") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType3") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("ServerType1") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("ServerType2") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("TlsVersion") REFERENCES "TlsVersion"("TlsId")
+)
@@ -1,10 +1,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using MessagePack;
-using Microsoft.AspNetCore.Cors.Infrastructure;
+using AspNetCoreRateLimit;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";

 WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);

@@ -15,18 +17,29 @@ builder.Services.ConfigureHttpJsonOptions(options =>

 builder.Services.AddCors(options =>
 {
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });

+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
 WebApplication app = builder.Build();

-app.UseCors();
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);

 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
    CommunicationObject communicationObject = new()
    {
        Command = CommunicationCommand.GetScanningProgress
@@ -40,11 +53,15 @@ progressApi.MapGet("/", () =>
    byte[] msg = client.ReceiveFrameBytes();
    client.Close();
    
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });

+/*
 RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
 searchApi.MapGet("/{term}", (string term) =>
 {
    CommunicationObject communicationObject = new();
@@ -61,11 +78,11 @@ searchApi.MapGet("/{term}", (string term) =>
    
    return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-
+*/
 app.Run();

 [JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
+//[JsonSerializable(typeof(SearchResults))]
 [JsonSerializable(typeof(CommunicationObject))]
 internal partial class AppJsonSerializerContext : JsonSerializerContext
 {
@@ -9,6 +9,7 @@
    </PropertyGroup>

    <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
        <PackageReference Include="NetMQ" Version="4.0.1.13" />
    </ItemGroup>

@@ -1,7 +1,7 @@
 {
  "Logging": {
    "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
      "Microsoft.AspNetCore": "Warning"
    }
  },
@@ -1,29 +1,9 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000006pdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fd17d4b69379a42eb90f15b17ef6c846a5400_003F39_003F1eeed291_003F02000006pdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F096da63313a8436fb75089601a728c765200_003Fbb_003Ff6383783_003F02000007pdb1Low_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F357247e747614c8297021dd08d79da3d5200_003Ff2_003Fc4ae8c87_003F02000007pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000010pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F80cf8253b0c2409096d0c0e368300af28200_003Fdd_003F8098fc19_003F02000010pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb4Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F1f6cfdd1e3a14f6390237f6ab98b06af8000_003F0d_003Febdca535_003F02000011pdb4Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttributes_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9a4cb3a98df697f884fa5f45bc79e4291fa3e948c7f42560e14678e12055e_003FAttributes_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttribute_002ECoreCLR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbb9be2cb8efb72a7d2286fbdd10a293b55b3e2a912f49fac6a7719673268e4b_003FAttribute_002ECoreCLR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConsole_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2ccd2056ec55b7b67558d52e32a887ed5ac7e346fc429b218c188d0a99cb5be_003FConsole_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5029ca69753a60cb407b1053f5834320dadee066_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F99b5d033f8bfb58a6a753ca88a3d956f2228bd48_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F13cbfe6856867dd1ed4e39575e46d816dae2a146a8ceec8f76b5897f5e0fe_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpContent_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a55e5df9328b51021ba85b8f12ff49eca8772dba9772c0d61d5e28edf255c_003FHttpContent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpMethodAttribute_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2e4732e712a98f9ac0d080ac68669b3db6d98781e0d5ad89886cb6bcc890b18_003FHttpMethodAttribute_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIFormatterResolver_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc7d856f45ddf907f9face6c2bdba7836cc70fed4bb2b8ebd83ee6917584af3_003FIFormatterResolver_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIOThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F125fae6d49819da7be4fb21fbb4a936a7ec325971cf9264a24af55bf7111ad8_003FIOThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8fe420eaf60c4dfca87ce1d5f1cdfa4816200_003Fbd_003F71bcce16_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5cffb8c94881849b6aa8abe7ba5e7cafe05d991f859903020362d6aac371_003FMessagePackReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe5f8b0187e62d74a4b9fefd657c192dab367bb342ef5ffbd90a4ed2ea428ec7a_003FMessagePackSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AOutgoingSocketExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc4824d483bf85fa615ad2125f13f6d75788d45d1510ec1cfb8226831da54f_003FOutgoingSocketExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F77bf2fbffc5491eadd62f1dbebc233798cd7fcb9b39ab7ee3bb35519f4d94ecc_003FPingOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002ERawSocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2f645da43e51b8be94c9217511b45c23384f041ffa9aad041f0ddc158d732f0_003FPing_002ERawSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8d57f5f5fd3290d6a89a5b767ad89988dd893c988eba430cd461b8b88d7ad9d_003FPing_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARep_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Feea8e921c916164ccc376e162da148b8215450dfae96e08bdd29119165c67f_003FRep_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AReq_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F33fafcc2bb6e91ae8abb5a52936d39cd93951ad9208861e758ca93efa619eaf3_003FReq_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATask_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F35f3a54f5acb408a3e219b2de039f1a39557b7e4515f11238cba07b60c0ce_003FTask_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATextWriter_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdda89b2ed0975050b6847325357a756a5866a5435e3ddb4feff535ba36facb7_003FTextWriter_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example
@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.
@@ -0,0 +1,16 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <NuxtLayout>
+    <NuxtPage page-key="static" />
+  </NuxtLayout>
+
+</template>
@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
@@ -0,0 +1,17 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});
@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}
@@ -0,0 +1,11 @@
+<script setup lang="ts">
+const route = useRoute()
+</script>
+
+<template>
+  <div>
+    <h1>Nuxt Routing set up successfully!</h1>
+    <p>Current route: {{ route.path }}</p>
+    <a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
+  </div>
+</template>
@@ -0,0 +1,67 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <div v-if="dict">
+      <table class="table-auto border-gray-300">
+        <thead>
+          <tr>
+            <th class="px-4 py-2">Metric</th>
+            <th class="px-4 py-2">Value</th>
+          </tr>
+        </thead>
+
+        <tbody>
+          <tr v-for="d in dict">
+            <td class="px-4 py-2">{{d.description}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</template>
+
+<style scoped>
+
+</style>
@@ -0,0 +1 @@
+
@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}
@@ -0,0 +1,16 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {},
+  },
+  plugins: [],
+}
+
@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}
@@ -0,0 +1,19 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+}
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+}
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};
@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}
@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}
Author	SHA1	Message	Date
owner	580d687f62	Add user-agent	2024-12-21 21:57:01 +01:00
owner	af7b385641	Merge pull request 'Add cached progress response' (#36 ) from ImplementAPICache into main Reviewed-on: #36	2024-12-19 17:37:01 +00:00
owner	7562bbf7d1	Add cached progress response	2024-12-19 18:36:13 +01:00
owner	ac645b01b9	Merge pull request 'OptimiseFrontend' (#35 ) from OptimiseFrontend into main Reviewed-on: #35	2024-12-18 13:28:03 +00:00
owner	97321492ba	Add headers	2024-12-18 14:27:31 +01:00
owner	8628d31bec	Add nuxt-purgecss to further minimize bundle size.	2024-12-18 12:02:35 +01:00
owner	23d0a8b978	Merge pull request 'UseDictionary' (#34 ) from UseDictionary into main Reviewed-on: #34	2024-12-17 20:33:08 +00:00
owner	8b55cf7bb6	Reworked the /progress page	2024-12-17 13:15:47 +01:00
owner	3822339dd9	Add a time-based cache for the progress result from the API	2024-12-17 10:46:10 +01:00
owner	1c74ae9de5	Added frontend and working fetching.	2024-12-14 13:26:21 +01:00
owner	bab8fa6c25	Added frontend project with some basic api fetching implemented.	2024-12-06 12:33:48 +01:00
owner	cd49d32dd7	Merge pull request 'Misc changes such as performance tuning, added support for more than 64 threads for the scanner.' (#24 ) from UseForeignKeys into main Reviewed-on: #24	2024-12-04 12:24:34 +00:00
owner	5fe35192c6	Misc changes such as performance tuning, added support for more than 64 threads for the scanner.	2024-12-04 13:20:45 +01:00
owner	a1f5b08ac3	Merge pull request 'Make it so the filter does not iterate over already filtered IPs.' (#22 ) from OptimizeContentFilter into main Reviewed-on: #22	2024-11-29 18:18:00 +00:00
owner	2cff817044	Make it so the filter does not iterate over already filtered IPs.	2024-11-29 19:13:27 +01:00
owner	407bbc1556	Merge pull request 'Reworked the queue items.' (#21 ) from SplitUpQueueItem into main Reviewed-on: #21	2024-11-29 13:21:44 +00:00
owner	f2ace6f571	Reworked the queue items.	2024-11-29 12:59:07 +01:00
owner	3034e66126	Changed GET to HEAD for checking robots txt.	2024-11-29 11:02:51 +01:00
owner	701ffff27e	Merge pull request 'Duscarded object is now a struct.' (#17 ) from ConvertDiscardedToStruct into main Reviewed-on: #17	2024-11-28 13:37:01 +00:00