Merge pull request 'Fixed a new memory leak' (#47) from Analyzer into main

Reviewed-on: #47

.gitignore

@@ -7,6 +7,7 @@
 **/**/bin/*
 **/obj/*
 **/**/obj/*
+*.gz
 
 # User-specific stuff
 .idea/**/workspace.xml

Analyze/Analyze.csproj

@@ -1,9 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
+    <ItemGroup>
+      <Folder Include="CompressedDatabases\" />
+    </ItemGroup>
+
 </Project>

Analyze/Program.cs

@@ -0,0 +1,3 @@
+// See https://aka.ms/new-console-template for more information
+
+Console.WriteLine("Hello, World!");

Backend/Backend.csproj

@@ -2,21 +2,15 @@
     
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-
-        <Platform>x64</Platform>
-        <Optimize>true</Optimize>
+        <PublishAot>false</PublishAot>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
     
     <ItemGroup>
       <ProjectReference Include="..\Models\Models.csproj" />
     </ItemGroup>
     
-    <ItemGroup>
-      <PackageReference Include="FuzzySharp" Version="2.0.2" />
-      <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-    
 </Project>

Backend/Handler/Communication.cs

@@ -1,200 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using Backend.Helper;
-using Models.Handler;
-using Models.Model.Backend;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-
-namespace Backend.Handler;
-
-public class Communication
-{
-    private readonly DbHandler _dbHandler;
-    private readonly ThreadHandler _threadHandler;
-    private readonly IpScanner _ipScanner;
-    private readonly ContentFilter _contentFilter;
-    private bool _isRunning = true;
-    private string _basePath;
-    
-    public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
-    {
-        _dbHandler = dbHandler;
-        _threadHandler = threadHandler;
-        _ipScanner = ipScanner;
-        _contentFilter = contentFilter;
-        _basePath = basePath;
-    }
-    
-    public WaitHandle[] Start()
-    {
-        WaitHandle[] waitHandles = new WaitHandle[1];
-        EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-        waitHandles[0] = handle;
-        
-        Thread thread = new(Server!);
-        thread.Start(handle);
-        
-        return waitHandles;
-    }
-    
-    private void Server(object obj)
-    {
-        using ResponseSocket rep = new();
-        
-        //rep.Options.IPv4Only = true;
-        
-        rep.Bind("tcp://127.0.0.1:5556");
-        
-        while (_isRunning)
-        {
-            byte[] message = rep.ReceiveFrameBytes();
-            
-            CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
-
-            //rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
-            
-            if (communicationObject is null)
-            {
-                continue;
-            }
-            
-            OnServerOnReceiveReady(communicationObject, rep);
-        }
-        
-        ((EventWaitHandle) obj).Set();
-    }
-    
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
-    private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
-    {
-        switch (communicationObject.Command)
-        {
-            case CommunicationCommand.GetScanningProgress:
-            {
-                DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
-                
-                long discardedIndexes = _dbHandler.GetDiscardedIndexes();
-                
-                ScanningStatus status = new();
-                // 4294967296 is all Ipv4 addresses.
-                
-                if (discardedIndexes != 0)
-                {
-                    status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
-                }
-                else
-                {
-                    status.PercentageOfIpv4Scanned = 0.0000000001f;
-                }
-                
-                status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
-                status.TotalFiltered = _dbHandler.GetFilteredIndexes();
-                status.TotalDiscarded = discardedIndexes;
-                status.MyDbSize = databaseSizes.MyDbSize;
-                status.FilteredDbSize = databaseSizes.FilteredDbSize;
-                status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
-
-                byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
-                
-                rep.SendFrame(serializedResult);
-                
-                break;
-            }
-            
-            case CommunicationCommand.DbReindex:
-            {
-                _dbHandler.ReIndex();
-                
-                SendStringResponse(rep, "All Dbs have been reindexed.");
-                break;
-            }
-            
-            case CommunicationCommand.DbVacuum:
-            {
-                _dbHandler.Vacuum();
-                
-                SendStringResponse(rep, "All Dbs have been vacuumed.");
-                break;
-            }
-            
-            case CommunicationCommand.GetSearches:
-            {
-                if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
-                {
-                    SendSearchResponse(rep, communicationObject.SearchTerm);
-                }
-                
-                break;
-            }
-            
-            case CommunicationCommand.StopScanning:
-            {
-                _isRunning = false;
-                break;
-            }
-            
-            case CommunicationCommand.ChangeRuntimeVariable:
-            {
-                if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
-                
-                if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _dbHandler.SetContentWaitTime(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _dbHandler.SetDiscardedWaitTime(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _ipScanner.SetTimeout(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _contentFilter.SetTimeout(value);
-                }
-                
-                rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
-                
-                break;
-            }
-        }
-    }
-    
-    private static void SendStringResponse(ResponseSocket rep, string response)
-    {
-        byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
-        
-        rep.SendFrame(serializedResult);
-    }
-
-    private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
-    {
-        //SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
-        SearchResults result = new()
-        {
-            Results = []
-        };
-        SearchResult lol = new()
-        {
-            Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
-            Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
-        };
-        
-        result.Results.Add(lol);
-        
-        string serializedResult = JsonSerializer.Serialize(result);
-        
-        rep.SendFrame(serializedResult);
-    }
-}

Backend/Handler/ContentFilter.cs

@@ -6,27 +6,44 @@ using Models.Model.Backend;
 
 namespace Backend.Handler;
 
+public class Content
+{
+    public int Port1 { get; set; }
+    public int Port2 { get; set; }
+    public Ip Ip { get; set; }
+}
+
+public class ContentThread
+{
+    public int ThreadId { get; set; }
+    public EventWaitHandle? EventWaitHandle { get; set; }
+}
+
 public class ContentFilter
 {
     private readonly ConcurrentQueue<Filtered> _queue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<Content?> _contentQueue = new();
     private readonly DbHandler _dbHandler;
+    private readonly ThreadHandler _threadHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
     private bool _stop;
     private int _timeOut;
     private readonly string _basePath;
-    
-    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
+
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
+        _threadHandler = threadHandler;
         _unfilteredQueue = unfilteredQueue;
         
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
+        
         SetTimeout(3000);
     }
     
@@ -52,11 +69,23 @@ public class ContentFilter
         while (!_stop)
         {
             List<long> indexes = _dbHandler.GetUnfilteredIndexes();
+
+            if (indexes.Count == 0)
+            {
+                _stop = true;
+                _threadHandler.Stop();
+                break;
+            }
             
             for (int i = 0; i < indexes.Count; i++)
             {
                 if (_stop) break;
                 
+                if (_contentQueue.Count >= 500)
+                {
+                    Thread.Sleep(1000);
+                }
+                
                 Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                 
                 if (unfiltered.Filtered) continue;
@@ -77,13 +106,13 @@ public class ContentFilter
                 {
                     continue;
                 }
+
+                Content content = new();
+                content.Ip = ip;
+                content.Port1 = unfiltered.Port1;
+                content.Port2 = unfiltered.Port2;
                 
-                Filtered filtered = GetSiteData(ip);
-                
-                filtered.Port1 = unfiltered.Port1;
-                filtered.Port2 = unfiltered.Port2;
-                
-                _queue.Enqueue(filtered);
+                _contentQueue.Enqueue(content);
             }
             
             Thread.Sleep(_timeOut);
@@ -91,18 +120,65 @@ public class ContentFilter
         
         ((EventWaitHandle) obj).Set();
     }
-    
-    private Filtered GetSiteData(Ip ip)
+
+    public WaitHandle[] StartFilterThread(int threads)
     {
-        StartProcess(ip, 80);
-        StartProcess(ip, 443);
+        WaitHandle[] waitHandle = new WaitHandle[threads];
+
+        for (int i = 0; i < threads; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            ContentThread contentThread = new();
+            contentThread.ThreadId = i;
+            contentThread.EventWaitHandle = handle;
+            waitHandle[i] = handle;
+            
+            Thread thread = new(FilterThread!);
+            thread.Start(contentThread);
+            
+            Thread.Sleep(8);
+        }
+        
+        return waitHandle;
+    }
+
+    private void FilterThread(object obj)
+    {
+        Console.WriteLine("Filter Thread started.");
+        ContentThread thread = (ContentThread) obj;
+        
+        while (!_stop)
+        {
+            if (_contentQueue.IsEmpty)
+            {
+                Thread.Sleep(1000);
+            }
+            
+            _contentQueue.TryDequeue(out Content? content);
+
+            if (content is null)
+            {
+                continue;
+            }
+            
+            Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
+                
+            filtered.Port1 = content.Port1;
+            filtered.Port2 = content.Port2;
+                
+            _queue.Enqueue(filtered);
+        }
+        
+        thread.EventWaitHandle!.Set();
+    }
+    
+    private Filtered GetSiteData(Ip ip, int threadId)
+    {
+        StartProcess(ip, 80, threadId);
+        StartProcess(ip, 443, threadId);
         
         string url1 = "";
         string url2 = "";
-        string title1 = "";
-        string title2 = "";
-        string description1 = "";
-        string description2 = "";
         bool robotsTxt1 = false;
         bool robotsTxt2 = false;
         string serverType1 = "";
@@ -128,7 +204,7 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
+            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
             
             while (streamReader.Peek() != -1)
             {
@@ -161,29 +237,8 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html;
-            
-            if (ports[i] == 80)
-            {
-                if (string.IsNullOrWhiteSpace(url1)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result;
-            }
-            else
-            {
-                if (string.IsNullOrWhiteSpace(url2)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result;
-            }
-            
-            if (string.IsNullOrWhiteSpace(html)) continue;
-            
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
-            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; }
-            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; }
+            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
+            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }
         
         Filtered siteData = new()
@@ -191,10 +246,6 @@ public class ContentFilter
             Ip = ip,
             Url1 = url1,
             Url2 = url2,
-            Title1 = title1,
-            Title2 = title2,
-            Description1 = description1,
-            Description2 = description2,
             ServerType1 = serverType1,
             ServerType2 = serverType2,
             RobotsTXT1 = robotsTxt1,
@@ -220,7 +271,7 @@ public class ContentFilter
         return siteData;
     }
     
-    private void StartProcess(Ip ip, int port)
+    private void StartProcess(Ip ip, int port, int threadId)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
         
@@ -228,7 +279,7 @@ public class ContentFilter
         proc.StartInfo = new()
         {
             FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
+            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,

Backend/Handler/IpFilterHandler.cs

@@ -0,0 +1,131 @@
+using System.Collections.Concurrent;
+using Backend.Helper;
+using Models.Model.Backend;
+
+namespace Backend.Handler;
+
+public class IpFilterHandler
+{
+    private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
+    private ThreadHandler _threadHandler;
+    private bool _stop;
+    private bool _done;
+    private bool _stopAutoscaledThreads;
+    private int _timeout;
+    
+    public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
+        ConcurrentQueue<Ip> preFilteredQueue, ThreadHandler threadHandler)
+    {
+        _discardedQueue = discardedQueue;
+        _unfilteredQueue = unfilteredQueue;
+        _preFilteredQueue = preFilteredQueue;
+        _threadHandler = threadHandler;
+
+        _timeout = 16;
+    }
+    
+    public List<WaitHandle[]> Start(int threadCount)
+    {
+        WaitHandle[] waitHandle = new WaitHandle[64];
+        
+        int counter = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
+        
+        for (int i = 0; i < threadCount; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            
+            if (counter < 64)
+            {
+                waitHandle[counter] = handle;
+                counter++;
+                
+                Thread f = new (Filter!);
+                f.Start(handle);
+                
+                Console.WriteLine($"Filter thread ({i}) started");
+                Thread.Sleep(16);
+                
+                continue;
+            }
+            
+            counter = 0;
+            
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
+        }
+        
+        return waitHandles;
+    }
+    
+    private void Filter(object obj)
+    {
+        int counter = 0;
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                if (counter == 30_000)
+                {
+                    _threadHandler.Stop();
+                    _stop = true;
+                }
+                
+                counter++;
+                Thread.Sleep(128);
+                
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out Ip item);
+            
+            (int, int) ports = TcpClientHelper.CheckPort(item, 80, 443);
+            
+            if (ports is { Item1: 0, Item2: 0 })
+            {
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item, item.ResponseCode));
+                continue;
+            }
+            
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item, ports));
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
+    {
+        return new()
+        {
+            Ip = ip,
+            ResponseCode = responseCode
+        };
+    }
+    
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    {
+        Unfiltered unfiltered = new()
+        {
+            Ip = ip,
+            Port1 = ports.Item1,
+            Port2 = ports.Item2,
+            Filtered = false
+        };
+        
+        return new()
+        {
+            Unfiltered = unfiltered,
+            Operations = Operations.Insert
+        };
+    }
+
+    public void Stop()
+    {
+        _stop = true;
+    }
+}

Backend/Handler/IpScanner.cs

@@ -1,7 +1,11 @@
+using System.Buffers.Binary;
 using System.Collections.Concurrent;
 using System.Net;
 using System.Net.NetworkInformation;
+using System.Numerics;
+using System.Runtime.InteropServices;
 using Backend.Helper;
+using Models.Experimental;
 using Models.Handler;
 using Models.Model.Backend;
 
@@ -18,38 +22,37 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
-    private int _timeout;
+    private readonly int _timeout;
     
-    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
-        )
+    public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
+        ConcurrentQueue<Ip> preFilteredQueue)
     {
         _dbHandler = dbHandler;
+        _preFilteredQueue = preFilteredQueue;
         _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
 
-        SetTimeout(64);
+        _timeout = 32;
     }
     
-    public void SetTimeout(int milliseconds)
-    {
-        _timeout = milliseconds;
-    }
-    
-    public WaitHandle[] Start(int threads)
+    public List<WaitHandle[]> Start(int threads)
     {
         int threadsAmount = 0;
         if (threads % 2 == 0)
         {
             threadsAmount = 256 / threads;
         }
+
+        WaitHandle[] waitHandle = new WaitHandle[64];
         
-        WaitHandle[] waitHandles = new WaitHandle[threads];
+        int counter = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -63,13 +66,25 @@ public class IpScanner
                 Handle = handle
             };
             
-            waitHandles[i] = handle;
+            if (counter < 64)
+            {
+                waitHandle[counter] = handle;
+                counter++;
+                
+                Thread f = new (Scan!);
+                f.Start(scanSettings);
+                
+                Console.WriteLine($"Scanner thread ({i}) started");
+                Thread.Sleep(128);
+                
+                continue;
+            }
             
-            Thread f = new (Scan!);
-            f.Start(scanSettings);
+            counter = 0;
             
-            Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(1000);
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
         }
         
         return waitHandles;
@@ -92,15 +107,26 @@ public class IpScanner
         
         if (resumeNow is not null)
         {
+            if (resumeNow.Completed)
+            {
+                return;
+            }
+            
             scanSettings.Start = resumeNow.FirstByte;
             scanSettings.End = resumeNow.EndRange;
             secondByte = resumeNow.SecondByte;
             thirdByte = resumeNow.ThirdByte;
             fourthByte = resumeNow.FourthByte;
         }
+        else
+        {
+            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
+        }
         
+        // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
+        int x = 0;
         
         for (int i = scanSettings.Start; i < scanSettings.End; i++)
         {
@@ -119,19 +145,34 @@ public class IpScanner
                     if (i == 192 && k == 2) continue;
                     if (i == 192 && j == 88 && k == 99) continue;
                     
-                    if (_discardedQueue.Count >= 2000)
+                    if (_discardedQueue.Count >= 20_000)
                     {
-                        Thread.Sleep(500);
+                        Thread.Sleep(1000);
+                    }
+                    
+                    if (_preFilteredQueue.Count >= 20_000)
+                    {
+                        Thread.Sleep(1000);
                     }
                     
                     for (int l = fourthByte; l < 256; l++)
                     {
+                        if (x == 75_000)
+                        {
+                            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
+                            x = 0;
+                        }
+                        
+                        x++;
+                        
                         if (_stop)
                         {
                             resumeObject.FourthByte = l;
                             break;
                         }
-
+                        
+                        
+                        
                         Ip ip = new()
                         {
                             Ip1 = i,
@@ -139,18 +180,29 @@ public class IpScanner
                             Ip3 = k,
                             Ip4 = l
                         };
-
+                        
                         IPStatus responseCode = IPStatus.Unknown;
                         
                         try
                         {
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
-                            _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
-                            if (address is not null)
+                            //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
+                            
+                            responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
+                            
+                            /*if (l % 2 == 0)
                             {
-                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = IPStatus.Success;
                             }
+                            else
+                            {
+                                responseCode = IPStatus.TimedOut;
+                            }*/
+                            
+                            //responseCode = IPStatus.TimedOut;
+                            
+                            //Thread.Sleep(0);
                         }
                         catch
                         {
@@ -163,15 +215,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
-                        
-                        if (ports is { Item1: 0, Item2: 0 })
-                        {
-                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
-                            continue;
-                        }
-                        
-                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
+                        _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
                     }
                     
                     if (_stop)
@@ -193,13 +237,42 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
+        Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped.");
+        
+        if (_stop)
+        {
+            resumeObject.Paused = true;
+        }
+        else
+        {
+            resumeObject.Completed = true;
+        }
+        
+        resumeObject.Operation = Operations.Update;
+        
         _resumeQueue.Enqueue(resumeObject);
         
         scanSettings.Handle!.Set();
     }
+
+    private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
+    {
+        ScannerResumeObject resumeObject = new();
+        resumeObject.ThreadNumber = threadNumber;
+        resumeObject.StartRange = startRange;
+        resumeObject.EndRange = endRange;
+        resumeObject.FirstByte = firstByte;
+        resumeObject.SecondByte = secondByte;
+        resumeObject.ThirdByte = thirdByte;
+        resumeObject.FourthByte = fourthByte;
+        resumeObject.Paused = paused;
+        resumeObject.Completed = completed;
+        resumeObject.Operation = operation;
+        
+        _resumeQueue.Enqueue(resumeObject);
+    }
     
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
@@ -210,25 +283,110 @@ public class IpScanner
         };
     }
     
-    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    private static Ip CreateUnfilteredQueueItem(Ip ip, int responseCode)
     {
-        Unfiltered unfiltered = new()
-        {
-            Ip = ip,
-            Port1 = ports.Item1,
-            Port2 = ports.Item2,
-            Filtered = false
-        };
+        ip.ResponseCode = responseCode;
         
-        return new()
-        {
-            Unfiltered = unfiltered,
-            Operations = Operations.Insert
-        };
+        return ip;
     }
     
     public void Stop()
     {
         _stop = true;
     }
+    
+    private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
+    {
+        int length = ipSpan.Length;
+        long nonCanonical;
+        fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
+            nonCanonical = ParseNonCanonical(name, 0, ref length, true);
+        
+        return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
+    }
+    
+    private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
+    {
+        long* numPtr = stackalloc long[4];
+        long num1 = 0;
+        bool flag = false;
+        int index1 = 0;
+        int index2;
+        for (index2 = start; index2 < end; ++index2)
+        {
+            char ch = name[index2];
+            num1 = 0L;
+            int num2 = 10;
+            if (ch == '0')
+            {
+                num2 = 8;
+                ++index2;
+                flag = true;
+                if (index2 < end)
+                {
+                    switch (name[index2])
+                    {
+                        case 'X':
+                        case 'x':
+                            num2 = 16;
+                            ++index2;
+                            flag = false;
+                            break;
+                    }
+                }
+            }
+            for (; index2 < end; ++index2)
+            {
+                char c = name[index2];
+                int num3;
+                if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
+                    num3 = (int) c - 48;
+                else if (num2 == 8 && '0' <= c && c <= '7')
+                    num3 = (int) c - 48;
+                else if (num2 == 16 && 'a' <= c && c <= 'f')
+                    num3 = (int) c + 10 - 97;
+                else if (num2 == 16 && 'A' <= c && c <= 'F')
+                    num3 = (int) c + 10 - 65;
+                else
+                    break;
+                num1 = num1 * (long) num2 + (long) num3;
+                if (num1 > (long) uint.MaxValue)
+                    return -1;
+                flag = true;
+            }
+            if (index2 < end && name[index2] == '.')
+            {
+                if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
+                    return -1;
+                numPtr[index1] = num1;
+                ++index1;
+                flag = false;
+            }
+            else
+                break;
+        }
+        if (!flag)
+            return -1;
+        if (index2 < end)
+        {
+            char ch;
+            if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
+                return -1;
+            end = index2;
+        }
+        numPtr[index1] = num1;
+        switch (index1)
+        {
+            case 0:
+                return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
+            case 1:
+                return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
+            case 2:
+                return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
+            case 3:
+                return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
+            default:
+                return -1;
+        }
+    }
 }

Backend/Handler/ThreadHandler.cs

@@ -7,69 +7,124 @@ namespace Backend.Handler;
 public class ThreadHandler
 {
     private readonly DbHandler _dbHandler;
-    private readonly Communication _communication;
     private readonly IpScanner _ipScanner;
     private readonly ContentFilter _contentFilter;
+    private readonly IpFilterHandler _ipFilterHandler;
     
-    private bool _communicationStopped;
     private bool _ipScannerStopped;
     private bool _contentFilterStopped;
+    private bool _ipFilterStopped;
+
+    private bool _stage1 = false;
+    private bool _stage2 = true;
+    private bool _stage3 = false;
+    
+    ConcurrentQueue<Filtered> filteredQueue = new();
+    ConcurrentQueue<Discarded> discardedQueue = new();
+    ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
+    ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
+    ConcurrentQueue<Ip> preFilteredQueue = new();
     
     public ThreadHandler(string path)
     {
-        ConcurrentQueue<Filtered> filteredQueue = new();
-        ConcurrentQueue<Discarded> discardedQueue = new();
-        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
-        ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
-        
-        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
-        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
-        _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
+        _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, this);
     }
     
     public void Start()
     {
         Thread scanner = new(StartScanner);
+        Thread ipFilter = new(StartIpFilter);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
-        Thread communication = new(StartCommunicationHandler);
+        Thread contentFilterThread = new(StartContentFilterThread);
+        Thread prefilterDb = new(StartPreFilterDbHandler);
+        Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
+        //Thread check = new(CheckQueue);
         
-        scanner.Start();
-        indexer.Start();
-        database.Start();
-        discarded.Start();
-        filtered.Start();
-        resume.Start();
-        communication.Start();
+        //check.Start();
         
-        scanner.Join();
-        indexer.Join();
-        database.Join();
-        discarded.Join();
-        filtered.Join();
-        resume.Join();
-        communication.Join();
+        if (_stage1)
+        {
+            discarded.Start(); // de-queues from discardedQueue
+            prefilterDb.Start(); // de-queues from preFilteredQueue
+            scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
+            resume.Start(); // de-queues from resumeQueue
+            
+            scanner.Join();
+            Stop();
+            
+            discarded.Join();
+            prefilterDb.Join();
+            resume.Join();
+        }
+        
+        if (_stage2)
+        {
+            database.Start(); // de-queues from unfilteredQueue
+            discarded.Start(); // de-queues from discardedQueue
+            ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
+            fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
+
+            ipFilter.Join();
+            Stop();
+            database.Join();
+            discarded.Join();
+            fillIpFilterQueue.Join();
+        }
+        
+        if (_stage3)
+        {
+            filtered.Start(); // de-queues from filteredQueue
+            database.Start(); // de-queues from unfilteredQueue
+            indexer.Start(); // en-queues to unfilteredQueue and contentQueue
+            contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
+            
+            contentFilterThread.Join();
+            filtered.Join();
+            database.Join();
+            indexer.Join();
+        }
     }
     
+    private void CheckQueue()
+    {
+        while (true)
+        {
+            Console.Clear();
+            Console.WriteLine($"filteredQueue.Count:       {filteredQueue.Count}");
+            Console.WriteLine($"discardedQueue.Count:      {discardedQueue.Count}");
+            Console.WriteLine($"unfilteredQueue.Count:     {unfilteredQueue.Count}");
+            Console.WriteLine($"scannerResumeQueue.Count:  {scannerResumeQueue.Count}");
+            Console.WriteLine($"preFilteredQueue.Count:    {preFilteredQueue.Count}");
+            Thread.Sleep(5);
+        }
+    }
+
+    
     private void StartScanner()
     {
-        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
+        Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
         
-        WaitHandle[] wait = _ipScanner.Start(64);
+        List<WaitHandle[]> wait = _ipScanner.Start(256);
         
-        WaitHandle.WaitAll(wait);
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
         
         Console.WriteLine("Scanner finished");
-        
-        _ipScannerStopped = true;
     }
     
     private void StartContentFilter()
     {
+        Thread.Sleep(5000);
+        
         WaitHandle[] wait = _contentFilter.Start();
         
         WaitHandle.WaitAll(wait);
@@ -78,17 +133,50 @@ public class ThreadHandler
         
         _contentFilterStopped = true;
     }
+
+    private void StartContentFilterThread()
+    {
+        WaitHandle[] wait = _contentFilter.StartFilterThread(8);
+        
+        WaitHandle.WaitAll(wait);
+    }
+
+    private void StartFillIpFilterQueue()
+    {
+        _dbHandler.GetPreFilterQueueItem();
+    }
+
+    private void StartIpFilter()
+    {
+        Thread.Sleep(1000);
+        
+        List<WaitHandle[]> wait = _ipFilterHandler.Start(1024);
+
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
+        
+        Console.WriteLine("Ip filter finished");
+        
+        _ipFilterStopped = true;
+    }
     
     private void StartDbHandler()
     {
         _dbHandler.UnfilteredDbHandler();
     }
-
+    
     private void StartFilteredDbHandler()
     {
         _dbHandler.FilteredDbHandler();
     }
 
+    private void StartPreFilterDbHandler()
+    {
+        _dbHandler.PrefilteredDbHandler();
+    }
+    
     private void StartResumeDbHandler()
     {
         _dbHandler.ResumeDbHandler();
@@ -96,42 +184,26 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(2);
+        WaitHandle[] wait = _dbHandler.Start(4);
         
         WaitHandle.WaitAll(wait);
         
         Console.WriteLine("Discarded DbHandler finished");
     }
     
-    private void StartCommunicationHandler()
-    {
-        WaitHandle[] wait = _communication.Start();
-        
-        WaitHandle.WaitAll(wait);
-        
-        Console.WriteLine("Communicator finished");
-        
-        _communicationStopped = true;
-        
-        Stop();
-    }
-    
-    private void Stop()
+    public void Stop()
     {
+        Console.WriteLine("Stopping...");
         _ipScanner.Stop();
         _contentFilter.Stop();
+        _ipFilterHandler.Stop();
+        Console.WriteLine("Stopping Extra...");
         
-        bool stopping = true;
+        Thread.Sleep(30_000);
         
-        while (stopping)
-        {
-            if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
-            {
-                _dbHandler.Stop();
-                stopping = false;
-            }
-            
-            Thread.Sleep(3000);
-        }
+        Console.WriteLine("Stopping Super Extra...");
+        _dbHandler.Stop();
+        
+        Console.WriteLine("Stopped.");
     }
 }

Backend/Helper/FilesystemHelper.cs

@@ -1,43 +0,0 @@
-using Models.Model.Backend;
-
-namespace Backend.Helper;
-
-public static class FilesystemHelper
-{
-    public static DatabaseSizes GetDatabaseSizes(string basePath)
-    {
-        DatabaseSizes databaseSizes = new();
-        
-        FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
-        databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
-        
-        databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
-        
-        fileInfo = new($"{basePath}/Models/Filtered.db");
-        databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
-        
-        return databaseSizes;
-    }
-
-    private static long GetDiscardedDbSizes(string basePath)
-    {
-        string folder = $"{basePath}/Models";
-        string[] files = Directory.GetFiles(folder, "*.db");
-        
-        long size = 0;
-        
-        for (int i = 0; i < files.Length; i++)
-        {
-            if (!files[i].Contains("Discarded")) continue;
-            FileInfo fileInfo = new(files[i]);
-            size += fileInfo.Length;
-        }
-        
-        return size;
-    }
-    
-    private static double ToSize(this long value, SizeUnits unit)
-    {
-        return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
-    }
-}

Backend/Helper/FilterHelper.cs

@@ -2,12 +2,10 @@ using System.Text.RegularExpressions;
 
 namespace Backend.Helper;
 
-public static partial class FilterHelper
+public static class FilterHelper
 {
     // https://stackoverflow.com/a/56116499
     private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
-    private const string TitlePattern = "<title>(.*)</title>";
-    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
     
     public static void GetDomain(string data, out string result)
     {
@@ -42,32 +40,6 @@ public static partial class FilterHelper
         result = GetSubstring(data, start, end);
     }
     
-    public static void GetTitle(string data, out string result)
-    {
-        result = "";
-        
-        Regex titleRegex = MyRegex1();
-        Match match = titleRegex.Match(data);
-        
-        if (match.Success)
-        {
-            result = match.Groups[1].Value;
-        }
-    }
-    
-    public static void GetDescription(string data, out string result)
-    {
-        result = "";
-        
-        Regex titleRegex = MyRegex2();
-        Match match = titleRegex.Match(data);
-        
-        if (match.Success)
-        {
-            result = match.Groups[1].Value;
-        }
-    }
-    
     public static void GetHttpVersion(string data, out string result)
     {
         result = "";
@@ -198,7 +170,7 @@ public static partial class FilterHelper
             case 0:
                 return;
             case 2:
-                result = $"{temp[1].Trim()} UUHHHMMM";
+                result = $"{temp[1].Trim()}";
                 return;
             default:
                 result = temp[2].Trim();
@@ -241,17 +213,7 @@ public static partial class FilterHelper
     
     private static bool ValidateUrl(string url)
     {
-        Regex rgx = MyRegex();
+        Regex rgx = new(DomainPattern);
         return rgx.IsMatch(url);
     }
-    
-    //Generate the RegEx at compile time.
-    [GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
-    private static partial Regex MyRegex();
-    
-    [GeneratedRegex(TitlePattern)]
-    private static partial Regex MyRegex1();
-    
-    [GeneratedRegex(DescriptionPattern)]
-    private static partial Regex MyRegex2();
 }

Backend/Helper/HttpClientHelper.cs

@@ -1,43 +1,11 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
 
 namespace Backend.Helper;
 
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
-    {
-        using HttpClient client = new();
-        
-        if (port == 80)
-        {
-            client.BaseAddress = new($"http://{url}");
-        }
-        else
-        {
-            client.BaseAddress = new($"https://{url}");
-        }
-        
-        client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(30);
-        
-        HttpResponseMessage? response = null;
-        
-        try
-        {
-            response = await client.GetAsync("/");
-        }
-        catch
-        {
-            //
-        }
-        
-        if (response is null || !response.IsSuccessStatusCode)
-        {
-            return "";
-        }
-        
-        return await response.Content.ReadAsStringAsync();
-    }
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
     {
@@ -45,19 +13,35 @@ public static class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {//
+        {
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch

Backend/Helper/SearchHelper.cs

@@ -1,44 +0,0 @@
-using FuzzySharp;
-using Models.Handler;
-using Models.Model.External;
-
-namespace Backend.Helper;
-
-public static class SearchHelper
-{
-    public static SearchResults Search(string searchText, DbHandler dbHandler)
-    {
-        if (string.IsNullOrWhiteSpace(searchText))
-        {
-            SearchResult searchResult = new();
-            searchResult.Description = "asd";
-            searchResult.Title = "asd";
-            searchResult.Url = "asd";
-            SearchResults searchResults = new();
-            searchResults.Results =
-            [
-                searchResult
-            ];
-            return searchResults;
-        }
-        
-        List<SearchResult?> temp = dbHandler.GetSearchResults();
-        SearchResults searchResultsList = new();
-        
-        for (int i = 0; i < temp.Count; i++)
-        {
-            if (temp[i] is null) continue;
-            
-            SearchResult result = new();
-            
-            if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
-            
-            result.Url = temp[i]?.Url;
-            result.Title = temp[i]?.Title;
-            
-            searchResultsList.Results.Add(result);
-        }
-        
-        return searchResultsList;
-    }
-}

Backend/Helper/TcpClientHelper.cs

@@ -1,20 +1,24 @@
 using System.Net;
 using System.Net.Sockets;
+using Models.Model.Backend;
 
 namespace Backend.Helper;
 
 public static class TcpClientHelper
 {
-    public static (int, int) CheckPort(string ip, params int[] ports)
+    public static (int, int) CheckPort(Ip ip, params int[] ports)
     {
         // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached,
         // and it would just return a result.error, for example.
         for (int i = 0; i < ports.Length; i++) {
+            using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            socket.SendTimeout = 250;
+            
             try
             {
-                using TcpClient client = new();
-                client.Connect(ip, ports[i]);
-                // If the connection is successful, update the result array with the port number
+                socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
+                socket.Close();
+                // If the connection is not successful, update the ports array with 0.
             }
             catch
             {

Manager/Commands.cs

@@ -1,122 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using Models.Model.Backend;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-
-namespace Manager;
-
-public static class Commands
-{
-    public static void GetProgress()
-    {
-        Console.WriteLine("Getting progress ...");
-        
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.GetScanningProgress;
-        
-        ScanningStatus temp = GetProgress(communicationObject);
-        
-        Console.WriteLine($"Total filtered:           {temp.TotalFiltered:n0}");
-        Console.WriteLine($"Total discarded:          {temp.TotalDiscarded:n0}");
-        Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}");
-        Console.WriteLine($"Total Ips left:           {temp.AmountOfIpv4Left:n0}");
-        Console.WriteLine($"Filtered DB size:         {temp.FilteredDbSize} Kb");
-        Console.WriteLine($"Discarded DB size:        {temp.DiscardedDbSize} Kb");
-        Console.WriteLine($"Mydb DB size:             {temp.MyDbSize} Kb");
-    }
-    
-    public static void StopServer()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.StopScanning;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void Vacuum()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.DbVacuum;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void ReIndex()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.DbReindex;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-
-    public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
-    {
-        CommunicationObject communicationObject = new()
-        {
-            Command = CommunicationCommand.ChangeRuntimeVariable,
-            Variable = runtimeVariable.ToString(),
-            VariableValue = value
-        };
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void GetHelp()
-    {
-        Console.WriteLine("Available commands:");
-        Console.WriteLine("  stop  - stops the server");
-        Console.WriteLine("  clear - clears the console");
-        Console.WriteLine("  q     - quits the program");
-        Console.WriteLine("  p     - print the progress information of the scanner");
-        Console.WriteLine("  g     - manual garbage collect on the server");
-        Console.WriteLine("  r     - manual reindex the databases");
-        Console.WriteLine("  v     - manual vacuum the databases");
-        Console.WriteLine("  R     - change runtime variable");
-        Console.WriteLine("  lR    - list runtime variables");
-        Console.WriteLine("  help  - shows this help");
-        Console.WriteLine();
-    }
-    
-    public static void PrintRuntimeVariables()
-    {
-        Console.WriteLine("Runtime variables:");
-        Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()}   - Sets the timeout in milliseconds for the scanner");
-        Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()}    - Sets the timeout in milliseconds for the content filter");
-        Console.WriteLine($"{RuntimeVariable.DbContent.ToString()}        - Sets the wait time in milliseconds for the content database if the queue is empty");
-        Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()}      - Sets the wait time in milliseconds for the discarded database if the queue is empty");
-    }
-
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
-    {
-        //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
-        
-        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-        
-        using RequestSocket client = new();
-        client.Connect("tcp://127.0.0.1:5556");
-        client.SendFrame(lol);
-        byte[] msg = client.ReceiveFrameBytes();
-        client.Close();
-        
-        return JsonSerializer.Deserialize<string>(msg);
-    }
-
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    private static ScanningStatus GetProgress(CommunicationObject communicationObject)
-    {
-        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-        
-        using RequestSocket client = new();
-        client.Connect("tcp://127.0.0.1:5556");
-        client.SendFrame(lol);
-        byte[] msg = client.ReceiveFrameBytes();
-        client.Close();
-        
-        return JsonSerializer.Deserialize<ScanningStatus>(msg);
-    }
-}

Manager/Manager.csproj

@@ -1,25 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-
-        <!--<Platform>x64</Platform>
-        <Optimize>true</Optimize>
-        <PublishAot>true</PublishAot>
-        <PublishTrimmed>true</PublishTrimmed>
-        <TrimMode>link</TrimMode>
-        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
-    </PropertyGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\Models\Models.csproj" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-
-</Project>

Manager/Program.cs

@@ -1,90 +0,0 @@
-using Manager;
-using Models.Model.Backend;
-
-bool stop = false;
-
-do
-{
-    string? input = Console.ReadLine();
-    
-    if (string.Equals(input, "stop"))
-    {
-        Commands.StopServer();
-    }
-    
-    else if (string.Equals(input, "q"))
-    {
-        stop = true;
-    }
-    
-    else if (string.Equals(input, "clear"))
-    {
-        Console.Clear();
-    }
-    
-    else if (string.Equals(input, "p"))
-    {
-        Commands.GetProgress();
-    }
-    
-    else if (string.Equals(input, "v"))
-    {
-        Commands.Vacuum();
-    }
-
-    else if (string.Equals(input, "r"))
-    {
-        Commands.ReIndex();
-    }
-    
-    else if (string.Equals(input, "R"))
-    {
-        Console.WriteLine("Variable name.");
-        string? variable = Console.ReadLine();
-        
-        Console.WriteLine("Variable value.");
-        string? value = Console.ReadLine();
-        
-        if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
-        {
-            Console.WriteLine("Please enter a value.");
-            return;
-        }
-
-        if (variable == RuntimeVariable.ScannerTimeout.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
-        }
-        
-        else if (variable == RuntimeVariable.ContentFilter.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
-        }
-        
-        else if (variable == RuntimeVariable.DbDiscarded.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
-        }
-        
-        else if (variable == RuntimeVariable.DbContent.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
-        }
-    }
-    
-    else if (string.Equals(input, "lR"))
-    {
-        Commands.PrintRuntimeVariables();
-    }
-    
-    else if (string.Equals(input, "help"))
-    {
-        Commands.GetHelp();
-    }
-    
-    else
-    {
-        Commands.GetHelp();
-    }
-    
-} while (!stop);

Models/Experimental/CustomPing.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using Models.Model.Backend;
+namespace Models.Experimental;
+public class CustomPing
+{
+    public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
+    {
+        SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
+        using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
+        int ipHeaderLength = 20;
+            
+        try
+        {
+            rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
+            byte[] numArray = new byte[136];
+            long timestamp = Stopwatch.GetTimestamp();
+            
+            // TODO: WTF ???
+            EndPoint lol = socketConfig.EndPoint;
+                
+            while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
+            {
+                int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
+                    
+                IPStatus status;
+                if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
+                    return status;
+            }
+        }
+        catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
+        {
+        }
+            
+        return IPStatus.TimedOut;
+    }
+}

Models/Experimental/MessageConstant.cs

@@ -0,0 +1,59 @@
+using System.Net.NetworkInformation;
+namespace Models.Experimental;
+public struct MessageConstant
+{
+  public static IPStatus MapV4TypeToIpStatus(int type, int code)
+  {
+    IPStatus ipStatus1;
+    switch ((IcmpV4MessageType) type)
+    {
+      case IcmpV4MessageType.EchoReply:
+        ipStatus1 = IPStatus.Success;
+        break;
+      case IcmpV4MessageType.DestinationUnreachable:
+        IPStatus ipStatus2;
+        switch ((byte) code)
+        {
+          case 0:
+            ipStatus2 = IPStatus.DestinationNetworkUnreachable;
+            break;
+          case 1:
+            ipStatus2 = IPStatus.DestinationHostUnreachable;
+            break;
+          case 2:
+            ipStatus2 = IPStatus.DestinationProtocolUnreachable;
+            break;
+          case 3:
+            ipStatus2 = IPStatus.DestinationPortUnreachable;
+            break;
+          default:
+            ipStatus2 = IPStatus.DestinationUnreachable;
+            break;
+        }
+        ipStatus1 = ipStatus2;
+        break;
+      case IcmpV4MessageType.SourceQuench:
+        ipStatus1 = IPStatus.SourceQuench;
+        break;
+      case IcmpV4MessageType.TimeExceeded:
+        ipStatus1 = IPStatus.TtlExpired;
+        break;
+      case IcmpV4MessageType.ParameterProblemBadIpHeader:
+        ipStatus1 = IPStatus.BadHeader;
+        break;
+      default:
+        ipStatus1 = IPStatus.Unknown;
+        break;
+    }
+    return ipStatus1;
+  }
+}
+
+internal enum IcmpV4MessageType : byte
+{
+  EchoReply = 0,
+  DestinationUnreachable = 3,
+  SourceQuench = 4,
+  TimeExceeded = 11, // 0x0B
+  ParameterProblemBadIpHeader = 12, // 0x0C
+}

Models/Experimental/RawSocket.cs

@@ -0,0 +1,91 @@
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Models.Model.Backend;
+
+namespace Models.Experimental;
+
+public class RawSocket
+{
+    public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
+    {
+        Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
+        rawSocket.ReceiveTimeout = socketConfig.Timeout;
+        rawSocket.SendTimeout = socketConfig.Timeout;
+        rawSocket.Connect(socketConfig.EndPoint);
+        int num = 1;
+        rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
+        return rawSocket;
+    }
+    
+    public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
+    {
+        byte num = (byte) (receiveBuffer[0] & 15U);
+        ipHeaderLength = 4 * num;
+        
+        int start = ipHeaderLength;
+        int srcOffset = ipHeaderLength + 8;
+        
+        IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
+        byte[] numArray = new byte[bytesReceived - srcOffset];
+        Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
+        reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
+        return true;
+    }
+    
+    public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
+    {
+        int length = sizeof (IcmpHeader);
+        byte[] sendMessageBuffer = new byte[length];
+        new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
+        ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
+        sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
+        sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
+        return sendMessageBuffer;
+    }
+    
+    private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
+    {
+        uint num1 = 0;
+        for (int index = 0; index < buffer.Length; index += 2)
+        {
+            ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
+            num1 += num2;
+        }
+        while (num1 >> 16 != 0U)
+            num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
+        return (ushort) ~num1;
+    }
+}
+
+public class SocketConfig
+{
+    public EndPoint EndPoint;
+    public readonly int Timeout;
+    public readonly CustomProtocolType ProtocolType;
+    public readonly byte[] SendBuffer;
+
+    public SocketConfig(
+        EndPoint endPoint,
+        int timeout,
+        CustomProtocolType protocolType,
+        byte[] sendBuffer)
+    {
+        EndPoint = endPoint;
+        Timeout = timeout;
+        ProtocolType = protocolType;
+        SendBuffer = sendBuffer;
+    }
+}
+
+
+public struct IcmpHeader
+{
+    public byte Type;
+    public byte Code;
+    public ushort HeaderChecksum;
+    public ushort Identifier;
+    public ushort SequenceNumber;
+}

Models/Handler/DbHandler.cs

@@ -1,7 +1,10 @@
 using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
 using Microsoft.Data.Sqlite;
+using Models.Helper;
 using Models.Model.Backend;
-using Models.Model.External;
 
 namespace Models.Handler;
 
@@ -11,41 +14,78 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _discardedConnectionString;
+    private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
-    private readonly List<string> _discardedConnectionStrings = [];
+    private readonly string _compressedConnectionString;
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
                                            " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
+    private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                           " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
+                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
+    
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
-                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
+                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
+                                              " Url1, Url2, ServerType1, ServerType2," +
                                               " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
                                               " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
                                               " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
-                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2," +
-                                              " @title1, @title2, @description1, @description2, @url1, @url2, @serverType1," +
-                                              " @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2," +
-                                              " @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion," +
-                                              " @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2," +
-                                              " @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
+                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
+                                              " @url1, @url2, " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
+                                              " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
+                                              " @robotsTXT1, @robotsTXT2," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion1)," +
+                                              " (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion2)," +
+                                              " (SELECT CertificateIssuerId FROM CertificateIssuerCountry WHERE Country = @certificateIssuerCountry)," +
+                                              " (SELECT CertificateOrganizationId FROM CertificateOrganizationName WHERE Name = @certificateOrganizationName), " +
+                                              " @ipV6, " +
+                                              " (SELECT TlsId FROM TlsVersion WHERE Version = @tlsVersion)," +
+                                              " (SELECT CipherId FROM CipherSuite WHERE Suite = @cipherSuite)," +
+                                              " (SELECT KeyExchangeId FROM KeyExchangeAlgorithm WHERE Algorithm = @keyExchangeAlgorithm)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType1)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType2)," +
+                                              " (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType3)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding1)," +
+                                              " (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding2)," +
+                                              " (SELECT ALPNId FROM ALPN WHERE ALPNValue = @aLPN)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection1)," +
+                                              " (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection2))";
+
+    private const string InsertIntoFilteredServerType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ServerType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredHttpVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO HttpVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCertificateIssuerCountry = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateIssuerCountry (Country) VALUES (@country)";
+    private const string InsertIntoFilteredCertificateOrganizationName = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateOrganizationName (Name) VALUES (@name)";
+    private const string InsertIntoFilteredTlsVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO TlsVersion (Version) VALUES (@version)";
+    private const string InsertIntoFilteredCipherSuite = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CipherSuite (Suite) VALUES (@suite)";
+    private const string InsertIntoFilteredKeyExchangeAlgorithm = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO KeyExchangeAlgorithm (Algorithm) VALUES (@algorithm)";
+    private const string InsertIntoFilteredPublicKeyType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO PublicKeyType (Type) VALUES (@type)";
+    private const string InsertIntoFilteredAcceptEncoding = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO AcceptEncoding (Encoding) VALUES (@encoding)";
+    private const string InsertIntoFilteredALPN = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ALPN (ALPNValue) VALUES (@alpnValue)";
+    private const string InsertIntoFilteredConnection = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO Connection (ConnectionValue) VALUES (@connectionValue)";
     
     private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
-                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+                                               " INSERT INTO Discarded (PackedData)" +
+                                               " VALUES (@packedData)";
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
-                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
+
+    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@@ -53,20 +93,26 @@ public class DbHandler
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
-    private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
+    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
+    private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
+    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Id == @id ORDER BY Id ASC LIMIT 1;";
+    private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
+    private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
+    
     private const string ReIndexDatabasesStatement = "REINDEX;";
     
     private const string VacuumDatabasesStatement = "VACUUM;";
     
-    private readonly object _readFilteredLock = new();
     private readonly object _readAndDeleteResumeLock = new();
     
     private bool _stop;
     private bool _pause;
     private bool _paused;
+    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
@@ -76,12 +122,15 @@ public class DbHandler
     public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, 
+        ConcurrentQueue<Ip> preFilteredQueue,
+        string basePath)
     {
         _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
+        _preFilteredQueue = preFilteredQueue;
 
         SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
@@ -89,9 +138,10 @@ public class DbHandler
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
-        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
+        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
+        _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -153,6 +203,24 @@ public class DbHandler
         
         Console.WriteLine("Filtered DbHandler stopped.");
     }
+
+    public void PrefilteredDbHandler()
+    {
+        Console.WriteLine("PreFiltered Db handler started.");
+        
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(4);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out Ip queueItem);
+            
+            InsertPrefiltered(queueItem);
+        }
+    }
     
     public void ResumeDbHandler()
     {
@@ -167,12 +235,9 @@ public class DbHandler
                 continue;
             }
             
-            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
+            _resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
             
-            if (queueItem is not null)
-            {
-                InsertResumeObject(queueItem);
-            }
+            InsertResumeObject(queueItem);
         }
         
         Console.WriteLine("Resume DbHandler stopped.");
@@ -185,7 +250,7 @@ public class DbHandler
         for (int i = 0; i < threads; i++)
         {
             EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-
+            
             DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
             {
                 Handle = handle,
@@ -197,7 +262,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(1000);
+            Thread.Sleep(150);
         }
         
         return waitHandles;
@@ -208,9 +273,20 @@ public class DbHandler
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
         
-        while (!_stop)
+        int i = 0;
+        
+        SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        SqliteCommand command = new(InsertIntoDiscarded, connection);
+        
+        StringBuilder stringBuilder = new();
+        
+        bool running = true;
+        
+        while (!_stop && running)
         {
             if (_discardedQueue.IsEmpty || _pause)
             {
@@ -218,17 +294,75 @@ public class DbHandler
                 _paused = true;
                 continue;
             }
+
+            if (_stop && _discardedQueue.IsEmpty)
+            {
+                running = false;
+            }
             
-            _discardedQueue.TryDequeue(out Discarded queueItem);
+            if (i == 2048)
+            {
+                command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+                _ = command.ExecuteNonQuery();
+                
+                // Re-use the command object.
+                command.Parameters.Clear();
+                
+                stringBuilder.Clear();
+                
+                i = 0;
+            }
             
-            InsertDiscarded(queueItem, connectionString);
+            if (_discardedQueue.TryDequeue(out Discarded queueItem))
+            {
+                stringBuilder.Append(queueItem.Ip.PackIp());
+                stringBuilder.Append(':');
+                stringBuilder.Append(queueItem.ResponseCode);
+                stringBuilder.Append(',');
+            }
+            
+            i++;
         }
         
+        if (stringBuilder.Length != 0)
+        {
+            command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+            _ = command.ExecuteNonQuery();
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        
         discardedDbHandlerSetting.Handle!.Set();
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
     
+    private void DropAndCreateDiscarded(int threadNumber)
+    {
+        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
+        
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string dropStatement = "DROP TABLE Discarded;";
+        const string vacuum = "VACUUM;";
+        
+        using SqliteConnection connection = new(databaseName);
+        connection.Open();
+        
+        SqliteCommand command = new(dropStatement, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(vacuum, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(createStatement, connection);
+        command.ExecuteNonQuery();
+        command.Dispose();
+        
+        connection.Close();
+    }
+    
     private void InsertUnfiltered(Unfiltered unfiltered)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
@@ -248,30 +382,80 @@ public class DbHandler
         connection.Close();
     }
     
-    private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
-    {
-        using SqliteConnection connection = new(dbConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoDiscarded, connection);
-        
-        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
-        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
-        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
-        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
-        command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
-        _ = command.ExecuteNonQuery();
-        
-        connection.Close();
-    }
-    
     private void InsertFiltered(Filtered filtered)
     {
         using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
-        using SqliteCommand command = new(InsertIntoFiltered, connection);
+        SqliteCommand command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType1);
+        _ = command.ExecuteNonQuery();
         
+        command = new(InsertIntoFilteredServerType, connection);
+        command.Parameters.AddWithValue("@type", filtered.ServerType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredHttpVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.HttpVersion2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateIssuerCountry, connection);
+        command.Parameters.AddWithValue("@country", filtered.CertificateIssuerCountry);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCertificateOrganizationName, connection);
+        command.Parameters.AddWithValue("@name", filtered.CertificateOrganizationName);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredTlsVersion, connection);
+        command.Parameters.AddWithValue("@version", filtered.TlsVersion);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredCipherSuite, connection);
+        command.Parameters.AddWithValue("@suite", filtered.CipherSuite);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredKeyExchangeAlgorithm, connection);
+        command.Parameters.AddWithValue("@algorithm", filtered.KeyExchangeAlgorithm);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredPublicKeyType, connection);
+        command.Parameters.AddWithValue("@type", filtered.PublicKeyType3);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredAcceptEncoding, connection);
+        command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredALPN, connection);
+        command.Parameters.AddWithValue("@alpnValue", filtered.ALPN);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection1);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFilteredConnection, connection);
+        command.Parameters.AddWithValue("@connectionValue", filtered.Connection2);
+        _ = command.ExecuteNonQuery();
+        
+        command = new(InsertIntoFiltered, connection);
         command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
         command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
         command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
@@ -280,10 +464,6 @@ public class DbHandler
         command.Parameters.AddWithValue("@port2", filtered.Port2);
         command.Parameters.AddWithValue("@url1", filtered.Url1);
         command.Parameters.AddWithValue("@url2", filtered.Url2);
-        command.Parameters.AddWithValue("@title1", filtered.Title1);
-        command.Parameters.AddWithValue("@title2", filtered.Title2);
-        command.Parameters.AddWithValue("@description1", filtered.Description1);
-        command.Parameters.AddWithValue("@description2", filtered.Description2);
         command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
         command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
         command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
@@ -304,8 +484,9 @@ public class DbHandler
         command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
         command.Parameters.AddWithValue("@connection1", filtered.Connection1);
         command.Parameters.AddWithValue("@connection2", filtered.Connection2);
-        
         _ = command.ExecuteNonQuery();
+        command.Dispose();
+        
         connection.Close();
     }
     
@@ -313,19 +494,50 @@ public class DbHandler
     {
         using SqliteConnection connection = new(_resumeConnectionString);
         connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoResume, connection);
+        SqliteCommand command;
+
+        if (resumeObject.Operation == Operations.Insert)
+        {
+            command = new(InsertIntoResume, connection);
+            command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
+            command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
+        }
+        else
+        {
+            command = new(UpdateResumeStatement, connection);
+        }
         
         command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
-        command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
-        command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
         command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
         command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
         command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
         command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
+        command.Parameters.AddWithValue("@paused", resumeObject.Paused);
+        command.Parameters.AddWithValue("@completed", resumeObject.Completed);
         
         _ = command.ExecuteNonQuery();
         connection.Close();
+        
+        command.Dispose();
+    }
+
+    private void InsertPrefiltered(Ip filterQueueItem)
+    {
+        using SqliteConnection connection = new(_preFilteredConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertPreFilteredStatement, connection);
+        
+        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip1);
+        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip2);
+        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip3);
+        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip4);
+        command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
+        command.Parameters.AddWithValue("@filtered", 0);
+        
+        _ = command.ExecuteNonQuery();
+        
+        connection.Close();
     }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
@@ -395,52 +607,84 @@ public class DbHandler
         return ids;
     }
     
-    public long GetFilteredIndexes()
+    public void GetPreFilterQueueItem()
     {
-        long rowId = 0;
+        using SqliteConnection connection = new(_preFilteredConnectionString);
+        connection.Open();
+
+        const string temp = "SELECT seq FROM sqlite_sequence;";
         
-        using SqliteConnection connection = new(_filteredConnectionString);
+        SqliteCommand command = new(ReadPreFilteredStatement, connection);
+        SqliteCommand command2 = new(UpdatePreFilteredStatement, connection);
+        SqliteCommand command3 = new(temp, connection);
+        SqliteDataReader reader = command3.ExecuteReader();
+
+        long count = 0;
+        
+        while (reader.Read())
+        {
+            count = reader.GetInt64(0);
+        }
+        
+        reader.Close();
+        
+        for (long i = 0; i < count; i++)
+        {
+            if (_preFilteredQueue.Count > 1000)
+            {
+                Thread.Sleep(5);
+            }
+            
+            command.Parameters.AddWithValue("@id", i);
+            reader = command.ExecuteReader();
+            command.Parameters.Clear();
+            
+            Ip ip = new();
+
+            if (!reader.HasRows)
+            {
+                Console.WriteLine("Uuhhh");
+            }
+             
+            while (reader.Read())
+            {
+                ip.Ip1 = reader.GetInt32(0);
+                ip.Ip2 = reader.GetInt32(1);
+                ip.Ip3 = reader.GetInt32(2);
+                ip.Ip4 = reader.GetInt32(3);
+                ip.ResponseCode = reader.GetInt32(4);
+            }
+            
+            reader.Close();
+            
+            _preFilteredQueue.Enqueue(ip);
+            
+            command2.Parameters.AddWithValue("@id", i);
+            command2.ExecuteNonQuery();
+            command2.Parameters.Clear();
+        }
+    }
+
+    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    {
+        using SqliteConnection connection = new(connectionString);
         connection.Open();
         
-        using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
+        using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
         using SqliteDataReader reader = command.ExecuteReader();
         
+        long rowId = 0;
+        
         if (!reader.HasRows)
         {
-            return 0;
+            return rowId;
         }
         
         while (reader.Read())
         {
-            rowId = reader.GetInt64(0);
+            rowId += reader.GetInt64(0);
         }
-        
-        return rowId;
-    }
-    
-    public long GetDiscardedIndexes()
-    {
-        long rowId = 0;
-        
-        for (int i = 0; i < _discardedConnectionStrings.Count; i++)
-        {
-            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
-            connection.Open();
-            
-            using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return 0;
-            }
-            
-            while (reader.Read())
-            {
-                rowId += reader.GetInt64(0);
-            }
-        }
-        
+
         return rowId;
     }
 
@@ -495,36 +739,6 @@ public class DbHandler
         return true;
     }
     
-    public List<SearchResult?> GetSearchResults()
-    {
-        lock (_readFilteredLock)
-        {
-            using SqliteConnection connection = new(_filteredConnectionString);
-            connection.Open();
-            
-            using SqliteCommand command = new(ReadFilteredStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return [];
-            }
-            
-            List<SearchResult?> results = [];
-            
-            while (reader.Read())
-            {
-                SearchResult result = new();
-                result.Title = reader.GetString(0);
-                result.Url = reader.GetString(1);
-                
-                results.Add(result);
-            }
-            
-            return results;
-        }
-    }
-    
     public ScannerResumeObject? GetResumeObject(int threadNumber)
     {
         lock (_readAndDeleteResumeLock)
@@ -532,7 +746,7 @@ public class DbHandler
             using SqliteConnection connection = new(_resumeConnectionString);
             connection.Open();
             
-            using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
+            using SqliteCommand command = new(ReadResumeStatement, connection);
             command.Parameters.AddWithValue("@threadNumber", threadNumber);
             
             using SqliteDataReader reader = command.ExecuteReader();
@@ -553,96 +767,20 @@ public class DbHandler
                 resumeObject.SecondByte = reader.GetInt32(4);
                 resumeObject.ThirdByte = reader.GetInt32(5);
                 resumeObject.FourthByte = reader.GetInt32(6);
+                resumeObject.Paused = reader.GetBoolean(7);
+                resumeObject.Completed = reader.GetBoolean(8);
             }
             
             return resumeObject;
         }
     }
-    
-    public void ReIndex()
-    {
-        _pause = true;
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
-        
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_discardedConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
-    
-    public void Vacuum()
-    {
-        _pause = true;
-        
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
 
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_discardedConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
-
-    private string CreateDiscardedDb(int threadNumber)
+    private (string, string) CreateDiscardedDb(int threadNumber)
     {
+        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 TEXT NOT NULL, Ip2 TEXT NOT NULL, Ip3 TEXT NOT NULL, Ip4 TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-
-        _discardedConnectionStrings.Add(databaseName);
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
         
         using SqliteConnection connection = new(databaseName);
         connection.Open();
@@ -650,7 +788,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return databaseName;
+        return (absolutePath, databaseName);
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -0,0 +1,50 @@
+using System.IO.Compression;
+using System.Text;
+
+namespace Models.Helper;
+
+public static class CompressionHelper
+{
+    public static void CompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
+        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize);
+        originalFileStream.CopyTo(compressor);
+    }
+    
+    public static void DecompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
+        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
+        decompressor.CopyTo(decompressedFileStream);
+    }
+    
+    public static string CompressString(string text)
+    {
+        byte[] byteArray = Encoding.UTF8.GetBytes(text);
+        
+        using MemoryStream memoryStream = new();
+        
+        using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize))
+        {
+            compressionStream.Write(byteArray, 0, byteArray.Length);
+        }
+        
+        return Convert.ToBase64String(memoryStream.ToArray());
+    }
+    
+    public static string DecompressString(string compressedText)
+    {
+        byte[] byteArray = Convert.FromBase64String(compressedText);
+        
+        using MemoryStream memoryStream = new(byteArray);
+        
+        using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress);
+        
+        using StreamReader reader = new(decompressionStream);
+        
+        return reader.ReadToEnd();
+    }
+}

Models/Helper/GetDatabasesHelper.cs

@@ -0,0 +1,9 @@
+namespace Models.Helper;
+
+public static class GetDatabasesHelper
+{
+    public static int GetTotalCompressedDatabases(string path)
+    {
+        return Directory.GetFiles(path, "*.gz").Length;
+    }
+}

Models/Model/Backend/CustomProtocolType.cs

@@ -0,0 +1,6 @@
+namespace Models.Model.Backend;
+
+public enum CustomProtocolType
+{
+    Icmp = 1
+}

Models/Model/Backend/DatabaseSizes.cs

@@ -1,10 +0,0 @@
-namespace Models.Model.Backend;
-
-public struct DatabaseSizes
-{
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/Backend/FilterQueueItem.cs

@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public struct FilterQueueItem
+{
+    public Ip Ip { get; set; }
+    public int ResponseCode { get; set; }
+}

Models/Model/Backend/Filtered.cs

@@ -3,10 +3,6 @@ namespace Models.Model.Backend;
 public class Filtered
 {
     public Ip Ip { get; set; }
-    public string Title1 { get; set; } = "";
-    public string Title2 { get; set; } = "";
-    public string Description1 { get; set; } = "";
-    public string Description2 { get; set; } = "";
     public string Url1 { get; set; } = "";
     public string Url2 { get; set; } = "";
     public int Port1 { get; set; }

Models/Model/Backend/IP.cs

@@ -9,9 +9,16 @@ public struct Ip
     public int Ip3 { get; set; }
     
     public int Ip4 { get; set; }
-
+    
+    public int ResponseCode { get; set; }
+    
     public override string ToString()
     {
         return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
     }
+    
+    public uint PackIp()
+    {
+        return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4;
+    }
 }

Models/Model/Backend/RuntimeVariable.cs

@@ -1,9 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum RuntimeVariable
-{
-    DbContent,
-    DbDiscarded,
-    ContentFilter,
-    ScannerTimeout
-}

Models/Model/Backend/ScannerResumeObject.cs

@@ -9,4 +9,7 @@ public class ScannerResumeObject
     public int ThirdByte { get; set; }
     public int FourthByte { get; set; }
     public int ThreadNumber { get; set; }
+    public Operations Operation { get; set; }
+    public bool Paused { get; set; }
+    public bool Completed { get; set; }
 }

Models/Model/Backend/SizeUnits.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.Backend;
-public enum SizeUnits
-{
-    Byte,
-    KB,
-    MB,
-    GB,
-}

Models/Model/External/CommunicationCommand.cs

@@ -1,11 +0,0 @@
-namespace Models.Model.External;
-
-public enum CommunicationCommand
-{
-    GetScanningProgress,
-    GetSearches,
-    StopScanning,
-    DbReindex,
-    DbVacuum,
-    ChangeRuntimeVariable,
-}

Models/Model/External/CommunicationObject.cs

@@ -1,17 +0,0 @@
-namespace Models.Model.External;
-
-//[MessagePackObject]
-public class CommunicationObject
-{
-    //[Key(0)]
-    public CommunicationCommand Command { get; set; }
-    
-    //[Key(1)]
-    public string? SearchTerm { get; set; } = "";
-    
-    //[Key(2)]
-    public string? Variable { get; set; } = "";
-    
-    //[Key(3)]
-    public string? VariableValue { get; set; } = "";
-}

Models/Model/External/CommunicationResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class CommunicationResult
-{
-    public List<SearchResult?>? Result { get; set; }
-    
-    public ScanningStatus? Status { get; set; }
-}

Models/Model/External/ScanningStatus.cs

@@ -1,18 +0,0 @@
-namespace Models.Model.External;
-
-public struct ScanningStatus
-{
-    public float PercentageOfIpv4Scanned { get; set; }
-    
-    public long TotalFiltered { get; set; }
-    
-    public long AmountOfIpv4Left { get; set; }
-    
-    public long TotalDiscarded { get; set; }
-    
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/External/SearchResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResult
-{
-    public string? Title { get; set; } = "";
-    public string? Url { get; set; } = "";
-    public string? Description { get; set; } = "";
-}

Models/Model/External/SearchResults.cs

@@ -1,6 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResults
-{
-    public List<SearchResult?>? Results { get; set; }
-}

Models/Models.csproj

@@ -1,13 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
-      <PackageReference Include="SQLite" Version="3.13.0" />
+      <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
     </ItemGroup>
 </Project>

Models/Sql/DropAndCreateFiltered.sql

@@ -1,10 +1,68 @@
 VACUUM;
 
+CREATE TABLE IF NOT EXISTS "ServerType" (
+    "ServerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "HttpVersion" (
+    "HttpId" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateIssuerCountry" (
+	"CertificateIssuerId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Country" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CertificateOrganizationName" (
+	"CertificateOrganizationId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Name" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "TlsVersion" (
+	"TlsId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Version" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "CipherSuite" (
+	"CipherId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Suite" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "KeyExchangeAlgorithm" (
+	"KeyExchangeId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Algorithm" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "PublicKeyType" (
+	"PublicKeyId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Type" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "AcceptEncoding" (
+	"AcceptId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"Encoding" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "ALPN" (
+	"ALPNId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ALPNValue" TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS "Connection" (
+	"ConnectionId" INTEGER PRIMARY KEY AUTOINCREMENT,
+	"ConnectionValue" TEXT NOT NULL UNIQUE
+);
+
 DROP TABLE Filtered;
 
 CREATE TABLE "Filtered" (
 	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"	INTEGER NOT NULL,
+	"Ip3"	INTEGER NOT NULL,
+	"Ip4"	INTEGER NOT NULL,
 	"Port1"	INTEGER NOT NULL,
 	"Port2"	INTEGER NOT NULL,
 	"Title1"	TEXT NOT NULL,
@@ -13,25 +71,42 @@ CREATE TABLE "Filtered" (
 	"Description2"	TEXT NOT NULL,
 	"Url1"	TEXT NOT NULL,
 	"Url2"	TEXT NOT NULL,
-	"ServerType1"	TEXT NOT NULL,
-	"ServerType2"	TEXT NOT NULL,
+	"ServerType1"	INTEGER NOT NULL,
+	"ServerType2"	INTEGER NOT NULL,
 	"RobotsTXT1"	TEXT NOT NULL,
 	"RobotsTXT2"	TEXT NOT NULL,
-	"HttpVersion1"	TEXT NOT NULL,
-	"HttpVersion2"	TEXT NOT NULL,
-	"CertificateIssuerCountry"	TEXT NOT NULL,
-	"CertificateOrganizationName"	TEXT NOT NULL,
+	"HttpVersion1"	INTEGER NOT NULL,
+	"HttpVersion2"	INTEGER NOT NULL,
+	"CertificateIssuerCountry"	INTEGER NOT NULL,
+	"CertificateOrganizationName"	INTEGER NOT NULL,
 	"IpV6"	TEXT NOT NULL,
-	"TlsVersion"	TEXT NOT NULL,
-	"CipherSuite"	TEXT NOT NULL,
-	"KeyExchangeAlgorithm"	TEXT NOT NULL,
-	"PublicKeyType1"	TEXT NOT NULL,
-	"PublicKeyType2"	TEXT NOT NULL,
-	"PublicKeyType3"	TEXT NOT NULL,
-	"AcceptEncoding1"	TEXT NOT NULL,
-	"AcceptEncoding2"	TEXT NOT NULL,
-	"ALPN"	TEXT NOT NULL,
-	"Connection1" TEXT NOT NULL,
-	"Connection2" TEXT NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
+	"TlsVersion"	INTEGER NOT NULL,
+	"CipherSuite"	INTEGER NOT NULL,
+	"KeyExchangeAlgorithm"	INTEGER NOT NULL,
+	"PublicKeyType1"	INTEGER NOT NULL,
+	"PublicKeyType2"	INTEGER NOT NULL,
+	"PublicKeyType3"	INTEGER NOT NULL,
+	"AcceptEncoding1"	INTEGER NOT NULL,
+	"AcceptEncoding2"	INTEGER NOT NULL,
+	"ALPN"	INTEGER NOT NULL,
+	"Connection1"	INTEGER NOT NULL,
+	"Connection2"	INTEGER NOT NULL,
+	PRIMARY KEY("Id" AUTOINCREMENT),
+	FOREIGN KEY("ALPN") REFERENCES "ALPN"("ALPNId"),
+	FOREIGN KEY("AcceptEncoding1") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("AcceptEncoding2") REFERENCES "AcceptEncoding"("AcceptId"),
+	FOREIGN KEY("CertificateIssuerCountry") REFERENCES "CertificateIssuerCountry"("CertificateIssuerId"),
+	FOREIGN KEY("CertificateOrganizationName") REFERENCES "CertificateOrganizationName"("CertificateOrganizationId"),
+	FOREIGN KEY("CipherSuite") REFERENCES "CipherSuite"("CipherId"),
+	FOREIGN KEY("Connection1") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("Connection2") REFERENCES "Connection"("ConnectionId"),
+	FOREIGN KEY("HttpVersion1") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("HttpVersion2") REFERENCES "HttpVersion"("HttpId"),
+	FOREIGN KEY("KeyExchangeAlgorithm") REFERENCES "KeyExchangeAlgorithm"("KeyExchangeId"),
+	FOREIGN KEY("PublicKeyType1") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType2") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("PublicKeyType3") REFERENCES "PublicKeyType"("PublicKeyId"),
+	FOREIGN KEY("ServerType1") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("ServerType2") REFERENCES "ServerType"("ServerId"),
+	FOREIGN KEY("TlsVersion") REFERENCES "TlsVersion"("TlsId")
 )

Proxy/Program.cs

@@ -1,72 +0,0 @@
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using MessagePack;
-using Microsoft.AspNetCore.Cors.Infrastructure;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-
-WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
-
-builder.Services.ConfigureHttpJsonOptions(options =>
-{
-    options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
-});
-
-builder.Services.AddCors(options =>
-{
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
-});
-
-WebApplication app = builder.Build();
-
-app.UseCors();
-
-RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
-{
-    CommunicationObject communicationObject = new()
-    {
-        Command = CommunicationCommand.GetScanningProgress
-    };
-    
-    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-    
-    using RequestSocket client = new();
-    client.Connect("tcp://127.0.0.1:5556");
-    client.SendFrame(bytes);
-    byte[] msg = client.ReceiveFrameBytes();
-    client.Close();
-    
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
-});
-
-RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
-searchApi.MapGet("/{term}", (string term) =>
-{
-    CommunicationObject communicationObject = new();
-    communicationObject.Command = CommunicationCommand.GetSearches;
-    communicationObject.SearchTerm = term;
-    
-    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-    
-    using RequestSocket client = new();
-    client.Connect("tcp://127.0.0.1:5556");
-    client.SendFrame(bytes);
-    string msg = client.ReceiveFrameString();
-    client.Close();
-    
-    return JsonSerializer.Deserialize<SearchResults?>(msg);
-});
-
-app.Run();
-
-[JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
-[JsonSerializable(typeof(CommunicationObject))]
-internal partial class AppJsonSerializerContext : JsonSerializerContext
-{
-}

Proxy/Properties/launchSettings.json

@@ -1,15 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "",
-      "applicationUrl": "http://localhost:5224",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    }
-  }
-}

Proxy/Proxy.csproj

@@ -1,19 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <Nullable>enable</Nullable>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <InvariantGlobalization>true</InvariantGlobalization>
-        <PublishAot>false</PublishAot>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\Models\Models.csproj" />
-    </ItemGroup>
-    
-</Project>

Proxy/Proxy.http

@@ -1,11 +0,0 @@
-@Proxy_HostAddress = http://localhost:5224
-
-GET {{Proxy_HostAddress}}/progress/
-Accept: application/json
-
-###
-
-GET {{Proxy_HostAddress}}/search/asd
-Accept: application/json
-
-###

Proxy/appsettings.Development.json

@@ -1,8 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

Proxy/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

README.md

@@ -2,4 +2,8 @@
 
 Rasmus Search Engine
 
-This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
+This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
+
+This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture.
+
+I'm just trying to minimize memory usage while maximizing performance.

RSE.sln

@@ -4,11 +4,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -24,17 +20,9 @@ Global
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,5 +1,42 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueueSegment_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4f72364deba8e6142d1f42386f1fd282d2e8c6bca41d389c28da4c3835d90_003FConcurrentQueueSegment_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F85_003Fa8fb60ce_003FConcurrentQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADnsEndPoint_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F3d_003F64a36162_003FDnsEndPoint_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIcmpV6MessageType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F4e_003Fec7b627f_003FIcmpV6MessageType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F6079bfa8f3c0474bb21ade75ce5d010318000_003F15_003F57afed32_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4682fe5857f946a98dcb2cd6f0a403983f200_003F3d_003Fca9070b2_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARawSocketPermissions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003Fc7_003Fb8dcfca9_003FRawSocketPermissions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteConnection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc1d827416891bc9ceaece7826dbd14f2ab1ef304d85bcc1f2187646a67c3c_003FSqliteConnection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F13_003Fbe8f056e_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F90_003F0588ec81_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}

frontend/.gitignore

@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example

frontend/README.md

@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.

frontend/app.vue

@@ -0,0 +1,20 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+    <ul>
+      <li><nuxt-link to="/">Search</nuxt-link></li>
+      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
+    </ul>
+  </div>
+
+  <NuxtPage page-key="static" />
+</template>

frontend/assets/css/main.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/nuxt.config.ts

@@ -0,0 +1,21 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  nitro: {
+    static: true,
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});

frontend/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}

frontend/pages/index.vue

@@ -0,0 +1,15 @@
+<script setup lang="ts">
+const value = ref<string | null>(null)
+
+// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
+</script>
+
+
+<template>
+  <div class="h-screen flex items-center justify-center">
+    <div class="flex space-x-2">
+      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
+    </div>
+  </div>
+</template>

frontend/pages/progress.vue

@@ -0,0 +1,92 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <table class="table-auto border-gray-300">
+      <thead>
+        <tr>
+          <th class="px-4 py-2">Metric</th>
+          <th class="px-4 py-2">Value</th>
+        </tr>
+      </thead>
+
+      <tbody v-if="dict">
+        <tr v-for="d in dict">
+          <td class="px-4 py-2">{{d.description}}</td>
+          <td class="px-4 py-2">{{d.value}}</td>
+        </tr>
+      </tbody>
+      <tbody v-else>
+      <tr>
+        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total filtered</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total Discarded</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Amount of Ipv4 left</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Discarded db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Filtered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Unfiltered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      </tbody>
+    </table>
+  </div>
+</template>
+
+<style scoped>
+
+</style>

frontend/pages/searchResult.vue

@@ -0,0 +1,8 @@
+<script setup lang="ts">
+const route = useRoute();
+  console.log(route.query);
+</script>
+
+<template>
+  
+</template>

frontend/public/robots.txt

@@ -0,0 +1 @@
+

frontend/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}

frontend/tailwind.config.js

@@ -0,0 +1,22 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {
+      width: {
+        '128': '32rem',
+        '192' : '48rem',
+        '256': '64rem',
+      }
+    },
+  },
+  plugins: [],
+}
+

frontend/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}

frontend/types/global.d.ts

@@ -0,0 +1,29 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+};
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+};
+
+interface SearchResult {
+    title: string;
+    description: string;
+    url: string;
+};
+
+interface SearchResults {
+    results: SearchResult[];
+};
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};

frontend/utils/cacheUtil.ts

@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}

frontend/utils/convertProgressTypeToDictionary.ts

@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}

global.json

@@ -0,0 +1,7 @@
+{
+  "sdk": {
+    "version": "9.0.0",
+    "rollForward": "latestMajor",
+    "allowPrerelease": true
+  }
+}