Removed .txt

Reviewed-on: #45

.gitignore

@@ -7,6 +7,7 @@
 **/**/bin/*
 **/obj/*
 **/**/obj/*
+*.gz
 
 # User-specific stuff
 .idea/**/workspace.xml

Analyze/Analyze.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <Folder Include="CompressedDatabases\" />
+    </ItemGroup>
+
+</Project>

Analyze/Program.cs

@@ -0,0 +1,3 @@
+// See https://aka.ms/new-console-template for more information
+
+Console.WriteLine("Hello, World!");

Backend/Backend.csproj

@@ -2,21 +2,15 @@
     
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-
-        <!--<Platform>x64</Platform>-->
-        <Optimize>false</Optimize>
+        <PublishAot>false</PublishAot>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
     
     <ItemGroup>
       <ProjectReference Include="..\Models\Models.csproj" />
     </ItemGroup>
     
-    <ItemGroup>
-      <PackageReference Include="FuzzySharp" Version="2.0.2" />
-      <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-    
 </Project>

Backend/Handler/Communication.cs

@@ -1,201 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using Backend.Helper;
-using Models.Handler;
-using Models.Model.Backend;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-
-namespace Backend.Handler;
-
-public class Communication
-{
-    private readonly DbHandler _dbHandler;
-    private readonly ThreadHandler _threadHandler;
-    private readonly IpScanner _ipScanner;
-    private readonly ContentFilter _contentFilter;
-    private bool _isRunning = true;
-    private string _basePath;
-    
-    public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
-    {
-        _dbHandler = dbHandler;
-        _threadHandler = threadHandler;
-        _ipScanner = ipScanner;
-        _contentFilter = contentFilter;
-        _basePath = basePath;
-    }
-    
-    public WaitHandle[] Start()
-    {
-        WaitHandle[] waitHandles = new WaitHandle[1];
-        EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-        waitHandles[0] = handle;
-        
-        Thread thread = new(Server!);
-        thread.Start(handle);
-        
-        return waitHandles;
-    }
-    
-    private void Server(object obj)
-    {
-        using ResponseSocket rep = new();
-        
-        //rep.Options.IPv4Only = true;
-        
-        rep.Bind("tcp://127.0.0.1:5556");
-        
-        while (_isRunning)
-        {
-            byte[] message = rep.ReceiveFrameBytes();
-            
-            CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
-
-            //rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
-            
-            if (communicationObject is null)
-            {
-                continue;
-            }
-            
-            OnServerOnReceiveReady(communicationObject, rep);
-        }
-        
-        ((EventWaitHandle) obj).Set();
-    }
-    
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
-    private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
-    {
-        switch (communicationObject.Command)
-        {
-            case CommunicationCommand.GetScanningProgress:
-            {
-                DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
-                
-                long discardedIndexes = _dbHandler.GetDiscardedIndexes();
-                
-                ScanningStatus status = new();
-                // 4294967296 is all Ipv4 addresses.
-                
-                if (discardedIndexes != 0)
-                {
-                    status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
-                }
-                else
-                {
-                    // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
-                    status.PercentageOfIpv4Scanned = 0.0000000001f;
-                }
-                
-                status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
-                status.TotalFiltered = _dbHandler.GetFilteredIndexes();
-                status.TotalDiscarded = discardedIndexes;
-                status.MyDbSize = databaseSizes.MyDbSize;
-                status.FilteredDbSize = databaseSizes.FilteredDbSize;
-                status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
-                
-                byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
-                
-                rep.SendFrame(serializedResult);
-                
-                break;
-            }
-            
-            case CommunicationCommand.DbReindex:
-            {
-                _dbHandler.ReIndex();
-                
-                SendStringResponse(rep, "All Dbs have been reindexed.");
-                break;
-            }
-            
-            case CommunicationCommand.DbVacuum:
-            {
-                _dbHandler.Vacuum();
-                
-                SendStringResponse(rep, "All Dbs have been vacuumed.");
-                break;
-            }
-            
-            case CommunicationCommand.GetSearches:
-            {
-                if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
-                {
-                    SendSearchResponse(rep, communicationObject.SearchTerm);
-                }
-                
-                break;
-            }
-            
-            case CommunicationCommand.StopScanning:
-            {
-                _isRunning = false;
-                break;
-            }
-            
-            case CommunicationCommand.ChangeRuntimeVariable:
-            {
-                if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
-                
-                if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _dbHandler.SetContentWaitTime(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _dbHandler.SetDiscardedWaitTime(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _ipScanner.SetTimeout(value);
-                }
-                
-                if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
-                {
-                    int value = int.Parse(communicationObject.VariableValue);
-                    _contentFilter.SetTimeout(value);
-                }
-                
-                rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
-                
-                break;
-            }
-        }
-    }
-    
-    private static void SendStringResponse(ResponseSocket rep, string response)
-    {
-        byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
-        
-        rep.SendFrame(serializedResult);
-    }
-
-    private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
-    {
-        //SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
-        SearchResults result = new()
-        {
-            Results = []
-        };
-        SearchResult lol = new()
-        {
-            Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
-            Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
-        };
-        
-        result.Results.Add(lol);
-        
-        string serializedResult = JsonSerializer.Serialize(result);
-        
-        rep.SendFrame(serializedResult);
-    }
-}

Backend/Handler/ContentFilter.cs

@@ -6,27 +6,44 @@ using Models.Model.Backend;
 
 namespace Backend.Handler;
 
+public class Content
+{
+    public int Port1 { get; set; }
+    public int Port2 { get; set; }
+    public Ip Ip { get; set; }
+}
+
+public class ContentThread
+{
+    public int ThreadId { get; set; }
+    public EventWaitHandle? EventWaitHandle { get; set; }
+}
+
 public class ContentFilter
 {
     private readonly ConcurrentQueue<Filtered> _queue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<Content?> _contentQueue = new();
     private readonly DbHandler _dbHandler;
+    private readonly ThreadHandler _threadHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
     private bool _stop;
     private int _timeOut;
     private readonly string _basePath;
-    
-    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
+
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
+        _threadHandler = threadHandler;
         _unfilteredQueue = unfilteredQueue;
         
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
+        
         SetTimeout(3000);
     }
     
@@ -52,11 +69,23 @@ public class ContentFilter
         while (!_stop)
         {
             List<long> indexes = _dbHandler.GetUnfilteredIndexes();
+
+            if (indexes.Count == 0)
+            {
+                _stop = true;
+                _threadHandler.Stop();
+                break;
+            }
             
             for (int i = 0; i < indexes.Count; i++)
             {
                 if (_stop) break;
                 
+                if (_contentQueue.Count >= 500)
+                {
+                    Thread.Sleep(1000);
+                }
+                
                 Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                 
                 if (unfiltered.Filtered) continue;
@@ -77,13 +106,13 @@ public class ContentFilter
                 {
                     continue;
                 }
+
+                Content content = new();
+                content.Ip = ip;
+                content.Port1 = unfiltered.Port1;
+                content.Port2 = unfiltered.Port2;
                 
-                Filtered filtered = GetSiteData(ip);
-                
-                filtered.Port1 = unfiltered.Port1;
-                filtered.Port2 = unfiltered.Port2;
-                
-                _queue.Enqueue(filtered);
+                _contentQueue.Enqueue(content);
             }
             
             Thread.Sleep(_timeOut);
@@ -91,18 +120,65 @@ public class ContentFilter
         
         ((EventWaitHandle) obj).Set();
     }
-    
-    private Filtered GetSiteData(Ip ip)
+
+    public WaitHandle[] StartFilterThread(int threads)
     {
-        StartProcess(ip, 80);
-        StartProcess(ip, 443);
+        WaitHandle[] waitHandle = new WaitHandle[threads];
+
+        for (int i = 0; i < threads; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            ContentThread contentThread = new();
+            contentThread.ThreadId = i;
+            contentThread.EventWaitHandle = handle;
+            waitHandle[i] = handle;
+            
+            Thread thread = new(FilterThread!);
+            thread.Start(contentThread);
+            
+            Thread.Sleep(8);
+        }
+        
+        return waitHandle;
+    }
+
+    private void FilterThread(object obj)
+    {
+        Console.WriteLine("Filter Thread started.");
+        ContentThread thread = (ContentThread) obj;
+        
+        while (!_stop)
+        {
+            if (_contentQueue.IsEmpty)
+            {
+                Thread.Sleep(1000);
+            }
+            
+            _contentQueue.TryDequeue(out Content? content);
+
+            if (content is null)
+            {
+                continue;
+            }
+            
+            Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
+                
+            filtered.Port1 = content.Port1;
+            filtered.Port2 = content.Port2;
+                
+            _queue.Enqueue(filtered);
+        }
+        
+        thread.EventWaitHandle!.Set();
+    }
+    
+    private Filtered GetSiteData(Ip ip, int threadId)
+    {
+        StartProcess(ip, 80, threadId);
+        StartProcess(ip, 443, threadId);
         
         string url1 = "";
         string url2 = "";
-        string title1 = "";
-        string title2 = "";
-        string description1 = "";
-        string description2 = "";
         bool robotsTxt1 = false;
         bool robotsTxt2 = false;
         string serverType1 = "";
@@ -128,7 +204,7 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
+            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
             
             while (streamReader.Peek() != -1)
             {
@@ -161,37 +237,6 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            if (ports[i] == 80)
-            {
-                if (string.IsNullOrWhiteSpace(url1)) continue;
-                
-                try
-                {
-                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
-                    title1 = temp.Item1;
-                    description1 = temp.Item2;
-                }
-                catch
-                {
-                    //
-                }
-            }
-            else
-            {
-                if (string.IsNullOrWhiteSpace(url2)) continue;
-                
-                try
-                {
-                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
-                    title2 = temp.Item1;
-                    description2 = temp.Item2;
-                }
-                catch
-                {
-                    //
-                }
-            }
-            
             if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
             if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }
@@ -201,10 +246,6 @@ public class ContentFilter
             Ip = ip,
             Url1 = url1,
             Url2 = url2,
-            Title1 = title1,
-            Title2 = title2,
-            Description1 = description1,
-            Description2 = description2,
             ServerType1 = serverType1,
             ServerType2 = serverType2,
             RobotsTXT1 = robotsTxt1,
@@ -230,7 +271,7 @@ public class ContentFilter
         return siteData;
     }
     
-    private void StartProcess(Ip ip, int port)
+    private void StartProcess(Ip ip, int port, int threadId)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
         
@@ -238,7 +279,7 @@ public class ContentFilter
         proc.StartInfo = new()
         {
             FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
+            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,

Backend/Handler/IpFilterHandler.cs

@@ -0,0 +1,227 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using Backend.Helper;
+using Models.Handler;
+using Models.Model.Backend;
+
+namespace Backend.Handler;
+
+public class IpFilterHandler
+{
+    private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private DbHandler _dbHandler;
+    private ThreadHandler _threadHandler;
+    private bool _stop;
+    private bool _fillerStop;
+    private bool _stopAutoscaledThreads;
+    private int _timeout;
+    
+    public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        DbHandler dbHandler, ThreadHandler threadHandler)
+    {
+        _discardedQueue = discardedQueue;
+        _unfilteredQueue = unfilteredQueue;
+        _preFilteredQueue = preFilteredQueue;
+        _dbHandler = dbHandler;
+        _threadHandler = threadHandler;
+
+        _timeout = 16;
+    }
+    
+    public List<WaitHandle[]> Start(int threadCount)
+    {
+        WaitHandle[] waitHandle = new WaitHandle[64];
+        
+        int counter = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
+        
+        for (int i = 0; i < threadCount; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            
+            if (counter < 64)
+            {
+                waitHandle[counter] = handle;
+                counter++;
+                
+                Thread f = new (Filter!);
+                f.Start(handle);
+                
+                Console.WriteLine($"Filter thread ({i}) started");
+                Thread.Sleep(16);
+                
+                continue;
+            }
+            
+            counter = 0;
+            
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
+        }
+        
+        return waitHandles;
+    }
+
+    public void AutoScaler()
+    {
+        int i = 0;
+        int j = 0;
+        
+        while (!_stop)
+        {
+            if (_preFilteredQueue.Count >= 2000)
+            {
+                if (i == 10)
+                {
+                    _stopAutoscaledThreads = false;
+                    Console.WriteLine("Autoscaler started");
+                    
+                    while (!_stopAutoscaledThreads)
+                    {
+                        if (_preFilteredQueue.Count <= 2000)
+                        {
+                            if (j == 1000)
+                            {
+                                _stopAutoscaledThreads = true;
+                            }
+
+                            j++;
+                            
+                            Thread.Sleep(128);
+                        }
+                        else
+                        {
+                            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+                            Thread f = new (Filter_AutoScaler!);
+                            f.Start(handle);
+                            Thread.Sleep(16);
+                        }
+                    }
+                }
+
+                i++;
+            }
+            else
+            {
+                i = 0;
+                j = 0;
+            }
+            
+            Thread.Sleep(128);
+        }
+    }
+
+    private void Filter(object obj)
+    {
+        int counter = 0;
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty && _fillerStop)
+            {
+                if (counter == 100)
+                {
+                    _threadHandler.Stop();
+                    _stop = true;
+                }
+                
+                counter++;
+                Thread.Sleep(128);
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            
+            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+        
+            if (ports is { Item1: 0, Item2: 0 })
+            {
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                continue;
+            }
+        
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+
+    public void FillFilterQueue()
+    {
+        Console.WriteLine("Fill FilterQueue started.");
+        while (!_stop)
+        {
+            if (_preFilteredQueue.Count > 500) continue;
+            
+            if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
+            {
+                _preFilteredQueue.Enqueue(item);
+            }
+            else
+            {
+                _fillerStop = true;
+            }
+        }
+    }
+    
+    private void Filter_AutoScaler(object obj)
+    {
+        while (!_stopAutoscaledThreads)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(_timeout);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            
+            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+            
+            if (ports is { Item1: 0, Item2: 0 })
+            {
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                continue;
+            }
+            
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
+    {
+        return new()
+        {
+            Ip = ip,
+            ResponseCode = responseCode
+        };
+    }
+    
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    {
+        Unfiltered unfiltered = new()
+        {
+            Ip = ip,
+            Port1 = ports.Item1,
+            Port2 = ports.Item2,
+            Filtered = false
+        };
+        
+        return new()
+        {
+            Unfiltered = unfiltered,
+            Operations = Operations.Insert
+        };
+    }
+
+    public void Stop()
+    {
+        _stop = true;
+    }
+}

Backend/Handler/IpScanner.cs

@@ -1,7 +1,11 @@
+using System.Buffers.Binary;
 using System.Collections.Concurrent;
 using System.Net;
 using System.Net.NetworkInformation;
+using System.Numerics;
+using System.Runtime.InteropServices;
 using Backend.Helper;
+using Models.Experimental;
 using Models.Handler;
 using Models.Model.Backend;
 
@@ -18,27 +22,22 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
-    private int _timeout;
+    private readonly int _timeout;
     
-    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
-        )
+    public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue)
     {
         _dbHandler = dbHandler;
+        _preFilteredQueue = preFilteredQueue;
         _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
 
-        SetTimeout(64);
-    }
-    
-    public void SetTimeout(int milliseconds)
-    {
-        _timeout = milliseconds;
+        _timeout = 32;
     }
     
     public List<WaitHandle[]> Start(int threads)
@@ -48,24 +47,12 @@ public class IpScanner
         {
             threadsAmount = 256 / threads;
         }
-        
-        WaitHandle[] waitHandle1;
-        WaitHandle[] waitHandle2;
-        
-        // This is jank. But it was to get it production ready.
-        if (threads <= 64)
-        {
-            waitHandle1 = new WaitHandle[threads];
-            waitHandle2 = new WaitHandle[threads];
-        }
-        else
-        {
-            waitHandle1 = new WaitHandle[64];
-            waitHandle2 = new WaitHandle[64];
-        }
+
+        WaitHandle[] waitHandle = new WaitHandle[64];
         
         int counter = 0;
-        int counter2 = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -79,32 +66,27 @@ public class IpScanner
                 Handle = handle
             };
             
-            if (i < 64)
+            if (counter < 64)
             {
-                waitHandle1[counter] = handle;
+                waitHandle[counter] = handle;
                 counter++;
-            }
-            else
-            {
-                waitHandle2[counter2] = handle;
-                counter2++;
+                
+                Thread f = new (Scan!);
+                f.Start(scanSettings);
+                
+                Console.WriteLine($"Scanner thread ({i}) started");
+                Thread.Sleep(128);
+                
+                continue;
             }
             
-            Thread f = new (Scan!);
-            f.Start(scanSettings);
+            counter = 0;
             
-            Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(100);
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
         }
         
-        List<WaitHandle[]> waitHandles = new();
-        
-        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
-        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
-        
-        waitHandles.Add(waitHandle1);
-        waitHandles.Add(waitHandle2);
-        
         return waitHandles;
     }
     
@@ -125,16 +107,26 @@ public class IpScanner
         
         if (resumeNow is not null)
         {
+            if (resumeNow.Completed)
+            {
+                return;
+            }
+            
             scanSettings.Start = resumeNow.FirstByte;
             scanSettings.End = resumeNow.EndRange;
             secondByte = resumeNow.SecondByte;
             thirdByte = resumeNow.ThirdByte;
             fourthByte = resumeNow.FourthByte;
         }
+        else
+        {
+            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
+        }
         
         // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
+        int x = 0;
         
         for (int i = scanSettings.Start; i < scanSettings.End; i++)
         {
@@ -155,11 +147,24 @@ public class IpScanner
                     
                     if (_discardedQueue.Count >= 2000)
                     {
-                        Thread.Sleep(500);
+                        Thread.Sleep(1000);
+                    }
+                    
+                    if (_preFilteredQueue.Count >= 2000)
+                    {
+                        Thread.Sleep(1000);
                     }
                     
                     for (int l = fourthByte; l < 256; l++)
                     {
+                        if (x == 75_000)
+                        {
+                            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
+                            x = 0;
+                        }
+                        
+                        x++;
+                        
                         if (_stop)
                         {
                             resumeObject.FourthByte = l;
@@ -173,18 +178,27 @@ public class IpScanner
                             Ip3 = k,
                             Ip4 = l
                         };
-
+                        
                         IPStatus responseCode = IPStatus.Unknown;
                         
                         try
                         {
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
-                            _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
-                            if (address is not null)
+                            //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
+
+                            if (i % 2 == 0)
                             {
-                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = IPStatus.Success;
                             }
+                            else
+                            {
+                                responseCode = IPStatus.TimedOut;
+                            }
+                            
+                            //CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
+                            Thread.Sleep(16);
+                            
                         }
                         catch
                         {
@@ -197,15 +211,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
-                        
-                        if (ports is { Item1: 0, Item2: 0 })
-                        {
-                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
-                            continue;
-                        }
-                        
-                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
+                        _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
                     }
                     
                     if (_stop)
@@ -231,10 +237,38 @@ public class IpScanner
             Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
+        if (_stop)
+        {
+            resumeObject.Paused = true;
+        }
+        else
+        {
+            resumeObject.Completed = true;
+        }
+        
+        resumeObject.Operation = Operations.Update;
+        
         _resumeQueue.Enqueue(resumeObject);
         
         scanSettings.Handle!.Set();
     }
+
+    private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
+    {
+        ScannerResumeObject resumeObject = new();
+        resumeObject.ThreadNumber = threadNumber;
+        resumeObject.StartRange = startRange;
+        resumeObject.EndRange = endRange;
+        resumeObject.FirstByte = firstByte;
+        resumeObject.SecondByte = secondByte;
+        resumeObject.ThirdByte = thirdByte;
+        resumeObject.FourthByte = fourthByte;
+        resumeObject.Paused = paused;
+        resumeObject.Completed = completed;
+        resumeObject.Operation = operation;
+        
+        _resumeQueue.Enqueue(resumeObject);
+    }
     
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
@@ -245,25 +279,114 @@ public class IpScanner
         };
     }
     
-    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
     {
-        Unfiltered unfiltered = new()
+        FilterQueueItem filterQueueItem = new()
         {
             Ip = ip,
-            Port1 = ports.Item1,
-            Port2 = ports.Item2,
-            Filtered = false
+            ResponseCode = responseCode
         };
         
-        return new()
-        {
-            Unfiltered = unfiltered,
-            Operations = Operations.Insert
-        };
+        return filterQueueItem;
     }
     
     public void Stop()
     {
         _stop = true;
     }
+    
+    private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
+    {
+        int length = ipSpan.Length;
+        long nonCanonical;
+        fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
+            nonCanonical = ParseNonCanonical(name, 0, ref length, true);
+        
+        return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
+    }
+    
+    private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
+    {
+        long* numPtr = stackalloc long[4];
+        long num1 = 0;
+        bool flag = false;
+        int index1 = 0;
+        int index2;
+        for (index2 = start; index2 < end; ++index2)
+        {
+            char ch = name[index2];
+            num1 = 0L;
+            int num2 = 10;
+            if (ch == '0')
+            {
+                num2 = 8;
+                ++index2;
+                flag = true;
+                if (index2 < end)
+                {
+                    switch (name[index2])
+                    {
+                        case 'X':
+                        case 'x':
+                            num2 = 16;
+                            ++index2;
+                            flag = false;
+                            break;
+                    }
+                }
+            }
+            for (; index2 < end; ++index2)
+            {
+                char c = name[index2];
+                int num3;
+                if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
+                    num3 = (int) c - 48;
+                else if (num2 == 8 && '0' <= c && c <= '7')
+                    num3 = (int) c - 48;
+                else if (num2 == 16 && 'a' <= c && c <= 'f')
+                    num3 = (int) c + 10 - 97;
+                else if (num2 == 16 && 'A' <= c && c <= 'F')
+                    num3 = (int) c + 10 - 65;
+                else
+                    break;
+                num1 = num1 * (long) num2 + (long) num3;
+                if (num1 > (long) uint.MaxValue)
+                    return -1;
+                flag = true;
+            }
+            if (index2 < end && name[index2] == '.')
+            {
+                if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
+                    return -1;
+                numPtr[index1] = num1;
+                ++index1;
+                flag = false;
+            }
+            else
+                break;
+        }
+        if (!flag)
+            return -1;
+        if (index2 < end)
+        {
+            char ch;
+            if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
+                return -1;
+            end = index2;
+        }
+        numPtr[index1] = num1;
+        switch (index1)
+        {
+            case 0:
+                return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
+            case 1:
+                return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
+            case 2:
+                return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
+            case 3:
+                return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
+            default:
+                return -1;
+        }
+    }
 }

Backend/Handler/ThreadHandler.cs

@@ -7,68 +7,115 @@ namespace Backend.Handler;
 public class ThreadHandler
 {
     private readonly DbHandler _dbHandler;
-    private readonly Communication _communication;
     private readonly IpScanner _ipScanner;
     private readonly ContentFilter _contentFilter;
+    private readonly IpFilterHandler _ipFilterHandler;
     
-    private bool _communicationStopped;
     private bool _ipScannerStopped;
     private bool _contentFilterStopped;
+    private bool _ipFilterStopped;
+
+    private bool _stage1;
+    private bool _stage2 = true;
+    private bool _stage3;
+    
+    ConcurrentQueue<Filtered> filteredQueue = new();
+    ConcurrentQueue<Discarded> discardedQueue = new();
+    ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
+    ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
+    ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
     
     public ThreadHandler(string path)
     {
-        ConcurrentQueue<Filtered> filteredQueue = new();
-        ConcurrentQueue<Discarded> discardedQueue = new();
-        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
-        ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
-        
-        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
-        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
-        _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
+        _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
     }
     
     public void Start()
     {
         Thread scanner = new(StartScanner);
+        Thread ipFilter = new(StartIpFilter);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
-        Thread communication = new(StartCommunicationHandler);
-        
-        scanner.Start();
-        indexer.Start();
-        database.Start();
-        discarded.Start();
-        filtered.Start();
-        resume.Start();
-        communication.Start();
-        
-        scanner.Join();
-        indexer.Join();
-        database.Join();
-        discarded.Join();
-        filtered.Join();
-        resume.Join();
-        communication.Join();
+        Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
+        Thread contentFilterThread = new(StartContentFilterThread);
+        Thread prefilterDb = new(StartPreFilterDbHandler);
+        Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
+        //Thread check = new(CheckQueue);
+
+        if (_stage1)
+        {
+            discarded.Start(); // de-queues from discardedQueue
+            prefilterDb.Start(); // de-queues from preFilteredQueue
+            scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
+            resume.Start(); // de-queues from resumeQueue
+            
+            discarded.Join();
+            prefilterDb.Join();
+            scanner.Join();
+            resume.Join();
+        }
+
+        if (_stage2)
+        {
+            database.Start(); // de-queues from unfilteredQueue
+            discarded.Start(); // de-queues from discardedQueue
+            ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
+            ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
+            fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
+            
+            database.Join();
+            discarded.Join();
+            ipFilter.Join();
+            ipFilterAutoScaler.Join();
+            fillIpFilterQueue.Join();
+        }
+
+        if (_stage3)
+        {
+            filtered.Start(); // de-queues from filteredQueue
+            database.Start(); // de-queues from unfilteredQueue
+            indexer.Start(); // en-queues to unfilteredQueue and contentQueue
+            contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
+            
+            contentFilterThread.Join();
+            filtered.Join();
+            database.Join();
+            indexer.Join();
+        }
+    }
+
+    private void CheckQueue()
+    {
+        while (true)
+        {
+            Console.Clear();
+            Console.WriteLine($"filteredQueue.Count:       {filteredQueue.Count}");
+            Console.WriteLine($"discardedQueue.Count:      {discardedQueue.Count}");
+            Console.WriteLine($"unfilteredQueue.Count:     {unfilteredQueue.Count}");
+            Console.WriteLine($"scannerResumeQueue.Count:  {scannerResumeQueue.Count}");
+            Console.WriteLine($"preFilteredQueue.Count:    {preFilteredQueue.Count}");
+            Thread.Sleep(5);
+        }
     }
     
     private void StartScanner()
     {
-        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
-        
-        List<WaitHandle[]> wait = _ipScanner.Start(128);
+        Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
         
+        List<WaitHandle[]> wait = _ipScanner.Start(256);
+
         for (int i = 0; i < wait.Count; i++)
         {
             WaitHandle.WaitAll(wait[i]);
         }
         
         Console.WriteLine("Scanner finished");
-        
-        _ipScannerStopped = true;
     }
     
     private void StartContentFilter()
@@ -83,6 +130,39 @@ public class ThreadHandler
         
         _contentFilterStopped = true;
     }
+
+    private void StartContentFilterThread()
+    {
+        WaitHandle[] wait = _contentFilter.StartFilterThread(64);
+        
+        WaitHandle.WaitAll(wait);
+    }
+
+    private void StartIpFilterAutoScaler()
+    {
+        _ipFilterHandler.AutoScaler();
+    }
+
+    private void StartFillIpFilterQueue()
+    {
+        _ipFilterHandler.FillFilterQueue();
+    }
+
+    private void StartIpFilter()
+    {
+        Thread.Sleep(1000);
+        
+        List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
+
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
+        
+        Console.WriteLine("Ip filter finished");
+        
+        _ipFilterStopped = true;
+    }
     
     private void StartDbHandler()
     {
@@ -93,6 +173,11 @@ public class ThreadHandler
     {
         _dbHandler.FilteredDbHandler();
     }
+
+    private void StartPreFilterDbHandler()
+    {
+        _dbHandler.PrefilteredDbHandler();
+    }
     
     private void StartResumeDbHandler()
     {
@@ -108,35 +193,18 @@ public class ThreadHandler
         Console.WriteLine("Discarded DbHandler finished");
     }
     
-    private void StartCommunicationHandler()
-    {
-        WaitHandle[] wait = _communication.Start();
-        
-        WaitHandle.WaitAll(wait);
-        
-        Console.WriteLine("Communicator finished");
-        
-        _communicationStopped = true;
-        
-        Stop();
-    }
-    
-    private void Stop()
+    public void Stop()
     {
+        Console.WriteLine("Stopping...");
         _ipScanner.Stop();
         _contentFilter.Stop();
+        _ipFilterHandler.Stop();
+        Console.WriteLine("Stopping Extra...");
         
-        bool stopping = true;
+        Thread.Sleep(30_000);
         
-        while (stopping)
-        {
-            if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
-            {
-                _dbHandler.Stop();
-                stopping = false;
-            }
-            
-            Thread.Sleep(3000);
-        }
+        Console.WriteLine("Stopping Super Extra...");
+        _dbHandler.Stop();
+        Console.WriteLine("Stopped.");
     }
 }

Backend/Helper/FilesystemHelper.cs

@@ -1,43 +0,0 @@
-using Models.Model.Backend;
-
-namespace Backend.Helper;
-
-public static class FilesystemHelper
-{
-    public static DatabaseSizes GetDatabaseSizes(string basePath)
-    {
-        DatabaseSizes databaseSizes = new();
-        
-        FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
-        databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
-        
-        databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
-        
-        fileInfo = new($"{basePath}/Models/Filtered.db");
-        databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
-        
-        return databaseSizes;
-    }
-
-    private static long GetDiscardedDbSizes(string basePath)
-    {
-        string folder = $"{basePath}/Models";
-        string[] files = Directory.GetFiles(folder, "*.db");
-        
-        long size = 0;
-        
-        for (int i = 0; i < files.Length; i++)
-        {
-            if (!files[i].Contains("Discarded")) continue;
-            FileInfo fileInfo = new(files[i]);
-            size += fileInfo.Length;
-        }
-        
-        return size;
-    }
-    
-    private static double ToSize(this long value, SizeUnits unit)
-    {
-        return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
-    }
-}

Backend/Helper/FilterHelper.cs

@@ -2,12 +2,10 @@ using System.Text.RegularExpressions;
 
 namespace Backend.Helper;
 
-public static partial class FilterHelper
+public static class FilterHelper
 {
     // https://stackoverflow.com/a/56116499
     private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
-    private const string TitlePattern = "<title>(.*)</title>";
-    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
     
     public static void GetDomain(string data, out string result)
     {
@@ -42,32 +40,6 @@ public static partial class FilterHelper
         result = GetSubstring(data, start, end);
     }
     
-    public static void GetTitle(string data, out string result)
-    {
-        result = "";
-        
-        Regex titleRegex = MyRegex1();
-        Match match = titleRegex.Match(data);
-        
-        if (match.Success)
-        {
-            result = match.Groups[1].Value;
-        }
-    }
-    
-    public static void GetDescription(string data, out string result)
-    {
-        result = "";
-        
-        Regex titleRegex = MyRegex2();
-        Match match = titleRegex.Match(data);
-        
-        if (match.Success)
-        {
-            result = match.Groups[1].Value;
-        }
-    }
-    
     public static void GetHttpVersion(string data, out string result)
     {
         result = "";
@@ -198,7 +170,7 @@ public static partial class FilterHelper
             case 0:
                 return;
             case 2:
-                result = $"{temp[1].Trim()} UUHHHMMM";
+                result = $"{temp[1].Trim()}";
                 return;
             default:
                 result = temp[2].Trim();
@@ -241,17 +213,7 @@ public static partial class FilterHelper
     
     private static bool ValidateUrl(string url)
     {
-        Regex rgx = MyRegex();
+        Regex rgx = new(DomainPattern);
         return rgx.IsMatch(url);
     }
-    
-    //Generate the RegEx at compile time.
-    [GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
-    private static partial Regex MyRegex();
-    
-    [GeneratedRegex(TitlePattern)]
-    private static partial Regex MyRegex1();
-    
-    [GeneratedRegex(DescriptionPattern)]
-    private static partial Regex MyRegex2();
 }

Backend/Helper/HttpClientHelper.cs

@@ -6,79 +6,6 @@ public static partial class HttpClientHelper
 {
     // Reddit, for example, will block the GET request if you don't have a user agent.
     private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
-    private const string TitlePattern = "<title>(.*)</title>";
-    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
-    private const string StartHeadTag = "<head>";
-    private const string EndHeadTag = "</head>";
-    
-    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
-    {
-        using HttpClient client = new();
-        
-        if (port == 80)
-        {
-            client.BaseAddress = new($"http://{url}");
-        }
-        else
-        {
-            client.BaseAddress = new($"https://{url}");
-        }
-        
-        client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
-        client.Timeout = TimeSpan.FromSeconds(30);
-        
-        HttpResponseMessage? response;
-        
-        try
-        {
-            response = await client.GetAsync("/");
-        }
-        catch
-        {
-            return ("", "");
-        }
-        
-        if (!response.IsSuccessStatusCode)
-        {
-            return ("", "");
-        }
-
-        string html = await response.Content.ReadAsStringAsync();
-        
-        int firstIndex = 0;
-        int lastIndex = 0;
-        
-        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
-        {
-            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
-            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
-        }
-        
-        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
-        html = "";
-        
-        string title = "";
-        string description = "";
-        
-        Regex titleRegex = TitleRegEx();
-        Match titleMatch = titleRegex.Match(head);
-        
-        if (titleMatch.Success)
-        {
-            title = titleMatch.Groups[1].Value;
-        }
-        
-        Regex descriptionRegex = DexcriptionRegEx();
-        Match descriptionMatch = descriptionRegex.Match(head);
-        
-        if (descriptionMatch.Success)
-        {
-            description = descriptionMatch.Groups[1].Value;
-        }
-        
-        return (title, description);
-    }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
     {
@@ -86,11 +13,25 @@ public static partial class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
@@ -110,9 +51,4 @@ public static partial class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
-
-    [GeneratedRegex(TitlePattern)]
-    private static partial Regex TitleRegEx();
-    [GeneratedRegex(DescriptionPattern)]
-    private static partial Regex DexcriptionRegEx();
 }

Backend/Helper/SearchHelper.cs

@@ -1,44 +0,0 @@
-using FuzzySharp;
-using Models.Handler;
-using Models.Model.External;
-
-namespace Backend.Helper;
-
-public static class SearchHelper
-{
-    public static SearchResults Search(string searchText, DbHandler dbHandler)
-    {
-        if (string.IsNullOrWhiteSpace(searchText))
-        {
-            SearchResult searchResult = new();
-            searchResult.Description = "asd";
-            searchResult.Title = "asd";
-            searchResult.Url = "asd";
-            SearchResults searchResults = new();
-            searchResults.Results =
-            [
-                searchResult
-            ];
-            return searchResults;
-        }
-        
-        List<SearchResult?> temp = dbHandler.GetSearchResults();
-        SearchResults searchResultsList = new();
-        
-        for (int i = 0; i < temp.Count; i++)
-        {
-            if (temp[i] is null) continue;
-            
-            SearchResult result = new();
-            
-            if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
-            
-            result.Url = temp[i]?.Url;
-            result.Title = temp[i]?.Title;
-            
-            searchResultsList.Results.Add(result);
-        }
-        
-        return searchResultsList;
-    }
-}

Backend/Helper/TcpClientHelper.cs

@@ -1,20 +1,24 @@
 using System.Net;
 using System.Net.Sockets;
+using Models.Model.Backend;
 
 namespace Backend.Helper;
 
 public static class TcpClientHelper
 {
-    public static (int, int) CheckPort(string ip, params int[] ports)
+    public static (int, int) CheckPort(Ip ip, params int[] ports)
     {
         // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached,
         // and it would just return a result.error, for example.
         for (int i = 0; i < ports.Length; i++) {
+            using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            socket.SendTimeout = 250;
+            
             try
             {
-                using TcpClient client = new();
-                client.Connect(ip, ports[i]);
-                // If the connection is successful, update the result array with the port number
+                socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
+                socket.Close();
+                // If the connection is not successful, update the ports array with 0.
             }
             catch
             {

Manager/Commands.cs

@@ -1,122 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Text.Json;
-using Models.Model.Backend;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-
-namespace Manager;
-
-public static class Commands
-{
-    public static void GetProgress()
-    {
-        Console.WriteLine("Getting progress ...");
-        
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.GetScanningProgress;
-        
-        ScanningStatus temp = GetProgress(communicationObject);
-        
-        Console.WriteLine($"Total filtered:           {temp.TotalFiltered:n0}");
-        Console.WriteLine($"Total discarded:          {temp.TotalDiscarded:n0}");
-        Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}");
-        Console.WriteLine($"Total Ips left:           {temp.AmountOfIpv4Left:n0}");
-        Console.WriteLine($"Filtered DB size:         {temp.FilteredDbSize} Kb");
-        Console.WriteLine($"Discarded DB size:        {temp.DiscardedDbSize} Kb");
-        Console.WriteLine($"Mydb DB size:             {temp.MyDbSize} Kb");
-    }
-    
-    public static void StopServer()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.StopScanning;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void Vacuum()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.DbVacuum;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void ReIndex()
-    {
-        CommunicationObject communicationObject = new();
-        communicationObject.Command = CommunicationCommand.DbReindex;
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-
-    public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
-    {
-        CommunicationObject communicationObject = new()
-        {
-            Command = CommunicationCommand.ChangeRuntimeVariable,
-            Variable = runtimeVariable.ToString(),
-            VariableValue = value
-        };
-        
-        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
-    }
-    
-    public static void GetHelp()
-    {
-        Console.WriteLine("Available commands:");
-        Console.WriteLine("  stop  - stops the server");
-        Console.WriteLine("  clear - clears the console");
-        Console.WriteLine("  q     - quits the program");
-        Console.WriteLine("  p     - print the progress information of the scanner");
-        Console.WriteLine("  g     - manual garbage collect on the server");
-        Console.WriteLine("  r     - manual reindex the databases");
-        Console.WriteLine("  v     - manual vacuum the databases");
-        Console.WriteLine("  R     - change runtime variable");
-        Console.WriteLine("  lR    - list runtime variables");
-        Console.WriteLine("  help  - shows this help");
-        Console.WriteLine();
-    }
-    
-    public static void PrintRuntimeVariables()
-    {
-        Console.WriteLine("Runtime variables:");
-        Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()}   - Sets the timeout in milliseconds for the scanner");
-        Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()}    - Sets the timeout in milliseconds for the content filter");
-        Console.WriteLine($"{RuntimeVariable.DbContent.ToString()}        - Sets the wait time in milliseconds for the content database if the queue is empty");
-        Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()}      - Sets the wait time in milliseconds for the discarded database if the queue is empty");
-    }
-
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
-    {
-        //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
-        
-        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-        
-        using RequestSocket client = new();
-        client.Connect("tcp://127.0.0.1:5556");
-        client.SendFrame(lol);
-        byte[] msg = client.ReceiveFrameBytes();
-        client.Close();
-        
-        return JsonSerializer.Deserialize<string>(msg);
-    }
-
-    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
-    private static ScanningStatus GetProgress(CommunicationObject communicationObject)
-    {
-        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-        
-        using RequestSocket client = new();
-        client.Connect("tcp://127.0.0.1:5556");
-        client.SendFrame(lol);
-        byte[] msg = client.ReceiveFrameBytes();
-        client.Close();
-        
-        return JsonSerializer.Deserialize<ScanningStatus>(msg);
-    }
-}

Manager/Manager.csproj

@@ -1,25 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-
-        <!--<Platform>x64</Platform>
-        <Optimize>true</Optimize>
-        <PublishAot>true</PublishAot>
-        <PublishTrimmed>true</PublishTrimmed>
-        <TrimMode>link</TrimMode>
-        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
-    </PropertyGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\Models\Models.csproj" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-
-</Project>

Manager/Program.cs

@@ -1,90 +0,0 @@
-using Manager;
-using Models.Model.Backend;
-
-bool stop = false;
-
-do
-{
-    string? input = Console.ReadLine();
-    
-    if (string.Equals(input, "stop"))
-    {
-        Commands.StopServer();
-    }
-    
-    else if (string.Equals(input, "q"))
-    {
-        stop = true;
-    }
-    
-    else if (string.Equals(input, "clear"))
-    {
-        Console.Clear();
-    }
-    
-    else if (string.Equals(input, "p"))
-    {
-        Commands.GetProgress();
-    }
-    
-    else if (string.Equals(input, "v"))
-    {
-        Commands.Vacuum();
-    }
-
-    else if (string.Equals(input, "r"))
-    {
-        Commands.ReIndex();
-    }
-    
-    else if (string.Equals(input, "R"))
-    {
-        Console.WriteLine("Variable name.");
-        string? variable = Console.ReadLine();
-        
-        Console.WriteLine("Variable value.");
-        string? value = Console.ReadLine();
-        
-        if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
-        {
-            Console.WriteLine("Please enter a value.");
-            return;
-        }
-
-        if (variable == RuntimeVariable.ScannerTimeout.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
-        }
-        
-        else if (variable == RuntimeVariable.ContentFilter.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
-        }
-        
-        else if (variable == RuntimeVariable.DbDiscarded.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
-        }
-        
-        else if (variable == RuntimeVariable.DbContent.ToString())
-        {
-            Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
-        }
-    }
-    
-    else if (string.Equals(input, "lR"))
-    {
-        Commands.PrintRuntimeVariables();
-    }
-    
-    else if (string.Equals(input, "help"))
-    {
-        Commands.GetHelp();
-    }
-    
-    else
-    {
-        Commands.GetHelp();
-    }
-    
-} while (!stop);

Models/Experimental/CustomPing.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using Models.Model.Backend;
+namespace Models.Experimental;
+public class CustomPing
+{
+    public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
+    {
+        SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
+        using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
+        int ipHeaderLength = 20;
+            
+        try
+        {
+            rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
+            byte[] numArray = new byte[136];
+            long timestamp = Stopwatch.GetTimestamp();
+            
+            // TODO: WTF ???
+            EndPoint lol = socketConfig.EndPoint;
+                
+            while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
+            {
+                int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
+                    
+                IPStatus status;
+                if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
+                    return status;
+            }
+        }
+        catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
+        {
+        }
+            
+        return IPStatus.TimedOut;
+    }
+}

Models/Experimental/MessageConstant.cs

@@ -0,0 +1,59 @@
+using System.Net.NetworkInformation;
+namespace Models.Experimental;
+public struct MessageConstant
+{
+  public static IPStatus MapV4TypeToIpStatus(int type, int code)
+  {
+    IPStatus ipStatus1;
+    switch ((IcmpV4MessageType) type)
+    {
+      case IcmpV4MessageType.EchoReply:
+        ipStatus1 = IPStatus.Success;
+        break;
+      case IcmpV4MessageType.DestinationUnreachable:
+        IPStatus ipStatus2;
+        switch ((byte) code)
+        {
+          case 0:
+            ipStatus2 = IPStatus.DestinationNetworkUnreachable;
+            break;
+          case 1:
+            ipStatus2 = IPStatus.DestinationHostUnreachable;
+            break;
+          case 2:
+            ipStatus2 = IPStatus.DestinationProtocolUnreachable;
+            break;
+          case 3:
+            ipStatus2 = IPStatus.DestinationPortUnreachable;
+            break;
+          default:
+            ipStatus2 = IPStatus.DestinationUnreachable;
+            break;
+        }
+        ipStatus1 = ipStatus2;
+        break;
+      case IcmpV4MessageType.SourceQuench:
+        ipStatus1 = IPStatus.SourceQuench;
+        break;
+      case IcmpV4MessageType.TimeExceeded:
+        ipStatus1 = IPStatus.TtlExpired;
+        break;
+      case IcmpV4MessageType.ParameterProblemBadIpHeader:
+        ipStatus1 = IPStatus.BadHeader;
+        break;
+      default:
+        ipStatus1 = IPStatus.Unknown;
+        break;
+    }
+    return ipStatus1;
+  }
+}
+
+internal enum IcmpV4MessageType : byte
+{
+  EchoReply = 0,
+  DestinationUnreachable = 3,
+  SourceQuench = 4,
+  TimeExceeded = 11, // 0x0B
+  ParameterProblemBadIpHeader = 12, // 0x0C
+}

Models/Experimental/RawSocket.cs

@@ -0,0 +1,91 @@
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Models.Model.Backend;
+
+namespace Models.Experimental;
+
+public class RawSocket
+{
+    public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
+    {
+        Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
+        rawSocket.ReceiveTimeout = socketConfig.Timeout;
+        rawSocket.SendTimeout = socketConfig.Timeout;
+        rawSocket.Connect(socketConfig.EndPoint);
+        int num = 1;
+        rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
+        return rawSocket;
+    }
+    
+    public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
+    {
+        byte num = (byte) (receiveBuffer[0] & 15U);
+        ipHeaderLength = 4 * num;
+        
+        int start = ipHeaderLength;
+        int srcOffset = ipHeaderLength + 8;
+        
+        IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
+        byte[] numArray = new byte[bytesReceived - srcOffset];
+        Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
+        reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
+        return true;
+    }
+    
+    public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
+    {
+        int length = sizeof (IcmpHeader);
+        byte[] sendMessageBuffer = new byte[length];
+        new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
+        ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
+        sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
+        sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
+        return sendMessageBuffer;
+    }
+    
+    private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
+    {
+        uint num1 = 0;
+        for (int index = 0; index < buffer.Length; index += 2)
+        {
+            ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
+            num1 += num2;
+        }
+        while (num1 >> 16 != 0U)
+            num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
+        return (ushort) ~num1;
+    }
+}
+
+public class SocketConfig
+{
+    public EndPoint EndPoint;
+    public readonly int Timeout;
+    public readonly CustomProtocolType ProtocolType;
+    public readonly byte[] SendBuffer;
+
+    public SocketConfig(
+        EndPoint endPoint,
+        int timeout,
+        CustomProtocolType protocolType,
+        byte[] sendBuffer)
+    {
+        EndPoint = endPoint;
+        Timeout = timeout;
+        ProtocolType = protocolType;
+        SendBuffer = sendBuffer;
+    }
+}
+
+
+public struct IcmpHeader
+{
+    public byte Type;
+    public byte Code;
+    public ushort HeaderChecksum;
+    public ushort Identifier;
+    public ushort SequenceNumber;
+}

Models/Handler/DbHandler.cs

@@ -13,8 +13,10 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
+    private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
     private readonly string _compressedConnectionString;
@@ -25,15 +27,20 @@ public class DbHandler
                                            " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
+    private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                           " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
+                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
+    
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
-                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
-                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
+                                              " Url1, Url2, ServerType1, ServerType2," +
                                               " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
                                               " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
                                               " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
                                               " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
-                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
+                                              " @url1, @url2, " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
                                               " @robotsTXT1, @robotsTXT2," +
@@ -73,8 +80,8 @@ public class DbHandler
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
-                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
 
     private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                                    " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -86,11 +93,16 @@ public class DbHandler
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
-    private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
     private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
+    private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
+    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
+    private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
+    private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
+    
     private const string ReIndexDatabasesStatement = "REINDEX;";
     
     private const string VacuumDatabasesStatement = "VACUUM;";
@@ -111,12 +123,15 @@ public class DbHandler
     public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, 
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        string basePath)
     {
         _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
+        _preFilteredQueue = preFilteredQueue;
 
         SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
@@ -127,6 +142,7 @@ public class DbHandler
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
         _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
+        _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -188,6 +204,24 @@ public class DbHandler
         
         Console.WriteLine("Filtered DbHandler stopped.");
     }
+
+    public void PrefilteredDbHandler()
+    {
+        Console.WriteLine("PreFiltered Db handler started.");
+        
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(4);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
+            
+            InsertPrefiltered(queueItem);
+        }
+    }
     
     public void ResumeDbHandler()
     {
@@ -202,12 +236,9 @@ public class DbHandler
                 continue;
             }
             
-            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
+            _resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
             
-            if (queueItem is not null)
-            {
-                InsertResumeObject(queueItem);
-            }
+            InsertResumeObject(queueItem);
         }
         
         Console.WriteLine("Resume DbHandler stopped.");
@@ -244,16 +275,6 @@ public class DbHandler
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
         (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
-
-        // Compress the databases if they're already filled.
-        if (GetDiscardedIndexesForSpecificDb(connectionString) >= 50_000_000 && !_compressing)
-        {
-            _compressing = true;
-            
-            Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
-            
-            _compressing = false;
-        }
         
         int i = 0;
         
@@ -266,13 +287,19 @@ public class DbHandler
                 continue;
             }
             
-            if (i >= 50_000_000 && !_compressing)
+            if (i >= 5_000_000 && !_compressing)
             {
                 _compressing = true;
                 
                 i = 0;
                 
-                Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
                 
                 _compressing = false;
             }
@@ -288,18 +315,6 @@ public class DbHandler
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
-
-    private void Compress(int threadNumber, string absolutePath, string connectionString)
-    {
-        InsertCompressedDatabase(threadNumber, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-        int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-        CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-        DropAndCreateDiscarded(threadNumber);
-    }
-    
     
     private void DropAndCreateDiscarded(int threadNumber)
     {
@@ -443,10 +458,6 @@ public class DbHandler
         command.Parameters.AddWithValue("@port2", filtered.Port2);
         command.Parameters.AddWithValue("@url1", filtered.Url1);
         command.Parameters.AddWithValue("@url2", filtered.Url2);
-        command.Parameters.AddWithValue("@title1", filtered.Title1);
-        command.Parameters.AddWithValue("@title2", filtered.Title2);
-        command.Parameters.AddWithValue("@description1", filtered.Description1);
-        command.Parameters.AddWithValue("@description2", filtered.Description2);
         command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
         command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
         command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
@@ -477,19 +488,31 @@ public class DbHandler
     {
         using SqliteConnection connection = new(_resumeConnectionString);
         connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoResume, connection);
+        SqliteCommand command;
+
+        if (resumeObject.Operation == Operations.Insert)
+        {
+            command = new(InsertIntoResume, connection);
+            command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
+            command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
+        }
+        else
+        {
+            command = new(UpdateResumeStatement, connection);
+        }
         
         command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
-        command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
-        command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
         command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
         command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
         command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
         command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
+        command.Parameters.AddWithValue("@paused", resumeObject.Paused);
+        command.Parameters.AddWithValue("@completed", resumeObject.Completed);
         
         _ = command.ExecuteNonQuery();
         connection.Close();
+        
+        command.Dispose();
     }
 
     private void InsertCompressedDatabase(int threadNumber, long rows)
@@ -505,6 +528,24 @@ public class DbHandler
         _ = command.ExecuteNonQuery();
         connection.Close();
     }
+
+    private void InsertPrefiltered(FilterQueueItem filterQueueItem)
+    {
+        using SqliteConnection connection = new(_preFilteredConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertPreFilteredStatement, connection);
+        
+        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
+        command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
+        command.Parameters.AddWithValue("@filtered", 0);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+    }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
@@ -573,79 +614,42 @@ public class DbHandler
         return ids;
     }
     
-    public long GetFilteredIndexes()
+    public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
     {
-        long rowId = 0;
-        
-        using SqliteConnection connection = new(_filteredConnectionString);
+        using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
         
-        using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
+        SqliteCommand command = new(ReadPreFilteredStatement, connection);
         using SqliteDataReader reader = command.ExecuteReader();
         
+        filterQueueItem = new();
+        Ip ip = new();
+        long id = 0;
+        
         if (!reader.HasRows)
         {
-            return 0;
+            return false;
         }
         
         while (reader.Read())
         {
-            rowId = reader.GetInt64(0);
+            ip.Ip1 = reader.GetInt32(0);
+            ip.Ip2 = reader.GetInt32(1);
+            ip.Ip3 = reader.GetInt32(2);
+            ip.Ip4 = reader.GetInt32(3);
+            filterQueueItem.ResponseCode = reader.GetInt32(4);
+            id = reader.GetInt64(5);
         }
         
-        return rowId;
-    }
-    
-    public long GetDiscardedIndexes()
-    {
-        long rowId = 0;
+        filterQueueItem.Ip = ip;
         
-        SqliteConnection connection;
-        SqliteCommand command;
-        SqliteDataReader reader;
-        
-        for (int i = 0; i < _discardedConnectionStrings.Count; i++)
-        {
-            connection = new(_discardedConnectionStrings[i]);
-            connection.Open();
-            
-            command = new(ReadDiscardedSeqIdsStatement, connection);
-            reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return rowId;
-            }
-            
-            while (reader.Read())
-            {
-                rowId += reader.GetInt64(0);
-            }
-            
-            connection.Close();
-        }
-        
-        connection = new(_compressedConnectionString);
-        connection.Open();
-        command = new(ReadCompressedDbRowsStatement, connection);
-        reader = command.ExecuteReader();
-        
-        if (!reader.HasRows)
-        {
-            return rowId;
-        }
-        
-        while (reader.Read())
-        {
-            rowId += reader.GetInt64(0);
-        }
-        
-        connection.Close();
-        connection.Dispose();
+        command = new(UpdatePreFilteredStatement, connection);
+        command.Parameters.AddWithValue("@id", id);
+
+        command.ExecuteNonQuery();
         command.Dispose();
-        reader.Dispose();
         
-        return rowId;
+        return true;
     }
 
     private static long GetDiscardedIndexesForSpecificDb(string connectionString)
@@ -759,7 +763,7 @@ public class DbHandler
             using SqliteConnection connection = new(_resumeConnectionString);
             connection.Open();
             
-            using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
+            using SqliteCommand command = new(ReadResumeStatement, connection);
             command.Parameters.AddWithValue("@threadNumber", threadNumber);
             
             using SqliteDataReader reader = command.ExecuteReader();
@@ -780,6 +784,8 @@ public class DbHandler
                 resumeObject.SecondByte = reader.GetInt32(4);
                 resumeObject.ThirdByte = reader.GetInt32(5);
                 resumeObject.FourthByte = reader.GetInt32(6);
+                resumeObject.Paused = reader.GetBoolean(7);
+                resumeObject.Completed = reader.GetBoolean(8);
             }
             
             return resumeObject;

Models/Helper/CompressionHelper.cs

@@ -8,7 +8,7 @@ public static class CompressionHelper
     {
         using FileStream originalFileStream = new(sourceFile, FileMode.Open);
         using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
-        using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize);
         originalFileStream.CopyTo(compressor);
     }
     

Models/Model/Backend/CustomProtocolType.cs

@@ -0,0 +1,6 @@
+namespace Models.Model.Backend;
+
+public enum CustomProtocolType
+{
+    Icmp = 1
+}

Models/Model/Backend/DatabaseSizes.cs

@@ -1,10 +0,0 @@
-namespace Models.Model.Backend;
-
-public struct DatabaseSizes
-{
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/Backend/FilterQueueItem.cs

@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public struct FilterQueueItem
+{
+    public Ip Ip { get; set; }
+    public int ResponseCode { get; set; }
+}

Models/Model/Backend/Filtered.cs

@@ -3,10 +3,6 @@ namespace Models.Model.Backend;
 public class Filtered
 {
     public Ip Ip { get; set; }
-    public string Title1 { get; set; } = "";
-    public string Title2 { get; set; } = "";
-    public string Description1 { get; set; } = "";
-    public string Description2 { get; set; } = "";
     public string Url1 { get; set; } = "";
     public string Url2 { get; set; } = "";
     public int Port1 { get; set; }

Models/Model/Backend/RuntimeVariable.cs

@@ -1,9 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum RuntimeVariable
-{
-    DbContent,
-    DbDiscarded,
-    ContentFilter,
-    ScannerTimeout
-}

Models/Model/Backend/ScannerResumeObject.cs

@@ -9,4 +9,7 @@ public class ScannerResumeObject
     public int ThirdByte { get; set; }
     public int FourthByte { get; set; }
     public int ThreadNumber { get; set; }
+    public Operations Operation { get; set; }
+    public bool Paused { get; set; }
+    public bool Completed { get; set; }
 }

Models/Model/Backend/SizeUnits.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.Backend;
-public enum SizeUnits
-{
-    Byte,
-    KB,
-    MB,
-    GB,
-}

Models/Model/External/CommunicationCommand.cs

@@ -1,11 +0,0 @@
-namespace Models.Model.External;
-
-public enum CommunicationCommand
-{
-    GetScanningProgress,
-    GetSearches,
-    StopScanning,
-    DbReindex,
-    DbVacuum,
-    ChangeRuntimeVariable,
-}

Models/Model/External/CommunicationObject.cs

@@ -1,17 +0,0 @@
-namespace Models.Model.External;
-
-//[MessagePackObject]
-public class CommunicationObject
-{
-    //[Key(0)]
-    public CommunicationCommand Command { get; set; }
-    
-    //[Key(1)]
-    public string? SearchTerm { get; set; } = "";
-    
-    //[Key(2)]
-    public string? Variable { get; set; } = "";
-    
-    //[Key(3)]
-    public string? VariableValue { get; set; } = "";
-}

Models/Model/External/CommunicationResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class CommunicationResult
-{
-    public List<SearchResult?>? Result { get; set; }
-    
-    public ScanningStatus? Status { get; set; }
-}

Models/Model/External/ScanningStatus.cs

@@ -1,18 +0,0 @@
-namespace Models.Model.External;
-
-public struct ScanningStatus
-{
-    public float PercentageOfIpv4Scanned { get; set; }
-    
-    public long TotalFiltered { get; set; }
-    
-    public long AmountOfIpv4Left { get; set; }
-    
-    public long TotalDiscarded { get; set; }
-    
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/External/SearchResults.cs

@@ -1,6 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResults
-{
-    public List<SearchResult?>? Results { get; set; }
-}

Models/Models.csproj

@@ -1,12 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
+      <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
     </ItemGroup>
 </Project>

Proxy/Program.cs

@@ -1,89 +0,0 @@
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using AspNetCoreRateLimit;
-using Microsoft.AspNetCore.HttpOverrides;
-using Microsoft.Extensions.Caching.Memory;
-using Models.Model.External;
-using NetMQ;
-using NetMQ.Sockets;
-const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
-
-WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
-
-builder.Services.ConfigureHttpJsonOptions(options =>
-{
-    options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
-});
-
-builder.Services.AddCors(options =>
-{
-    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
-});
-
-builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
-
-WebApplication app = builder.Build();
-
-app.UseForwardedHeaders(new()
-{
-    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
-});
-
-app.UseCors(myAllowSpecificOrigins);
-
-RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", (IMemoryCache memoryCache) =>
-{
-    const string cacheKey = "progress_status";
-    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
-    {
-        return scanningStatus;
-    }
-    
-    CommunicationObject communicationObject = new()
-    {
-        Command = CommunicationCommand.GetScanningProgress
-    };
-    
-    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-    
-    using RequestSocket client = new();
-    client.Connect("tcp://127.0.0.1:5556");
-    client.SendFrame(bytes);
-    byte[] msg = client.ReceiveFrameBytes();
-    client.Close();
-    
-    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
-    
-    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
-    
-    return scanningStatus;
-});
-
-/*
-RouteGroupBuilder searchApi = app.MapGroup("/search");
-searchApi.MapGet("/{term}", (string term) =>
-{
-    CommunicationObject communicationObject = new();
-    communicationObject.Command = CommunicationCommand.GetSearches;
-    communicationObject.SearchTerm = term;
-    
-    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
-    
-    using RequestSocket client = new();
-    client.Connect("tcp://127.0.0.1:5556");
-    client.SendFrame(bytes);
-    string msg = client.ReceiveFrameString();
-    client.Close();
-    
-    return JsonSerializer.Deserialize<SearchResults?>(msg);
-});
-*/
-app.Run();
-
-[JsonSerializable(typeof(ScanningStatus))]
-//[JsonSerializable(typeof(SearchResults))]
-[JsonSerializable(typeof(CommunicationObject))]
-internal partial class AppJsonSerializerContext : JsonSerializerContext
-{
-}

Proxy/Properties/launchSettings.json

@@ -1,15 +0,0 @@
-{
-  "$schema": "http://json.schemastore.org/launchsettings.json",
-  "profiles": {
-    "http": {
-      "commandName": "Project",
-      "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "",
-      "applicationUrl": "http://localhost:5224",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    }
-  }
-}

Proxy/Proxy.csproj

@@ -1,20 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <Nullable>enable</Nullable>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <InvariantGlobalization>true</InvariantGlobalization>
-        <PublishAot>false</PublishAot>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
-        <PackageReference Include="NetMQ" Version="4.0.1.13" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\Models\Models.csproj" />
-    </ItemGroup>
-    
-</Project>

Proxy/Proxy.http

@@ -1,11 +0,0 @@
-@Proxy_HostAddress = http://localhost:5224
-
-GET {{Proxy_HostAddress}}/progress/
-Accept: application/json
-
-###
-
-GET {{Proxy_HostAddress}}/search/asd
-Accept: application/json
-
-###

Proxy/appsettings.Development.json

@@ -1,8 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  }
-}

Proxy/appsettings.json

@@ -1,9 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Warning",
-      "Microsoft.AspNetCore": "Warning"
-    }
-  },
-  "AllowedHosts": "*"
-}

README.md

@@ -2,4 +2,8 @@
 
 Rasmus Search Engine
 
-This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
+This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
+
+This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture.
+
+I'm just trying to minimize memory usage while maximizing performance.

RSE.sln

@@ -4,9 +4,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -22,13 +20,9 @@ Global
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -2,14 +2,37 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueueSegment_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4f72364deba8e6142d1f42386f1fd282d2e8c6bca41d389c28da4c3835d90_003FConcurrentQueueSegment_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F85_003Fa8fb60ce_003FConcurrentQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADnsEndPoint_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F3d_003F64a36162_003FDnsEndPoint_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIcmpV6MessageType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F4e_003Fec7b627f_003FIcmpV6MessageType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4682fe5857f946a98dcb2cd6f0a403983f200_003F3d_003Fca9070b2_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARawSocketPermissions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003Fc7_003Fb8dcfca9_003FRawSocketPermissions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

frontend/app.vue

@@ -9,8 +9,12 @@ useHead({
 </script>
 
 <template>
-  <NuxtLayout>
-    <NuxtPage page-key="static" />
-  </NuxtLayout>
+  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+    <ul>
+      <li><nuxt-link to="/">Search</nuxt-link></li>
+      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
+    </ul>
+  </div>
 
+  <NuxtPage page-key="static" />
 </template>

frontend/nuxt.config.ts

@@ -13,5 +13,9 @@ export default defineNuxtConfig({
     },
   },
 
+  nitro: {
+    static: true,
+  },
+
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
 });

frontend/pages/index.vue

@@ -1,12 +1,15 @@
 <script setup lang="ts">
-const route = useRoute()
+const value = ref<string | null>(null)
+
+// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
 </script>
 
+
 <template>
   <div class="h-screen flex items-center justify-center">
     <div class="flex space-x-2">
-      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
-      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
+      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
     </div>
   </div>
 </template>

frontend/pages/progress.vue

@@ -39,23 +39,51 @@ fetchMyData();
 
 <template>
   <div class="h-screen grid place-items-center">
-    <div v-if="dict">
-      <table class="table-auto border-gray-300">
-        <thead>
-          <tr>
-            <th class="px-4 py-2">Metric</th>
-            <th class="px-4 py-2">Value</th>
-          </tr>
-        </thead>
+    <table class="table-auto border-gray-300">
+      <thead>
+        <tr>
+          <th class="px-4 py-2">Metric</th>
+          <th class="px-4 py-2">Value</th>
+        </tr>
+      </thead>
 
-        <tbody>
-          <tr v-for="d in dict">
-            <td class="px-4 py-2">{{d.description}}</td>
-            <td class="px-4 py-2">{{d.value}}</td>
-          </tr>
-        </tbody>
-      </table>
-    </div>
+      <tbody v-if="dict">
+        <tr v-for="d in dict">
+          <td class="px-4 py-2">{{d.description}}</td>
+          <td class="px-4 py-2">{{d.value}}</td>
+        </tr>
+      </tbody>
+      <tbody v-else>
+      <tr>
+        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total filtered</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total Discarded</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Amount of Ipv4 left</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Discarded db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Filtered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Unfiltered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      </tbody>
+    </table>
   </div>
 </template>
 

frontend/pages/searchResult.vue

@@ -0,0 +1,8 @@
+<script setup lang="ts">
+const route = useRoute();
+  console.log(route.query);
+</script>
+
+<template>
+  
+</template>

frontend/types/global.d.ts

@@ -6,12 +6,22 @@ interface ProgressType {
     discardedDbSize: bigint;
     filteredDbSize: bigint;
     myDbSize: bigint;
-}
+};
 
 interface ProgressDictionary {
     description: string;
     value: string;
-}
+};
+
+interface SearchResult {
+    title: string;
+    description: string;
+    url: string;
+};
+
+interface SearchResults {
+    results: SearchResult[];
+};
 
 type CacheEntry<T> = {
     data: T;

global.json

@@ -0,0 +1,7 @@
+{
+  "sdk": {
+    "version": "9.0.0",
+    "rollForward": "latestMajor",
+    "allowPrerelease": true
+  }
+}