Analyze/Analyze.csproj

@@ -2,7 +2,7 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>

Backend/Backend.csproj

@@ -2,15 +2,22 @@
     
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <PublishAot>false</PublishAot>
+
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <!--<Platform>x64</Platform>-->
+        <Optimize>false</Optimize>
     </PropertyGroup>
     
     <ItemGroup>
       <ProjectReference Include="..\Models\Models.csproj" />
     </ItemGroup>
     
+    <ItemGroup>
+      <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
+      <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+    
 </Project>

Backend/Handler/Communication.cs

@@ -0,0 +1,201 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Backend.Helper;
+using Models.Handler;
+using Models.Model.Backend;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+
+namespace Backend.Handler;
+
+public class Communication
+{
+    private readonly DbHandler _dbHandler;
+    private readonly ThreadHandler _threadHandler;
+    private readonly IpScanner _ipScanner;
+    private readonly ContentFilter _contentFilter;
+    private bool _isRunning = true;
+    private string _basePath;
+    
+    public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
+    {
+        _dbHandler = dbHandler;
+        _threadHandler = threadHandler;
+        _ipScanner = ipScanner;
+        _contentFilter = contentFilter;
+        _basePath = basePath;
+    }
+    
+    public WaitHandle[] Start()
+    {
+        WaitHandle[] waitHandles = new WaitHandle[1];
+        EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+        waitHandles[0] = handle;
+        
+        Thread thread = new(Server!);
+        thread.Start(handle);
+        
+        return waitHandles;
+    }
+    
+    private void Server(object obj)
+    {
+        using ResponseSocket rep = new();
+        
+        //rep.Options.IPv4Only = true;
+        
+        rep.Bind("tcp://127.0.0.1:5556");
+        
+        while (_isRunning)
+        {
+            byte[] message = rep.ReceiveFrameBytes();
+            
+            CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
+
+            //rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
+            
+            if (communicationObject is null)
+            {
+                continue;
+            }
+            
+            OnServerOnReceiveReady(communicationObject, rep);
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
+    private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
+    {
+        switch (communicationObject.Command)
+        {
+            case CommunicationCommand.GetScanningProgress:
+            {
+                DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
+                
+                long discardedIndexes = _dbHandler.GetDiscardedIndexes();
+                
+                ScanningStatus status = new();
+                // 4294967296 is all Ipv4 addresses.
+                
+                if (discardedIndexes != 0)
+                {
+                    status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
+                }
+                else
+                {
+                    // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
+                    status.PercentageOfIpv4Scanned = 0.0000000001f;
+                }
+                
+                status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
+                status.TotalFiltered = _dbHandler.GetFilteredIndexes();
+                status.TotalDiscarded = discardedIndexes;
+                status.MyDbSize = databaseSizes.MyDbSize;
+                status.FilteredDbSize = databaseSizes.FilteredDbSize;
+                status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
+                
+                byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
+                
+                rep.SendFrame(serializedResult);
+                
+                break;
+            }
+            
+            case CommunicationCommand.DbReindex:
+            {
+                _dbHandler.ReIndex();
+                
+                SendStringResponse(rep, "All Dbs have been reindexed.");
+                break;
+            }
+            
+            case CommunicationCommand.DbVacuum:
+            {
+                _dbHandler.Vacuum();
+                
+                SendStringResponse(rep, "All Dbs have been vacuumed.");
+                break;
+            }
+            
+            case CommunicationCommand.GetSearches:
+            {
+                if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
+                {
+                    SendSearchResponse(rep, communicationObject.SearchTerm);
+                }
+                
+                break;
+            }
+            
+            case CommunicationCommand.StopScanning:
+            {
+                _isRunning = false;
+                break;
+            }
+            
+            case CommunicationCommand.ChangeRuntimeVariable:
+            {
+                if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
+                
+                if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _dbHandler.SetContentWaitTime(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _dbHandler.SetDiscardedWaitTime(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _ipScanner.SetTimeout(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _contentFilter.SetTimeout(value);
+                }
+                
+                rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
+                
+                break;
+            }
+        }
+    }
+    
+    private static void SendStringResponse(ResponseSocket rep, string response)
+    {
+        byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
+        
+        rep.SendFrame(serializedResult);
+    }
+
+    private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
+    {
+        //SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
+        SearchResults result = new()
+        {
+            Results = []
+        };
+        SearchResult lol = new()
+        {
+            Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
+            Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
+        };
+        
+        result.Results.Add(lol);
+        
+        string serializedResult = JsonSerializer.Serialize(result);
+        
+        rep.SendFrame(serializedResult);
+    }
+}

Backend/Handler/ContentFilter.cs

@@ -16,7 +16,7 @@ public class Content
 public class ContentThread
 {
     public int ThreadId { get; set; }
-    public EventWaitHandle? EventWaitHandle { get; set; }
+    public EventWaitHandle EventWaitHandle { get; set; }
 }
 
 public class ContentFilter
@@ -25,19 +25,17 @@ public class ContentFilter
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Content?> _contentQueue = new();
     private readonly DbHandler _dbHandler;
-    private readonly ThreadHandler _threadHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
     private bool _stop;
     private int _timeOut;
     private readonly string _basePath;
     
-    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
-        _threadHandler = threadHandler;
         _unfilteredQueue = unfilteredQueue;
         
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
@@ -70,13 +68,6 @@ public class ContentFilter
         {
             List<long> indexes = _dbHandler.GetUnfilteredIndexes();
             
-            if (indexes.Count == 0)
-            {
-                _stop = true;
-                _threadHandler.Stop();
-                break;
-            }
-            
             for (int i = 0; i < indexes.Count; i++)
             {
                 if (_stop) break;
@@ -135,8 +126,6 @@ public class ContentFilter
             
             Thread thread = new(FilterThread!);
             thread.Start(contentThread);
-            
-            Thread.Sleep(8);
         }
         
         return waitHandle;
@@ -144,7 +133,6 @@ public class ContentFilter
 
     private void FilterThread(object obj)
     {
-        Console.WriteLine("Filter Thread started.");
         ContentThread thread = (ContentThread) obj;
         
         while (!_stop)
@@ -169,7 +157,7 @@ public class ContentFilter
             _queue.Enqueue(filtered);
         }
         
-        thread.EventWaitHandle!.Set();
+        thread.EventWaitHandle.Set();
     }
     
     private Filtered GetSiteData(Ip ip, int threadId)

Backend/Handler/IpFilterHandler.cs

@@ -1,7 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using Backend.Helper;
-using Models.Handler;
 using Models.Model.Backend;
 
 namespace Backend.Handler;
@@ -11,23 +10,17 @@ public class IpFilterHandler
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
-    private DbHandler _dbHandler;
-    private ThreadHandler _threadHandler;
     private bool _stop;
-    private bool _fillerStop;
     private bool _stopAutoscaledThreads;
     private int _timeout;
     
     public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        ConcurrentQueue<FilterQueueItem> filteredQueue)
-        DbHandler dbHandler, ThreadHandler threadHandler)
     {
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
-        _preFilteredQueue = preFilteredQueue;
+        _preFilteredQueue = filteredQueue;
-        _dbHandler = dbHandler;
-        _threadHandler = threadHandler;
 
         _timeout = 16;
     }
@@ -53,7 +46,7 @@ public class IpFilterHandler
                 f.Start(handle);
                 
                 Console.WriteLine($"Filter thread ({i}) started");
-                Thread.Sleep(16);
+                Thread.Sleep(128);
                 
                 continue;
             }
@@ -119,19 +112,12 @@ public class IpFilterHandler
 
     private void Filter(object obj)
     {
-        int counter = 0;
         while (!_stop)
         {
-            if (_preFilteredQueue.IsEmpty && _fillerStop)
+            if (_preFilteredQueue.IsEmpty)
             {
-                if (counter == 100)
+                Thread.Sleep(_timeout);
-                {
+                continue;
-                    _threadHandler.Stop();
-                    _stop = true;
-                }
-                
-                counter++;
-                Thread.Sleep(128);
             }
             
             _preFilteredQueue.TryDequeue(out FilterQueueItem item);
@@ -150,24 +136,6 @@ public class IpFilterHandler
         ((EventWaitHandle) obj).Set();
     }
     
-    public void FillFilterQueue()
-    {
-        Console.WriteLine("Fill FilterQueue started.");
-        while (!_stop)
-        {
-            if (_preFilteredQueue.Count > 500) continue;
-            
-            if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
-            {
-                _preFilteredQueue.Enqueue(item);
-            }
-            else
-            {
-                _fillerStop = true;
-            }
-        }
-    }
-    
     private void Filter_AutoScaler(object obj)
     {
         while (!_stopAutoscaledThreads)

Backend/Handler/IpScanner.cs

@@ -1,11 +1,7 @@
-using System.Buffers.Binary;
 using System.Collections.Concurrent;
 using System.Net;
 using System.Net.NetworkInformation;
-using System.Numerics;
-using System.Runtime.InteropServices;
 using Backend.Helper;
-using Models.Experimental;
 using Models.Handler;
 using Models.Model.Backend;
 
@@ -26,7 +22,7 @@ public class IpScanner
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
-    private readonly int _timeout;
+    private int _timeout;
     
     public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
@@ -37,7 +33,12 @@ public class IpScanner
         _discardedQueue = discardedQueue;
         _resumeQueue = resumeQueue;
         
-        _timeout = 32;
+        SetTimeout(16);
+    }
+    
+    public void SetTimeout(int milliseconds)
+    {
+        _timeout = milliseconds;
     }
     
     public List<WaitHandle[]> Start(int threads)
@@ -185,20 +186,12 @@ public class IpScanner
                         {
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
-                            //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
+                            _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
-
+                            if (address is not null)
-                            if (i % 2 == 0)
                             {
-                                responseCode = IPStatus.Success;
+                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                //Thread.Sleep(4);
                             }
-                            else
-                            {
-                                responseCode = IPStatus.TimedOut;
-                            }
-                            
-                            //CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
-                            Thread.Sleep(16);
-                            
                         }
                         catch
                         {
@@ -294,99 +287,4 @@ public class IpScanner
     {
         _stop = true;
     }
-    
-    private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
-    {
-        int length = ipSpan.Length;
-        long nonCanonical;
-        fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
-            nonCanonical = ParseNonCanonical(name, 0, ref length, true);
-        
-        return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
-    }
-    
-    private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
-    {
-        long* numPtr = stackalloc long[4];
-        long num1 = 0;
-        bool flag = false;
-        int index1 = 0;
-        int index2;
-        for (index2 = start; index2 < end; ++index2)
-        {
-            char ch = name[index2];
-            num1 = 0L;
-            int num2 = 10;
-            if (ch == '0')
-            {
-                num2 = 8;
-                ++index2;
-                flag = true;
-                if (index2 < end)
-                {
-                    switch (name[index2])
-                    {
-                        case 'X':
-                        case 'x':
-                            num2 = 16;
-                            ++index2;
-                            flag = false;
-                            break;
-                    }
-                }
-            }
-            for (; index2 < end; ++index2)
-            {
-                char c = name[index2];
-                int num3;
-                if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
-                    num3 = (int) c - 48;
-                else if (num2 == 8 && '0' <= c && c <= '7')
-                    num3 = (int) c - 48;
-                else if (num2 == 16 && 'a' <= c && c <= 'f')
-                    num3 = (int) c + 10 - 97;
-                else if (num2 == 16 && 'A' <= c && c <= 'F')
-                    num3 = (int) c + 10 - 65;
-                else
-                    break;
-                num1 = num1 * (long) num2 + (long) num3;
-                if (num1 > (long) uint.MaxValue)
-                    return -1;
-                flag = true;
-            }
-            if (index2 < end && name[index2] == '.')
-            {
-                if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
-                    return -1;
-                numPtr[index1] = num1;
-                ++index1;
-                flag = false;
-            }
-            else
-                break;
-        }
-        if (!flag)
-            return -1;
-        if (index2 < end)
-        {
-            char ch;
-            if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
-                return -1;
-            end = index2;
-        }
-        numPtr[index1] = num1;
-        switch (index1)
-        {
-            case 0:
-                return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
-            case 1:
-                return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
-            case 2:
-                return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
-            case 3:
-                return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
-            default:
-                return -1;
-        }
-    }
 }

Backend/Handler/ThreadHandler.cs

@@ -7,30 +7,29 @@ namespace Backend.Handler;
 public class ThreadHandler
 {
     private readonly DbHandler _dbHandler;
+    private readonly Communication _communication;
     private readonly IpScanner _ipScanner;
     private readonly ContentFilter _contentFilter;
     private readonly IpFilterHandler _ipFilterHandler;
     
+    private bool _communicationStopped;
     private bool _ipScannerStopped;
     private bool _contentFilterStopped;
     private bool _ipFilterStopped;
     
-    private bool _stage1;
+    public ThreadHandler(string path)
-    private bool _stage2 = true;
+    {
-    private bool _stage3;
-    
         ConcurrentQueue<Filtered> filteredQueue = new();
         ConcurrentQueue<Discarded> discardedQueue = new();
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
         ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
         ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
         
-    public ThreadHandler(string path)
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-    {
-        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
         _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
-        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
-        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
+        _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue);
     }
     
     public void Start()
@@ -42,66 +41,31 @@ public class ThreadHandler
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
+        Thread communication = new(StartCommunicationHandler);
         Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
         Thread contentFilterThread = new(StartContentFilterThread);
-        Thread prefilterDb = new(StartPreFilterDbHandler);
-        Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
-        //Thread check = new(CheckQueue);
         
-        if (_stage1)
+        ipFilter.Start();
-        {
+        scanner.Start();
-            discarded.Start(); // de-queues from discardedQueue
+        ipFilterAutoScaler.Start();
-            prefilterDb.Start(); // de-queues from preFilteredQueue
+        indexer.Start();
-            scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
+        database.Start();
-            resume.Start(); // de-queues from resumeQueue
+        discarded.Start();
+        filtered.Start();
+        resume.Start();
+        communication.Start();
+        contentFilterThread.Start();
         
-            discarded.Join();
-            prefilterDb.Join();
         scanner.Join();
-            resume.Join();
+        ipFilter.Join();
-        }
+        indexer.Join();
-
-        if (_stage2)
-        {
-            database.Start(); // de-queues from unfilteredQueue
-            discarded.Start(); // de-queues from discardedQueue
-            ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
-            ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
-            fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
-            
         database.Join();
         discarded.Join();
-            ipFilter.Join();
-            ipFilterAutoScaler.Join();
-            fillIpFilterQueue.Join();
-        }
-
-        if (_stage3)
-        {
-            filtered.Start(); // de-queues from filteredQueue
-            database.Start(); // de-queues from unfilteredQueue
-            indexer.Start(); // en-queues to unfilteredQueue and contentQueue
-            contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
-            
-            contentFilterThread.Join();
         filtered.Join();
-            database.Join();
+        resume.Join();
-            indexer.Join();
+        communication.Join();
-        }
+        ipFilterAutoScaler.Join();
-    }
+        contentFilterThread.Join();
-
-    private void CheckQueue()
-    {
-        while (true)
-        {
-            Console.Clear();
-            Console.WriteLine($"filteredQueue.Count:       {filteredQueue.Count}");
-            Console.WriteLine($"discardedQueue.Count:      {discardedQueue.Count}");
-            Console.WriteLine($"unfilteredQueue.Count:     {unfilteredQueue.Count}");
-            Console.WriteLine($"scannerResumeQueue.Count:  {scannerResumeQueue.Count}");
-            Console.WriteLine($"preFilteredQueue.Count:    {preFilteredQueue.Count}");
-            Thread.Sleep(5);
-        }
     }
     
     private void StartScanner()
@@ -116,6 +80,8 @@ public class ThreadHandler
         }
         
         Console.WriteLine("Scanner finished");
+        
+        _ipScannerStopped = true;
     }
     
     private void StartContentFilter()
@@ -133,7 +99,7 @@ public class ThreadHandler
 
     private void StartContentFilterThread()
     {
-        WaitHandle[] wait = _contentFilter.StartFilterThread(64);
+        WaitHandle[] wait = _contentFilter.StartFilterThread(4);
         
         WaitHandle.WaitAll(wait);
     }
@@ -143,11 +109,6 @@ public class ThreadHandler
         _ipFilterHandler.AutoScaler();
     }
 
-    private void StartFillIpFilterQueue()
-    {
-        _ipFilterHandler.FillFilterQueue();
-    }
-
     private void StartIpFilter()
     {
         Thread.Sleep(1000);
@@ -174,11 +135,6 @@ public class ThreadHandler
         _dbHandler.FilteredDbHandler();
     }
     
-    private void StartPreFilterDbHandler()
-    {
-        _dbHandler.PrefilteredDbHandler();
-    }
-    
     private void StartResumeDbHandler()
     {
         _dbHandler.ResumeDbHandler();
@@ -193,18 +149,35 @@ public class ThreadHandler
         Console.WriteLine("Discarded DbHandler finished");
     }
     
-    public void Stop()
+    private void StartCommunicationHandler()
+    {
+        WaitHandle[] wait = _communication.Start();
+        
+        WaitHandle.WaitAll(wait);
+        
+        Console.WriteLine("Communicator finished");
+        
+        _communicationStopped = true;
+        
+        Stop();
+    }
+    
+    private void Stop()
     {
-        Console.WriteLine("Stopping...");
         _ipScanner.Stop();
         _contentFilter.Stop();
-        _ipFilterHandler.Stop();
-        Console.WriteLine("Stopping Extra...");
         
-        Thread.Sleep(30_000);
+        bool stopping = true;
         
-        Console.WriteLine("Stopping Super Extra...");
+        while (stopping)
+        {
+            if (_ipScannerStopped && _contentFilterStopped && _ipFilterStopped)
+            {
                 _dbHandler.Stop();
-        Console.WriteLine("Stopped.");
+                stopping = false;
+            }
+            
+            Thread.Sleep(3000);
+        }
     }
 }

Backend/Helper/FilesystemHelper.cs

@@ -0,0 +1,43 @@
+using Models.Model.Backend;
+
+namespace Backend.Helper;
+
+public static class FilesystemHelper
+{
+    public static DatabaseSizes GetDatabaseSizes(string basePath)
+    {
+        DatabaseSizes databaseSizes = new();
+        
+        FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
+        databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
+        
+        databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
+        
+        fileInfo = new($"{basePath}/Models/Filtered.db");
+        databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
+        
+        return databaseSizes;
+    }
+
+    private static long GetDiscardedDbSizes(string basePath)
+    {
+        string folder = $"{basePath}/Models";
+        string[] files = Directory.GetFiles(folder, "*.db");
+        
+        long size = 0;
+        
+        for (int i = 0; i < files.Length; i++)
+        {
+            if (!files[i].Contains("Discarded")) continue;
+            FileInfo fileInfo = new(files[i]);
+            size += fileInfo.Length;
+        }
+        
+        return size;
+    }
+    
+    private static double ToSize(this long value, SizeUnits unit)
+    {
+        return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
+    }
+}

Backend/Helper/FilterHelper.cs

@@ -2,10 +2,12 @@ using System.Text.RegularExpressions;
 
 namespace Backend.Helper;
 
-public static class FilterHelper
+public static partial class FilterHelper
 {
     // https://stackoverflow.com/a/56116499
     private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
     
     public static void GetDomain(string data, out string result)
     {
@@ -40,6 +42,32 @@ public static class FilterHelper
         result = GetSubstring(data, start, end);
     }
     
+    public static void GetTitle(string data, out string result)
+    {
+        result = "";
+        
+        Regex titleRegex = MyRegex1();
+        Match match = titleRegex.Match(data);
+        
+        if (match.Success)
+        {
+            result = match.Groups[1].Value;
+        }
+    }
+    
+    public static void GetDescription(string data, out string result)
+    {
+        result = "";
+        
+        Regex titleRegex = MyRegex2();
+        Match match = titleRegex.Match(data);
+        
+        if (match.Success)
+        {
+            result = match.Groups[1].Value;
+        }
+    }
+    
     public static void GetHttpVersion(string data, out string result)
     {
         result = "";
@@ -170,7 +198,7 @@ public static class FilterHelper
             case 0:
                 return;
             case 2:
-                result = $"{temp[1].Trim()}";
+                result = $"{temp[1].Trim()} UUHHHMMM";
                 return;
             default:
                 result = temp[2].Trim();
@@ -213,7 +241,17 @@ public static class FilterHelper
     
     private static bool ValidateUrl(string url)
     {
-        Regex rgx = new(DomainPattern);
+        Regex rgx = MyRegex();
         return rgx.IsMatch(url);
     }
+    
+    //Generate the RegEx at compile time.
+    [GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
+    private static partial Regex MyRegex();
+    
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex MyRegex1();
+    
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex MyRegex2();
 }

Backend/Helper/HttpClientHelper.cs

@@ -1,4 +1,5 @@
 using System.Text.RegularExpressions;
+using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
@@ -6,6 +7,93 @@ public static partial class HttpClientHelper
 {
     // Reddit, for example, will block the GET request if you don't have a user agent.
     private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
+    {
+        using HttpClient client = new();
+        
+        if (port == 80)
+        {
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
+        }
+        else
+        {
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
+        }
+        
+        client.DefaultRequestHeaders.Accept.Clear();
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
+        
+        HttpResponseMessage? response;
+        
+        try
+        {
+            response = await client.GetAsync("/");
+        }
+        catch
+        {
+            return ("", "");
+        }
+        
+        if (!response.IsSuccessStatusCode)
+        {
+            return ("", "");
+        }
+
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
+        }
+        
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
+    }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
     {
@@ -51,4 +139,9 @@ public static partial class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }

Backend/Helper/SearchHelper.cs

@@ -0,0 +1,44 @@
+using FuzzySharp;
+using Models.Handler;
+using Models.Model.External;
+
+namespace Backend.Helper;
+
+public static class SearchHelper
+{
+    public static SearchResults Search(string searchText, DbHandler dbHandler)
+    {
+        if (string.IsNullOrWhiteSpace(searchText))
+        {
+            SearchResult searchResult = new();
+            searchResult.Description = "asd";
+            searchResult.Title = "asd";
+            searchResult.Url = "asd";
+            SearchResults searchResults = new();
+            searchResults.Results =
+            [
+                searchResult
+            ];
+            return searchResults;
+        }
+        
+        List<SearchResult?> temp = dbHandler.GetSearchResults();
+        SearchResults searchResultsList = new();
+        
+        for (int i = 0; i < temp.Count; i++)
+        {
+            if (temp[i] is null) continue;
+            
+            SearchResult result = new();
+            
+            if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
+            
+            result.Url = temp[i]?.Url;
+            result.Title = temp[i]?.Title;
+            
+            searchResultsList.Results.Add(result);
+        }
+        
+        return searchResultsList;
+    }
+}

Backend/Helper/TcpClientHelper.cs

@@ -1,4 +1,3 @@
-using System.Net;
 using System.Net.Sockets;
 using Models.Model.Backend;
 
@@ -16,7 +15,7 @@ public static class TcpClientHelper
             
             try
             {
-                socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
+                socket.Connect(ip.ToString(), ports[i]);
                 socket.Close();
                 // If the connection is not successful, update the ports array with 0.
             }

Backend/Scripts/443Header.txt

@@ -0,0 +1,14 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 1.0.0.95:443...
+* Connected to 1.0.0.95 (1.0.0.95) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS alert, handshake failure (552):
+{ [2 bytes data]
+* OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* closing connection #0
+curl: (35) OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure

Backend/Scripts/443Header0.txt

@@ -0,0 +1,76 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.72.3:443...
+* Connected to 192.0.72.3 (192.0.72.3) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2063 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [79 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=files.wordpress.com
+*  start date: Dec 16 09:49:37 2024 GMT
+*  expire date: Mar 16 09:49:36 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E6
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.72.3/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.72.3]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.72.3
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 302 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:34 GMT
+< content-type: text/html; charset=utf-8
+< location: https://developer.wordpress.com
+< vary: Cookie
+< x-nc: MISS hhn 3
+< x-content-type-options: nosniff
+< alt-svc: h3=":443"; ma=86400
+< 
+{ [0 bytes data]
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.72.3 left intact
+HTTP/2 302 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:34 GMT
+content-type: text/html; charset=utf-8
+location: https://developer.wordpress.com
+vary: Cookie
+x-nc: MISS hhn 3
+x-content-type-options: nosniff
+alt-svc: h3=":443"; ma=86400
+

Backend/Scripts/443Header1.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.251:443...
+* Connected to 192.0.66.251 (192.0.66.251) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [79 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.251/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.251]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.251
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:27 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+
  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.251 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:27 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/443Header2.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.254:443...
+* Connected to 192.0.66.254 (192.0.66.254) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [78 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.254/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.254]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.254
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:28 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+
  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.254 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:28 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/443Header3.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.253:443...
+* Connected to 192.0.66.253 (192.0.66.253) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [78 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.253/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.253]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.253
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:28 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+
  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.253 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:28 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/80Header.txt

@@ -0,0 +1,36 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 1.0.0.95:80...
+* Connected to 1.0.0.95 (1.0.0.95) port 80
+> HEAD / HTTP/1.1
+> Host: 1.0.0.95
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+< HTTP/1.1 403 Forbidden
+< Date: Fri, 07 Feb 2025 21:37:37 GMT
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 16
+< Connection: close
+< X-Frame-Options: SAMEORIGIN
+< Referrer-Policy: same-origin
+< Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+< Expires: Thu, 01 Jan 1970 00:00:01 GMT
+< Server: cloudflare
+< CF-RAY: 90e685af4a6b930d-CPH
+< 
+
  0    16    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* shutting down connection #0
+HTTP/1.1 403 Forbidden
+Date: Fri, 07 Feb 2025 21:37:37 GMT
+Content-Type: text/plain; charset=UTF-8
+Content-Length: 16
+Connection: close
+X-Frame-Options: SAMEORIGIN
+Referrer-Policy: same-origin
+Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+Expires: Thu, 01 Jan 1970 00:00:01 GMT
+Server: cloudflare
+CF-RAY: 90e685af4a6b930d-CPH
+

Backend/Scripts/80Header0.txt

@@ -0,0 +1,28 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.72.3:80...
+* Connected to 192.0.72.3 (192.0.72.3) port 80
+> HEAD / HTTP/1.1
+> Host: 192.0.72.3
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+< HTTP/1.1 301 Moved Permanently
+< Server: nginx
+< Date: Fri, 07 Feb 2025 22:58:33 GMT
+< Content-Type: text/html
+< Content-Length: 162
+< Connection: keep-alive
+< Location: https://192.0.72.3/
+< 
+
  0   162    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.72.3 left intact
+HTTP/1.1 301 Moved Permanently
+Server: nginx
+Date: Fri, 07 Feb 2025 22:58:33 GMT
+Content-Type: text/html
+Content-Length: 162
+Connection: keep-alive
+Location: https://192.0.72.3/
+

Backend/Scripts/80Header1.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.64.106:80...
+
  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Backend/Scripts/80Header2.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.68.86:80...
+
  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Backend/Scripts/80Header3.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.68.78:80...
+
  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0
  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Manager/Commands.cs

@@ -0,0 +1,122 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Models.Model.Backend;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+
+namespace Manager;
+
+public static class Commands
+{
+    public static void GetProgress()
+    {
+        Console.WriteLine("Getting progress ...");
+        
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.GetScanningProgress;
+        
+        ScanningStatus temp = GetProgress(communicationObject);
+        
+        Console.WriteLine($"Total filtered:           {temp.TotalFiltered:n0}");
+        Console.WriteLine($"Total discarded:          {temp.TotalDiscarded:n0}");
+        Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}");
+        Console.WriteLine($"Total Ips left:           {temp.AmountOfIpv4Left:n0}");
+        Console.WriteLine($"Filtered DB size:         {temp.FilteredDbSize} Kb");
+        Console.WriteLine($"Discarded DB size:        {temp.DiscardedDbSize} Kb");
+        Console.WriteLine($"Mydb DB size:             {temp.MyDbSize} Kb");
+    }
+    
+    public static void StopServer()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.StopScanning;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void Vacuum()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.DbVacuum;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void ReIndex()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.DbReindex;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+
+    public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
+    {
+        CommunicationObject communicationObject = new()
+        {
+            Command = CommunicationCommand.ChangeRuntimeVariable,
+            Variable = runtimeVariable.ToString(),
+            VariableValue = value
+        };
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void GetHelp()
+    {
+        Console.WriteLine("Available commands:");
+        Console.WriteLine("  stop  - stops the server");
+        Console.WriteLine("  clear - clears the console");
+        Console.WriteLine("  q     - quits the program");
+        Console.WriteLine("  p     - print the progress information of the scanner");
+        Console.WriteLine("  g     - manual garbage collect on the server");
+        Console.WriteLine("  r     - manual reindex the databases");
+        Console.WriteLine("  v     - manual vacuum the databases");
+        Console.WriteLine("  R     - change runtime variable");
+        Console.WriteLine("  lR    - list runtime variables");
+        Console.WriteLine("  help  - shows this help");
+        Console.WriteLine();
+    }
+    
+    public static void PrintRuntimeVariables()
+    {
+        Console.WriteLine("Runtime variables:");
+        Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()}   - Sets the timeout in milliseconds for the scanner");
+        Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()}    - Sets the timeout in milliseconds for the content filter");
+        Console.WriteLine($"{RuntimeVariable.DbContent.ToString()}        - Sets the wait time in milliseconds for the content database if the queue is empty");
+        Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()}      - Sets the wait time in milliseconds for the discarded database if the queue is empty");
+    }
+
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
+    {
+        //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
+        
+        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+        
+        using RequestSocket client = new();
+        client.Connect("tcp://127.0.0.1:5556");
+        client.SendFrame(lol);
+        byte[] msg = client.ReceiveFrameBytes();
+        client.Close();
+        
+        return JsonSerializer.Deserialize<string>(msg);
+    }
+
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    private static ScanningStatus GetProgress(CommunicationObject communicationObject)
+    {
+        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+        
+        using RequestSocket client = new();
+        client.Connect("tcp://127.0.0.1:5556");
+        client.SendFrame(lol);
+        byte[] msg = client.ReceiveFrameBytes();
+        client.Close();
+        
+        return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    }
+}

Manager/Manager.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <!--<Platform>x64</Platform>
+        <Optimize>true</Optimize>
+        <PublishAot>true</PublishAot>
+        <PublishTrimmed>true</PublishTrimmed>
+        <TrimMode>link</TrimMode>
+        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Models\Models.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+
+</Project>

Manager/Program.cs

@@ -0,0 +1,90 @@
+using Manager;
+using Models.Model.Backend;
+
+bool stop = false;
+
+do
+{
+    string? input = Console.ReadLine();
+    
+    if (string.Equals(input, "stop"))
+    {
+        Commands.StopServer();
+    }
+    
+    else if (string.Equals(input, "q"))
+    {
+        stop = true;
+    }
+    
+    else if (string.Equals(input, "clear"))
+    {
+        Console.Clear();
+    }
+    
+    else if (string.Equals(input, "p"))
+    {
+        Commands.GetProgress();
+    }
+    
+    else if (string.Equals(input, "v"))
+    {
+        Commands.Vacuum();
+    }
+
+    else if (string.Equals(input, "r"))
+    {
+        Commands.ReIndex();
+    }
+    
+    else if (string.Equals(input, "R"))
+    {
+        Console.WriteLine("Variable name.");
+        string? variable = Console.ReadLine();
+        
+        Console.WriteLine("Variable value.");
+        string? value = Console.ReadLine();
+        
+        if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
+        {
+            Console.WriteLine("Please enter a value.");
+            return;
+        }
+
+        if (variable == RuntimeVariable.ScannerTimeout.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
+        }
+        
+        else if (variable == RuntimeVariable.ContentFilter.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
+        }
+        
+        else if (variable == RuntimeVariable.DbDiscarded.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
+        }
+        
+        else if (variable == RuntimeVariable.DbContent.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
+        }
+    }
+    
+    else if (string.Equals(input, "lR"))
+    {
+        Commands.PrintRuntimeVariables();
+    }
+    
+    else if (string.Equals(input, "help"))
+    {
+        Commands.GetHelp();
+    }
+    
+    else
+    {
+        Commands.GetHelp();
+    }
+    
+} while (!stop);

Models/Experimental/CustomPing.cs

@@ -1,39 +0,0 @@
-using System.Diagnostics;
-using System.Net;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
-using Models.Model.Backend;
-namespace Models.Experimental;
-public class CustomPing
-{
-    public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
-    {
-        SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
-        using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
-        int ipHeaderLength = 20;
-            
-        try
-        {
-            rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
-            byte[] numArray = new byte[136];
-            long timestamp = Stopwatch.GetTimestamp();
-            
-            // TODO: WTF ???
-            EndPoint lol = socketConfig.EndPoint;
-                
-            while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
-            {
-                int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
-                    
-                IPStatus status;
-                if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
-                    return status;
-            }
-        }
-        catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
-        {
-        }
-            
-        return IPStatus.TimedOut;
-    }
-}

Models/Experimental/MessageConstant.cs

@@ -1,59 +0,0 @@
-using System.Net.NetworkInformation;
-namespace Models.Experimental;
-public struct MessageConstant
-{
-  public static IPStatus MapV4TypeToIpStatus(int type, int code)
-  {
-    IPStatus ipStatus1;
-    switch ((IcmpV4MessageType) type)
-    {
-      case IcmpV4MessageType.EchoReply:
-        ipStatus1 = IPStatus.Success;
-        break;
-      case IcmpV4MessageType.DestinationUnreachable:
-        IPStatus ipStatus2;
-        switch ((byte) code)
-        {
-          case 0:
-            ipStatus2 = IPStatus.DestinationNetworkUnreachable;
-            break;
-          case 1:
-            ipStatus2 = IPStatus.DestinationHostUnreachable;
-            break;
-          case 2:
-            ipStatus2 = IPStatus.DestinationProtocolUnreachable;
-            break;
-          case 3:
-            ipStatus2 = IPStatus.DestinationPortUnreachable;
-            break;
-          default:
-            ipStatus2 = IPStatus.DestinationUnreachable;
-            break;
-        }
-        ipStatus1 = ipStatus2;
-        break;
-      case IcmpV4MessageType.SourceQuench:
-        ipStatus1 = IPStatus.SourceQuench;
-        break;
-      case IcmpV4MessageType.TimeExceeded:
-        ipStatus1 = IPStatus.TtlExpired;
-        break;
-      case IcmpV4MessageType.ParameterProblemBadIpHeader:
-        ipStatus1 = IPStatus.BadHeader;
-        break;
-      default:
-        ipStatus1 = IPStatus.Unknown;
-        break;
-    }
-    return ipStatus1;
-  }
-}
-
-internal enum IcmpV4MessageType : byte
-{
-  EchoReply = 0,
-  DestinationUnreachable = 3,
-  SourceQuench = 4,
-  TimeExceeded = 11, // 0x0B
-  ParameterProblemBadIpHeader = 12, // 0x0C
-}

Models/Experimental/RawSocket.cs

@@ -1,91 +0,0 @@
-using System.Net;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using Models.Model.Backend;
-
-namespace Models.Experimental;
-
-public class RawSocket
-{
-    public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
-    {
-        Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
-        rawSocket.ReceiveTimeout = socketConfig.Timeout;
-        rawSocket.SendTimeout = socketConfig.Timeout;
-        rawSocket.Connect(socketConfig.EndPoint);
-        int num = 1;
-        rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
-        return rawSocket;
-    }
-    
-    public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
-    {
-        byte num = (byte) (receiveBuffer[0] & 15U);
-        ipHeaderLength = 4 * num;
-        
-        int start = ipHeaderLength;
-        int srcOffset = ipHeaderLength + 8;
-        
-        IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
-        byte[] numArray = new byte[bytesReceived - srcOffset];
-        Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
-        reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
-        return true;
-    }
-    
-    public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
-    {
-        int length = sizeof (IcmpHeader);
-        byte[] sendMessageBuffer = new byte[length];
-        new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
-        ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
-        sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
-        sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
-        return sendMessageBuffer;
-    }
-    
-    private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
-    {
-        uint num1 = 0;
-        for (int index = 0; index < buffer.Length; index += 2)
-        {
-            ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
-            num1 += num2;
-        }
-        while (num1 >> 16 != 0U)
-            num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
-        return (ushort) ~num1;
-    }
-}
-
-public class SocketConfig
-{
-    public EndPoint EndPoint;
-    public readonly int Timeout;
-    public readonly CustomProtocolType ProtocolType;
-    public readonly byte[] SendBuffer;
-
-    public SocketConfig(
-        EndPoint endPoint,
-        int timeout,
-        CustomProtocolType protocolType,
-        byte[] sendBuffer)
-    {
-        EndPoint = endPoint;
-        Timeout = timeout;
-        ProtocolType = protocolType;
-        SendBuffer = sendBuffer;
-    }
-}
-
-
-public struct IcmpHeader
-{
-    public byte Type;
-    public byte Code;
-    public ushort HeaderChecksum;
-    public ushort Identifier;
-    public ushort SequenceNumber;
-}

Models/Handler/DbHandler.cs

@@ -13,10 +13,8 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
     private readonly string _compressedConnectionString;
@@ -27,11 +25,6 @@ public class DbHandler
                                            " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
-    private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                           " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
-                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
-    
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
                                               " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
@@ -95,9 +88,6 @@ public class DbHandler
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
     private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
-    private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
-    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
-    private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
@@ -123,15 +113,12 @@ public class DbHandler
     public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, 
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
-        string basePath)
     {
         _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
-        _preFilteredQueue = preFilteredQueue;
 
         SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
@@ -142,7 +129,6 @@ public class DbHandler
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
         _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
-        _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -205,24 +191,6 @@ public class DbHandler
         Console.WriteLine("Filtered DbHandler stopped.");
     }
     
-    public void PrefilteredDbHandler()
-    {
-        Console.WriteLine("PreFiltered Db handler started.");
-        
-        while (!_stop)
-        {
-            if (_preFilteredQueue.IsEmpty)
-            {
-                Thread.Sleep(4);
-                continue;
-            }
-            
-            _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
-            
-            InsertPrefiltered(queueItem);
-        }
-    }
-    
     public void ResumeDbHandler()
     {
         Console.WriteLine("Resume DB handler started");
@@ -236,10 +204,13 @@ public class DbHandler
                 continue;
             }
             
-            _resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
+            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
             
+            if (queueItem is not null)
+            {
                 InsertResumeObject(queueItem);
             }
+        }
         
         Console.WriteLine("Resume DbHandler stopped.");
     }
@@ -529,24 +500,6 @@ public class DbHandler
         connection.Close();
     }
     
-    private void InsertPrefiltered(FilterQueueItem filterQueueItem)
-    {
-        using SqliteConnection connection = new(_preFilteredConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertPreFilteredStatement, connection);
-        
-        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
-        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
-        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
-        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
-        command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
-        command.Parameters.AddWithValue("@filtered", 0);
-        
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-    }
-    
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
@@ -614,42 +567,79 @@ public class DbHandler
         return ids;
     }
     
-    public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
+    public long GetFilteredIndexes()
     {
-        using SqliteConnection connection = new(_preFilteredConnectionString);
+        long rowId = 0;
+        
+        using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
-        SqliteCommand command = new(ReadPreFilteredStatement, connection);
+        using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
         using SqliteDataReader reader = command.ExecuteReader();
         
-        filterQueueItem = new();
-        Ip ip = new();
-        long id = 0;
-        
         if (!reader.HasRows)
         {
-            return false;
+            return 0;
         }
         
         while (reader.Read())
         {
-            ip.Ip1 = reader.GetInt32(0);
+            rowId = reader.GetInt64(0);
-            ip.Ip2 = reader.GetInt32(1);
-            ip.Ip3 = reader.GetInt32(2);
-            ip.Ip4 = reader.GetInt32(3);
-            filterQueueItem.ResponseCode = reader.GetInt32(4);
-            id = reader.GetInt64(5);
         }
         
-        filterQueueItem.Ip = ip;
+        return rowId;
+    }
     
-        command = new(UpdatePreFilteredStatement, connection);
+    public long GetDiscardedIndexes()
-        command.Parameters.AddWithValue("@id", id);
+    {
+        long rowId = 0;
 
-        command.ExecuteNonQuery();
+        SqliteConnection connection;
+        SqliteCommand command;
+        SqliteDataReader reader;
+        
+        for (int i = 0; i < _discardedConnectionStrings.Count; i++)
+        {
+            connection = new(_discardedConnectionStrings[i]);
+            connection.Open();
+            
+            command = new(ReadDiscardedSeqIdsStatement, connection);
+            reader = command.ExecuteReader();
+            
+            if (!reader.HasRows)
+            {
+                return rowId;
+            }
+            
+            while (reader.Read())
+            {
+                rowId += reader.GetInt64(0);
+            }
+            
+            connection.Close();
+        }
+        
+        connection = new(_compressedConnectionString);
+        connection.Open();
+        command = new(ReadCompressedDbRowsStatement, connection);
+        reader = command.ExecuteReader();
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+        
+        connection.Close();
+        connection.Dispose();
         command.Dispose();
+        reader.Dispose();
         
-        return true;
+        return rowId;
     }
 
     private static long GetDiscardedIndexesForSpecificDb(string connectionString)

Models/Model/Backend/CustomProtocolType.cs

@@ -1,6 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum CustomProtocolType
-{
-    Icmp = 1
-}

Models/Model/Backend/DatabaseSizes.cs

@@ -0,0 +1,10 @@
+namespace Models.Model.Backend;
+
+public struct DatabaseSizes
+{
+    public double DiscardedDbSize { get; set; }
+    
+    public double FilteredDbSize { get; set; }
+    
+    public double MyDbSize { get; set; }
+}

Models/Model/Backend/FilterQueueItem.cs

@@ -2,6 +2,6 @@ namespace Models.Model.Backend;
 
 public struct FilterQueueItem
 {
-    public Ip Ip { get; set; }
+    public Ip Ip { get; init; }
-    public int ResponseCode { get; set; }
+    public int ResponseCode { get; init; }
 }

Models/Model/Backend/RuntimeVariable.cs

@@ -0,0 +1,9 @@
+namespace Models.Model.Backend;
+
+public enum RuntimeVariable
+{
+    DbContent,
+    DbDiscarded,
+    ContentFilter,
+    ScannerTimeout
+}

Models/Model/Backend/SizeUnits.cs

@@ -0,0 +1,8 @@
+namespace Models.Model.Backend;
+public enum SizeUnits
+{
+    Byte,
+    KB,
+    MB,
+    GB,
+}

Models/Model/External/CommunicationCommand.cs

@@ -0,0 +1,11 @@
+namespace Models.Model.External;
+
+public enum CommunicationCommand
+{
+    GetScanningProgress,
+    GetSearches,
+    StopScanning,
+    DbReindex,
+    DbVacuum,
+    ChangeRuntimeVariable,
+}

Models/Model/External/CommunicationObject.cs

@@ -0,0 +1,17 @@
+namespace Models.Model.External;
+
+//[MessagePackObject]
+public class CommunicationObject
+{
+    //[Key(0)]
+    public CommunicationCommand Command { get; set; }
+    
+    //[Key(1)]
+    public string? SearchTerm { get; set; } = "";
+    
+    //[Key(2)]
+    public string? Variable { get; set; } = "";
+    
+    //[Key(3)]
+    public string? VariableValue { get; set; } = "";
+}

Models/Model/External/CommunicationResult.cs

@@ -0,0 +1,8 @@
+namespace Models.Model.External;
+
+public class CommunicationResult
+{
+    public List<SearchResult?>? Result { get; set; }
+    
+    public ScanningStatus? Status { get; set; }
+}

Models/Model/External/ScanningStatus.cs

@@ -0,0 +1,18 @@
+namespace Models.Model.External;
+
+public struct ScanningStatus
+{
+    public float PercentageOfIpv4Scanned { get; set; }
+    
+    public long TotalFiltered { get; set; }
+    
+    public long AmountOfIpv4Left { get; set; }
+    
+    public long TotalDiscarded { get; set; }
+    
+    public double DiscardedDbSize { get; set; }
+    
+    public double FilteredDbSize { get; set; }
+    
+    public double MyDbSize { get; set; }
+}

Models/Model/External/SearchResults.cs

@@ -0,0 +1,6 @@
+namespace Models.Model.External;
+
+public class SearchResults
+{
+    public List<SearchResult?>? Results { get; set; }
+}

Models/Models.csproj

@@ -1,13 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
+      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
     </ItemGroup>
 </Project>

Proxy/Program.cs

@@ -0,0 +1,89 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using AspNetCoreRateLimit;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
+
+WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
+
+builder.Services.ConfigureHttpJsonOptions(options =>
+{
+    options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
+});
+
+builder.Services.AddCors(options =>
+{
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
+});
+
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
+WebApplication app = builder.Build();
+
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);
+
+RouteGroupBuilder progressApi = app.MapGroup("/progress");
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
+{
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
+    CommunicationObject communicationObject = new()
+    {
+        Command = CommunicationCommand.GetScanningProgress
+    };
+    
+    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+    
+    using RequestSocket client = new();
+    client.Connect("tcp://127.0.0.1:5556");
+    client.SendFrame(bytes);
+    byte[] msg = client.ReceiveFrameBytes();
+    client.Close();
+    
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
+});
+
+
+RouteGroupBuilder searchApi = app.MapGroup("/search");
+searchApi.MapGet("/{term}", (string term) =>
+{
+    CommunicationObject communicationObject = new();
+    communicationObject.Command = CommunicationCommand.GetSearches;
+    communicationObject.SearchTerm = term;
+    
+    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+    
+    using RequestSocket client = new();
+    client.Connect("tcp://127.0.0.1:5556");
+    client.SendFrame(bytes);
+    string msg = client.ReceiveFrameString();
+    client.Close();
+    
+    return JsonSerializer.Deserialize<SearchResults?>(msg);
+});
+
+app.Run();
+
+[JsonSerializable(typeof(ScanningStatus))]
+//[JsonSerializable(typeof(SearchResults))]
+[JsonSerializable(typeof(CommunicationObject))]
+internal partial class AppJsonSerializerContext : JsonSerializerContext
+{
+}

Proxy/Properties/launchSettings.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "",
+      "applicationUrl": "http://localhost:5224",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

Proxy/Proxy.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
+        <PublishAot>false</PublishAot>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
+        <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Models\Models.csproj" />
+    </ItemGroup>
+    
+</Project>

Proxy/Proxy.http

@@ -0,0 +1,11 @@
+@Proxy_HostAddress = http://localhost:5224
+
+GET {{Proxy_HostAddress}}/progress/
+Accept: application/json
+
+###
+
+GET {{Proxy_HostAddress}}/search/asd
+Accept: application/json
+
+###

Proxy/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

Proxy/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

README.md

@@ -3,7 +3,3 @@
 Rasmus Search Engine
 
 This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
-
-This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture.
-
-I'm just trying to minimize memory usage while maximizing performance.

RSE.sln

@@ -4,6 +4,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
 EndProject
 Global
@@ -20,6 +24,14 @@ Global
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
 		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU

RSE.sln.DotSettings.user

@@ -4,33 +4,19 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueueSegment_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4f72364deba8e6142d1f42386f1fd282d2e8c6bca41d389c28da4c3835d90_003FConcurrentQueueSegment_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F85_003Fa8fb60ce_003FConcurrentQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADnsEndPoint_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F3d_003F64a36162_003FDnsEndPoint_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIcmpV6MessageType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F4e_003Fec7b627f_003FIcmpV6MessageType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4682fe5857f946a98dcb2cd6f0a403983f200_003F3d_003Fca9070b2_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARawSocketPermissions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003Fc7_003Fb8dcfca9_003FRawSocketPermissions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>

global.json

@@ -1,7 +0,0 @@
-{
-  "sdk": {
-    "version": "9.0.0",
-    "rollForward": "latestMajor",
-    "allowPrerelease": true
-  }
-}