Change favicon.ico to favicon.jpg

.gitignore

@@ -7,7 +7,6 @@
 **/**/bin/*
 **/obj/*
 **/**/obj/*
-*.gz
 
 # User-specific stuff
 .idea/**/workspace.xml

Analyze/Program.cs

@@ -1,3 +0,0 @@
-// See https://aka.ms/new-console-template for more information
-
-Console.WriteLine("Hello, World!");

Backend/Backend.csproj

@@ -2,15 +2,21 @@
     
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <PublishAot>false</PublishAot>
+
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+        <!--<Platform>x64</Platform>-->
+        <Optimize>false</Optimize>
     </PropertyGroup>
     
     <ItemGroup>
       <ProjectReference Include="..\Models\Models.csproj" />
     </ItemGroup>
     
+    <ItemGroup>
+      <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+    
 </Project>

Backend/Handler/Communication.cs

@@ -0,0 +1,200 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Backend.Helper;
+using Models.Handler;
+using Models.Model.Backend;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+
+namespace Backend.Handler;
+
+public class Communication
+{
+    private readonly DbHandler _dbHandler;
+    private readonly ThreadHandler _threadHandler;
+    private readonly IpScanner _ipScanner;
+    private readonly ContentFilter _contentFilter;
+    private bool _isRunning = true;
+    private string _basePath;
+    
+    public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
+    {
+        _dbHandler = dbHandler;
+        _threadHandler = threadHandler;
+        _ipScanner = ipScanner;
+        _contentFilter = contentFilter;
+        _basePath = basePath;
+    }
+    
+    public WaitHandle[] Start()
+    {
+        WaitHandle[] waitHandles = new WaitHandle[1];
+        EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+        waitHandles[0] = handle;
+        
+        Thread thread = new(Server!);
+        thread.Start(handle);
+        
+        return waitHandles;
+    }
+    
+    private void Server(object obj)
+    {
+        using ResponseSocket rep = new();
+        
+        //rep.Options.IPv4Only = true;
+        
+        rep.Bind("tcp://127.0.0.1:5556");
+        
+        while (_isRunning)
+        {
+            byte[] message = rep.ReceiveFrameBytes();
+            
+            CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
+
+            //rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
+            
+            if (communicationObject is null)
+            {
+                continue;
+            }
+            
+            OnServerOnReceiveReady(communicationObject, rep);
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
+    private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
+    {
+        switch (communicationObject.Command)
+        {
+            case CommunicationCommand.GetScanningProgress:
+            {
+                DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
+                
+                long discardedIndexes = _dbHandler.GetDiscardedIndexes();
+                
+                ScanningStatus status = new();
+                // 4294967296 is all Ipv4 addresses.
+                
+                if (discardedIndexes != 0)
+                {
+                    status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
+                }
+                else
+                {
+                    status.PercentageOfIpv4Scanned = 0.0000000001f;
+                }
+                
+                status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
+                status.TotalFiltered = _dbHandler.GetFilteredIndexes();
+                status.TotalDiscarded = discardedIndexes;
+                status.MyDbSize = databaseSizes.MyDbSize;
+                status.FilteredDbSize = databaseSizes.FilteredDbSize;
+                status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
+
+                byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
+                
+                rep.SendFrame(serializedResult);
+                
+                break;
+            }
+            
+            case CommunicationCommand.DbReindex:
+            {
+                _dbHandler.ReIndex();
+                
+                SendStringResponse(rep, "All Dbs have been reindexed.");
+                break;
+            }
+            
+            case CommunicationCommand.DbVacuum:
+            {
+                _dbHandler.Vacuum();
+                
+                SendStringResponse(rep, "All Dbs have been vacuumed.");
+                break;
+            }
+            
+            case CommunicationCommand.GetSearches:
+            {
+                if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
+                {
+                    SendSearchResponse(rep, communicationObject.SearchTerm);
+                }
+                
+                break;
+            }
+            
+            case CommunicationCommand.StopScanning:
+            {
+                _isRunning = false;
+                break;
+            }
+            
+            case CommunicationCommand.ChangeRuntimeVariable:
+            {
+                if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
+                
+                if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _dbHandler.SetContentWaitTime(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _dbHandler.SetDiscardedWaitTime(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _ipScanner.SetTimeout(value);
+                }
+                
+                if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
+                {
+                    int value = int.Parse(communicationObject.VariableValue);
+                    _contentFilter.SetTimeout(value);
+                }
+                
+                rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
+                
+                break;
+            }
+        }
+    }
+    
+    private static void SendStringResponse(ResponseSocket rep, string response)
+    {
+        byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
+        
+        rep.SendFrame(serializedResult);
+    }
+
+    private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
+    {
+        //SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
+        SearchResults result = new()
+        {
+            Results = []
+        };
+        SearchResult lol = new()
+        {
+            Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
+            Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
+        };
+        
+        result.Results.Add(lol);
+        
+        string serializedResult = JsonSerializer.Serialize(result);
+        
+        rep.SendFrame(serializedResult);
+    }
+}

Backend/Handler/ContentFilter.cs

@@ -6,44 +6,27 @@ using Models.Model.Backend;
 
 namespace Backend.Handler;
 
-public class Content
-{
-    public int Port1 { get; set; }
-    public int Port2 { get; set; }
-    public Ip Ip { get; set; }
-}
-
-public class ContentThread
-{
-    public int ThreadId { get; set; }
-    public EventWaitHandle? EventWaitHandle { get; set; }
-}
-
 public class ContentFilter
 {
     private readonly ConcurrentQueue<Filtered> _queue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
-    private readonly ConcurrentQueue<Content?> _contentQueue = new();
     private readonly DbHandler _dbHandler;
-    private readonly ThreadHandler _threadHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
     private bool _stop;
     private int _timeOut;
     private readonly string _basePath;
     
-    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
+    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
-        _threadHandler = threadHandler;
         _unfilteredQueue = unfilteredQueue;
         
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
-        
         SetTimeout(3000);
     }
     
@@ -70,22 +53,10 @@ public class ContentFilter
         {
             List<long> indexes = _dbHandler.GetUnfilteredIndexes();
             
-            if (indexes.Count == 0)
-            {
-                _stop = true;
-                _threadHandler.Stop();
-                break;
-            }
-            
             for (int i = 0; i < indexes.Count; i++)
             {
                 if (_stop) break;
                 
-                if (_contentQueue.Count >= 500)
-                {
-                    Thread.Sleep(1000);
-                }
-                
                 Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                 
                 if (unfiltered.Filtered) continue;
@@ -107,12 +78,12 @@ public class ContentFilter
                     continue;
                 }
                 
-                Content content = new();
+                Filtered filtered = GetSiteData(ip);
-                content.Ip = ip;
-                content.Port1 = unfiltered.Port1;
-                content.Port2 = unfiltered.Port2;
                 
-                _contentQueue.Enqueue(content);
+                filtered.Port1 = unfiltered.Port1;
+                filtered.Port2 = unfiltered.Port2;
+                
+                _queue.Enqueue(filtered);
             }
             
             Thread.Sleep(_timeOut);
@@ -121,64 +92,17 @@ public class ContentFilter
         ((EventWaitHandle) obj).Set();
     }
     
-    public WaitHandle[] StartFilterThread(int threads)
+    private Filtered GetSiteData(Ip ip)
     {
-        WaitHandle[] waitHandle = new WaitHandle[threads];
+        StartProcess(ip, 80);
-
+        StartProcess(ip, 443);
-        for (int i = 0; i < threads; i++)
-        {
-            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-            ContentThread contentThread = new();
-            contentThread.ThreadId = i;
-            contentThread.EventWaitHandle = handle;
-            waitHandle[i] = handle;
-            
-            Thread thread = new(FilterThread!);
-            thread.Start(contentThread);
-            
-            Thread.Sleep(8);
-        }
-        
-        return waitHandle;
-    }
-
-    private void FilterThread(object obj)
-    {
-        Console.WriteLine("Filter Thread started.");
-        ContentThread thread = (ContentThread) obj;
-        
-        while (!_stop)
-        {
-            if (_contentQueue.IsEmpty)
-            {
-                Thread.Sleep(1000);
-            }
-            
-            _contentQueue.TryDequeue(out Content? content);
-
-            if (content is null)
-            {
-                continue;
-            }
-            
-            Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
-                
-            filtered.Port1 = content.Port1;
-            filtered.Port2 = content.Port2;
-                
-            _queue.Enqueue(filtered);
-        }
-        
-        thread.EventWaitHandle!.Set();
-    }
-    
-    private Filtered GetSiteData(Ip ip, int threadId)
-    {
-        StartProcess(ip, 80, threadId);
-        StartProcess(ip, 443, threadId);
         
         string url1 = "";
         string url2 = "";
+        string title1 = "";
+        string title2 = "";
+        string description1 = "";
+        string description2 = "";
         bool robotsTxt1 = false;
         bool robotsTxt2 = false;
         string serverType1 = "";
@@ -204,7 +128,7 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
+            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
             
             while (streamReader.Peek() != -1)
             {
@@ -237,6 +161,41 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
+            string? html = "";
+            
+            if (ports[i] == 80)
+            {
+                if (string.IsNullOrWhiteSpace(url1)) continue;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
+            }
+            else
+            {
+                if (string.IsNullOrWhiteSpace(url2)) continue;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
+            }
+            
+            if (string.IsNullOrWhiteSpace(html)) continue;
+            
+            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
+            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
+            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
+            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
             if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
             if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }
@@ -246,6 +205,10 @@ public class ContentFilter
             Ip = ip,
             Url1 = url1,
             Url2 = url2,
+            Title1 = title1,
+            Title2 = title2,
+            Description1 = description1,
+            Description2 = description2,
             ServerType1 = serverType1,
             ServerType2 = serverType2,
             RobotsTXT1 = robotsTxt1,
@@ -271,7 +234,7 @@ public class ContentFilter
         return siteData;
     }
     
-    private void StartProcess(Ip ip, int port, int threadId)
+    private void StartProcess(Ip ip, int port)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
         
@@ -279,7 +242,7 @@ public class ContentFilter
         proc.StartInfo = new()
         {
             FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
+            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,

Backend/Handler/IpFilterHandler.cs

@@ -1,131 +0,0 @@
-using System.Collections.Concurrent;
-using Backend.Helper;
-using Models.Model.Backend;
-
-namespace Backend.Handler;
-
-public class IpFilterHandler
-{
-    private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
-    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
-    private ThreadHandler _threadHandler;
-    private bool _stop;
-    private bool _done;
-    private bool _stopAutoscaledThreads;
-    private int _timeout;
-    
-    public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<Ip> preFilteredQueue, ThreadHandler threadHandler)
-    {
-        _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
-        _preFilteredQueue = preFilteredQueue;
-        _threadHandler = threadHandler;
-
-        _timeout = 16;
-    }
-    
-    public List<WaitHandle[]> Start(int threadCount)
-    {
-        WaitHandle[] waitHandle = new WaitHandle[64];
-        
-        int counter = 0;
-        
-        List<WaitHandle[]> waitHandles = [];
-        
-        for (int i = 0; i < threadCount; i++)
-        {
-            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-            
-            if (counter < 64)
-            {
-                waitHandle[counter] = handle;
-                counter++;
-                
-                Thread f = new (Filter!);
-                f.Start(handle);
-                
-                Console.WriteLine($"Filter thread ({i}) started");
-                Thread.Sleep(16);
-                
-                continue;
-            }
-            
-            counter = 0;
-            
-            waitHandles.Add(waitHandle);
-            
-            waitHandle = new WaitHandle[64];
-        }
-        
-        return waitHandles;
-    }
-    
-    private void Filter(object obj)
-    {
-        int counter = 0;
-        while (!_stop)
-        {
-            if (_preFilteredQueue.IsEmpty)
-            {
-                if (counter == 30_000)
-                {
-                    _threadHandler.Stop();
-                    _stop = true;
-                }
-                
-                counter++;
-                Thread.Sleep(128);
-                
-                continue;
-            }
-            
-            _preFilteredQueue.TryDequeue(out Ip item);
-            
-            (int, int) ports = TcpClientHelper.CheckPort(item, 80, 443);
-            
-            if (ports is { Item1: 0, Item2: 0 })
-            {
-                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item, item.ResponseCode));
-                continue;
-            }
-            
-            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item, ports));
-        }
-        
-        ((EventWaitHandle) obj).Set();
-    }
-    
-    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
-    {
-        return new()
-        {
-            Ip = ip,
-            ResponseCode = responseCode
-        };
-    }
-    
-    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
-    {
-        Unfiltered unfiltered = new()
-        {
-            Ip = ip,
-            Port1 = ports.Item1,
-            Port2 = ports.Item2,
-            Filtered = false
-        };
-        
-        return new()
-        {
-            Unfiltered = unfiltered,
-            Operations = Operations.Insert
-        };
-    }
-
-    public void Stop()
-    {
-        _stop = true;
-    }
-}

Backend/Handler/IpScanner.cs

@@ -1,11 +1,7 @@
-using System.Buffers.Binary;
 using System.Collections.Concurrent;
 using System.Net;
 using System.Net.NetworkInformation;
-using System.Numerics;
-using System.Runtime.InteropServices;
 using Backend.Helper;
-using Models.Experimental;
 using Models.Handler;
 using Models.Model.Backend;
 
@@ -22,22 +18,27 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
-    private readonly int _timeout;
+    private int _timeout;
     
-    public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
+    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
-        ConcurrentQueue<Ip> preFilteredQueue)
+        )
     {
         _dbHandler = dbHandler;
-        _preFilteredQueue = preFilteredQueue;
         _discardedQueue = discardedQueue;
+        _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
 
-        _timeout = 32;
+        SetTimeout(64);
+    }
+    
+    public void SetTimeout(int milliseconds)
+    {
+        _timeout = milliseconds;
     }
     
     public List<WaitHandle[]> Start(int threads)
@@ -48,11 +49,22 @@ public class IpScanner
             threadsAmount = 256 / threads;
         }
 
-        WaitHandle[] waitHandle = new WaitHandle[64];
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
+        
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
         
         int counter = 0;
-        
+        int counter2 = 0;
-        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -66,26 +78,31 @@ public class IpScanner
                 Handle = handle
             };
 
-            if (counter < 64)
+            if (i < 64)
             {
-                waitHandle[counter] = handle;
+                waitHandle1[counter] = handle;
                 counter++;
+            }
+            else
+            {
+                waitHandle2[counter2] = handle;
+                counter2++;
+            }
             
             Thread f = new (Scan!);
             f.Start(scanSettings);
             
             Console.WriteLine($"Scanner thread ({i}) started");
-                Thread.Sleep(128);
+            Thread.Sleep(100);
-                
-                continue;
         }
         
-            counter = 0;
+        List<WaitHandle[]> waitHandles = new();
         
-            waitHandles.Add(waitHandle);
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
         
-            waitHandle = new WaitHandle[64];
+        waitHandles.Add(waitHandle1);
-        }
+        waitHandles.Add(waitHandle2);
         
         return waitHandles;
     }
@@ -107,26 +124,15 @@ public class IpScanner
         
         if (resumeNow is not null)
         {
-            if (resumeNow.Completed)
-            {
-                return;
-            }
-            
             scanSettings.Start = resumeNow.FirstByte;
             scanSettings.End = resumeNow.EndRange;
             secondByte = resumeNow.SecondByte;
             thirdByte = resumeNow.ThirdByte;
             fourthByte = resumeNow.FourthByte;
         }
-        else
-        {
-            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
-        }
         
-        // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
-        int x = 0;
         
         for (int i = scanSettings.Start; i < scanSettings.End; i++)
         {
@@ -145,34 +151,19 @@ public class IpScanner
                     if (i == 192 && k == 2) continue;
                     if (i == 192 && j == 88 && k == 99) continue;
                     
-                    if (_discardedQueue.Count >= 20_000)
+                    if (_discardedQueue.Count >= 2000)
                     {
-                        Thread.Sleep(1000);
+                        Thread.Sleep(500);
-                    }
-                    
-                    if (_preFilteredQueue.Count >= 20_000)
-                    {
-                        Thread.Sleep(1000);
                     }
                     
                     for (int l = fourthByte; l < 256; l++)
                     {
-                        if (x == 75_000)
-                        {
-                            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
-                            x = 0;
-                        }
-                        
-                        x++;
-                        
                         if (_stop)
                         {
                             resumeObject.FourthByte = l;
                             break;
                         }
 
-                        
-                        
                         Ip ip = new()
                         {
                             Ip1 = i,
@@ -187,22 +178,11 @@ public class IpScanner
                         {
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
-                            //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
+                            _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
-                            
+                            if (address is not null)
-                            responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
-                            
-                            /*if (l % 2 == 0)
                             {
-                                responseCode = IPStatus.Success;
+                                responseCode = ping.Send(address, _timeout, buf, null).Status;
                             }
-                            else
-                            {
-                                responseCode = IPStatus.TimedOut;
-                            }*/
-                            
-                            //responseCode = IPStatus.TimedOut;
-                            
-                            //Thread.Sleep(0);
                         }
                         catch
                         {
@@ -215,7 +195,15 @@ public class IpScanner
                             continue;
                         }
                         
-                        _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
+                        (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
+                        
+                        if (ports is { Item1: 0, Item2: 0 })
+                        {
+                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
+                            continue;
+                        }
+                        
+                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
                     }
                     
                     if (_stop)
@@ -237,43 +225,15 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-        }
             
-        Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped.");
+            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
-        
-        if (_stop)
-        {
-            resumeObject.Paused = true;
         }
-        else
-        {
-            resumeObject.Completed = true;
-        }
-        
-        resumeObject.Operation = Operations.Update;
         
         _resumeQueue.Enqueue(resumeObject);
         
         scanSettings.Handle!.Set();
     }
     
-    private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
-    {
-        ScannerResumeObject resumeObject = new();
-        resumeObject.ThreadNumber = threadNumber;
-        resumeObject.StartRange = startRange;
-        resumeObject.EndRange = endRange;
-        resumeObject.FirstByte = firstByte;
-        resumeObject.SecondByte = secondByte;
-        resumeObject.ThirdByte = thirdByte;
-        resumeObject.FourthByte = fourthByte;
-        resumeObject.Paused = paused;
-        resumeObject.Completed = completed;
-        resumeObject.Operation = operation;
-        
-        _resumeQueue.Enqueue(resumeObject);
-    }
-    
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
         return new()
@@ -283,110 +243,25 @@ public class IpScanner
         };
     }
     
-    private static Ip CreateUnfilteredQueueItem(Ip ip, int responseCode)
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
     {
-        ip.ResponseCode = responseCode;
+        Unfiltered unfiltered = new()
+        {
+            Ip = ip,
+            Port1 = ports.Item1,
+            Port2 = ports.Item2,
+            Filtered = false
+        };
         
-        return ip;
+        return new()
+        {
+            Unfiltered = unfiltered,
+            Operations = Operations.Insert
+        };
     }
     
     public void Stop()
     {
         _stop = true;
     }
-    
-    private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
-    {
-        int length = ipSpan.Length;
-        long nonCanonical;
-        fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
-            nonCanonical = ParseNonCanonical(name, 0, ref length, true);
-        
-        return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
-    }
-    
-    private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
-    {
-        long* numPtr = stackalloc long[4];
-        long num1 = 0;
-        bool flag = false;
-        int index1 = 0;
-        int index2;
-        for (index2 = start; index2 < end; ++index2)
-        {
-            char ch = name[index2];
-            num1 = 0L;
-            int num2 = 10;
-            if (ch == '0')
-            {
-                num2 = 8;
-                ++index2;
-                flag = true;
-                if (index2 < end)
-                {
-                    switch (name[index2])
-                    {
-                        case 'X':
-                        case 'x':
-                            num2 = 16;
-                            ++index2;
-                            flag = false;
-                            break;
-                    }
-                }
-            }
-            for (; index2 < end; ++index2)
-            {
-                char c = name[index2];
-                int num3;
-                if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
-                    num3 = (int) c - 48;
-                else if (num2 == 8 && '0' <= c && c <= '7')
-                    num3 = (int) c - 48;
-                else if (num2 == 16 && 'a' <= c && c <= 'f')
-                    num3 = (int) c + 10 - 97;
-                else if (num2 == 16 && 'A' <= c && c <= 'F')
-                    num3 = (int) c + 10 - 65;
-                else
-                    break;
-                num1 = num1 * (long) num2 + (long) num3;
-                if (num1 > (long) uint.MaxValue)
-                    return -1;
-                flag = true;
-            }
-            if (index2 < end && name[index2] == '.')
-            {
-                if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
-                    return -1;
-                numPtr[index1] = num1;
-                ++index1;
-                flag = false;
-            }
-            else
-                break;
-        }
-        if (!flag)
-            return -1;
-        if (index2 < end)
-        {
-            char ch;
-            if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
-                return -1;
-            end = index2;
-        }
-        numPtr[index1] = num1;
-        switch (index1)
-        {
-            case 0:
-                return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
-            case 1:
-                return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
-            case 2:
-                return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
-            case 3:
-                return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
-            default:
-                return -1;
-        }
-    }
 }

Backend/Handler/ThreadHandler.cs

@@ -7,111 +7,59 @@ namespace Backend.Handler;
 public class ThreadHandler
 {
     private readonly DbHandler _dbHandler;
+    private readonly Communication _communication;
     private readonly IpScanner _ipScanner;
     private readonly ContentFilter _contentFilter;
-    private readonly IpFilterHandler _ipFilterHandler;
     
+    private bool _communicationStopped;
     private bool _ipScannerStopped;
     private bool _contentFilterStopped;
-    private bool _ipFilterStopped;
-
-    private bool _stage1 = false;
-    private bool _stage2 = true;
-    private bool _stage3 = false;
     
+    public ThreadHandler(string path)
+    {
         ConcurrentQueue<Filtered> filteredQueue = new();
         ConcurrentQueue<Discarded> discardedQueue = new();
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
         ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
-    ConcurrentQueue<Ip> preFilteredQueue = new();
         
-    public ThreadHandler(string path)
+        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-    {
+        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
-        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
+        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
-        _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
+        _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
-        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
-        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, this);
     }
     
     public void Start()
     {
         Thread scanner = new(StartScanner);
-        Thread ipFilter = new(StartIpFilter);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
-        Thread contentFilterThread = new(StartContentFilterThread);
+        Thread communication = new(StartCommunicationHandler);
-        Thread prefilterDb = new(StartPreFilterDbHandler);
-        Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
-        //Thread check = new(CheckQueue);
         
-        //check.Start();
+        scanner.Start();
-        
+        indexer.Start();
-        if (_stage1)
+        database.Start();
-        {
+        discarded.Start();
-            discarded.Start(); // de-queues from discardedQueue
+        filtered.Start();
-            prefilterDb.Start(); // de-queues from preFilteredQueue
+        resume.Start();
-            scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
+        communication.Start();
-            resume.Start(); // de-queues from resumeQueue
         
         scanner.Join();
-            Stop();
-            
-            discarded.Join();
-            prefilterDb.Join();
-            resume.Join();
-        }
-        
-        if (_stage2)
-        {
-            database.Start(); // de-queues from unfilteredQueue
-            discarded.Start(); // de-queues from discardedQueue
-            ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
-            fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
-
-            ipFilter.Join();
-            Stop();
-            database.Join();
-            discarded.Join();
-            fillIpFilterQueue.Join();
-        }
-        
-        if (_stage3)
-        {
-            filtered.Start(); // de-queues from filteredQueue
-            database.Start(); // de-queues from unfilteredQueue
-            indexer.Start(); // en-queues to unfilteredQueue and contentQueue
-            contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
-            
-            contentFilterThread.Join();
-            filtered.Join();
-            database.Join();
         indexer.Join();
+        database.Join();
+        discarded.Join();
+        filtered.Join();
+        resume.Join();
+        communication.Join();
     }
-    }
-    
-    private void CheckQueue()
-    {
-        while (true)
-        {
-            Console.Clear();
-            Console.WriteLine($"filteredQueue.Count:       {filteredQueue.Count}");
-            Console.WriteLine($"discardedQueue.Count:      {discardedQueue.Count}");
-            Console.WriteLine($"unfilteredQueue.Count:     {unfilteredQueue.Count}");
-            Console.WriteLine($"scannerResumeQueue.Count:  {scannerResumeQueue.Count}");
-            Console.WriteLine($"preFilteredQueue.Count:    {preFilteredQueue.Count}");
-            Thread.Sleep(5);
-        }
-    }
-
     
     private void StartScanner()
     {
-        Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
+        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(256);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -119,6 +67,8 @@ public class ThreadHandler
         }
         
         Console.WriteLine("Scanner finished");
+        
+        _ipScannerStopped = true;
     }
     
     private void StartContentFilter()
@@ -134,34 +84,6 @@ public class ThreadHandler
         _contentFilterStopped = true;
     }
     
-    private void StartContentFilterThread()
-    {
-        WaitHandle[] wait = _contentFilter.StartFilterThread(8);
-        
-        WaitHandle.WaitAll(wait);
-    }
-
-    private void StartFillIpFilterQueue()
-    {
-        _dbHandler.GetPreFilterQueueItem();
-    }
-
-    private void StartIpFilter()
-    {
-        Thread.Sleep(1000);
-        
-        List<WaitHandle[]> wait = _ipFilterHandler.Start(1024);
-
-        for (int i = 0; i < wait.Count; i++)
-        {
-            WaitHandle.WaitAll(wait[i]);
-        }
-        
-        Console.WriteLine("Ip filter finished");
-        
-        _ipFilterStopped = true;
-    }
-    
     private void StartDbHandler()
     {
         _dbHandler.UnfilteredDbHandler();
@@ -172,11 +94,6 @@ public class ThreadHandler
         _dbHandler.FilteredDbHandler();
     }
     
-    private void StartPreFilterDbHandler()
-    {
-        _dbHandler.PrefilteredDbHandler();
-    }
-    
     private void StartResumeDbHandler()
     {
         _dbHandler.ResumeDbHandler();
@@ -191,19 +108,35 @@ public class ThreadHandler
         Console.WriteLine("Discarded DbHandler finished");
     }
     
-    public void Stop()
+    private void StartCommunicationHandler()
+    {
+        WaitHandle[] wait = _communication.Start();
+        
+        WaitHandle.WaitAll(wait);
+        
+        Console.WriteLine("Communicator finished");
+        
+        _communicationStopped = true;
+        
+        Stop();
+    }
+    
+    private void Stop()
     {
-        Console.WriteLine("Stopping...");
         _ipScanner.Stop();
         _contentFilter.Stop();
-        _ipFilterHandler.Stop();
-        Console.WriteLine("Stopping Extra...");
         
-        Thread.Sleep(30_000);
+        bool stopping = true;
         
-        Console.WriteLine("Stopping Super Extra...");
+        while (stopping)
+        {
+            if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
+            {
                 _dbHandler.Stop();
+                stopping = false;
+            }
             
-        Console.WriteLine("Stopped.");
+            Thread.Sleep(3000);
+        }
     }
 }

Backend/Helper/FilesystemHelper.cs

@@ -0,0 +1,43 @@
+using Models.Model.Backend;
+
+namespace Backend.Helper;
+
+public static class FilesystemHelper
+{
+    public static DatabaseSizes GetDatabaseSizes(string basePath)
+    {
+        DatabaseSizes databaseSizes = new();
+        
+        FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
+        databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
+        
+        databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
+        
+        fileInfo = new($"{basePath}/Models/Filtered.db");
+        databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
+        
+        return databaseSizes;
+    }
+
+    private static long GetDiscardedDbSizes(string basePath)
+    {
+        string folder = $"{basePath}/Models";
+        string[] files = Directory.GetFiles(folder, "*.db");
+        
+        long size = 0;
+        
+        for (int i = 0; i < files.Length; i++)
+        {
+            if (!files[i].Contains("Discarded")) continue;
+            FileInfo fileInfo = new(files[i]);
+            size += fileInfo.Length;
+        }
+        
+        return size;
+    }
+    
+    private static double ToSize(this long value, SizeUnits unit)
+    {
+        return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
+    }
+}

Backend/Helper/FilterHelper.cs

@@ -2,10 +2,12 @@ using System.Text.RegularExpressions;
 
 namespace Backend.Helper;
 
-public static class FilterHelper
+public static partial class FilterHelper
 {
     // https://stackoverflow.com/a/56116499
     private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
     
     public static void GetDomain(string data, out string result)
     {
@@ -40,6 +42,32 @@ public static class FilterHelper
         result = GetSubstring(data, start, end);
     }
     
+    public static void GetTitle(string data, out string result)
+    {
+        result = "";
+        
+        Regex titleRegex = MyRegex1();
+        Match match = titleRegex.Match(data);
+        
+        if (match.Success)
+        {
+            result = match.Groups[1].Value;
+        }
+    }
+    
+    public static void GetDescription(string data, out string result)
+    {
+        result = "";
+        
+        Regex titleRegex = MyRegex2();
+        Match match = titleRegex.Match(data);
+        
+        if (match.Success)
+        {
+            result = match.Groups[1].Value;
+        }
+    }
+    
     public static void GetHttpVersion(string data, out string result)
     {
         result = "";
@@ -170,7 +198,7 @@ public static class FilterHelper
             case 0:
                 return;
             case 2:
-                result = $"{temp[1].Trim()}";
+                result = $"{temp[1].Trim()} UUHHHMMM";
                 return;
             default:
                 result = temp[2].Trim();
@@ -213,7 +241,17 @@ public static class FilterHelper
     
     private static bool ValidateUrl(string url)
     {
-        Regex rgx = new(DomainPattern);
+        Regex rgx = MyRegex();
         return rgx.IsMatch(url);
     }
+    
+    //Generate the RegEx at compile time.
+    [GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
+    private static partial Regex MyRegex();
+    
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex MyRegex1();
+    
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex MyRegex2();
 }

Backend/Helper/HttpClientHelper.cs

@@ -1,47 +1,64 @@
-using System.Text.RegularExpressions;
+using System.Diagnostics;
 
 namespace Backend.Helper;
 
-public static partial class HttpClientHelper
+public static class HttpClientHelper
 {
-    // Reddit, for example, will block the GET request if you don't have a user agent.
+    public static async Task<string> GetHtml(string url, int port)
-    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    {
+        using HttpClient client = new();
+        
+        if (port == 80)
+        {
+            client.BaseAddress = new($"http://{url}");
+        }
+        else
+        {
+            client.BaseAddress = new($"https://{url}");
+        }
+        
+        client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(1);
+        
+        HttpResponseMessage? response;
+        
+        try
+        {
+            response = await client.GetAsync("/");
+        }
+        catch
+        {
+            return "";
+        }
+        
+        if (!response.IsSuccessStatusCode)
+        {
+            return "";
+        }
+        
+        return await response.Content.ReadAsStringAsync();
+    }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
     {
         using HttpClient client = new();
         
         if (port == 80)
-        {
-            try
         {
             client.BaseAddress = new($"http://{url}");
         }
-            catch
-            {
-                //
-            }
-        }
         else
-        {
-            try
         {
             client.BaseAddress = new($"https://{url}");
         }
-            catch
-            {
-                //
-            }
-        }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(1);
-        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {
+        {//
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch

Backend/Helper/SearchHelper.cs

@@ -0,0 +1,44 @@
+using FuzzySharp;
+using Models.Handler;
+using Models.Model.External;
+
+namespace Backend.Helper;
+
+public static class SearchHelper
+{
+    public static SearchResults Search(string searchText, DbHandler dbHandler)
+    {
+        if (string.IsNullOrWhiteSpace(searchText))
+        {
+            SearchResult searchResult = new();
+            searchResult.Description = "asd";
+            searchResult.Title = "asd";
+            searchResult.Url = "asd";
+            SearchResults searchResults = new();
+            searchResults.Results =
+            [
+                searchResult
+            ];
+            return searchResults;
+        }
+        
+        List<SearchResult?> temp = dbHandler.GetSearchResults();
+        SearchResults searchResultsList = new();
+        
+        for (int i = 0; i < temp.Count; i++)
+        {
+            if (temp[i] is null) continue;
+            
+            SearchResult result = new();
+            
+            if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
+            
+            result.Url = temp[i]?.Url;
+            result.Title = temp[i]?.Title;
+            
+            searchResultsList.Results.Add(result);
+        }
+        
+        return searchResultsList;
+    }
+}

Backend/Helper/TcpClientHelper.cs

@@ -1,24 +1,20 @@
 using System.Net;
 using System.Net.Sockets;
-using Models.Model.Backend;
 
 namespace Backend.Helper;
 
 public static class TcpClientHelper
 {
-    public static (int, int) CheckPort(Ip ip, params int[] ports)
+    public static (int, int) CheckPort(string ip, params int[] ports)
     {
         // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached,
         // and it would just return a result.error, for example.
         for (int i = 0; i < ports.Length; i++) {
-            using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
-            socket.SendTimeout = 250;
-            
             try
             {
-                socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
+                using TcpClient client = new();
-                socket.Close();
+                client.Connect(ip, ports[i]);
-                // If the connection is not successful, update the ports array with 0.
+                // If the connection is successful, update the result array with the port number
             }
             catch
             {

Manager/Commands.cs

@@ -0,0 +1,122 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Models.Model.Backend;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+
+namespace Manager;
+
+public static class Commands
+{
+    public static void GetProgress()
+    {
+        Console.WriteLine("Getting progress ...");
+        
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.GetScanningProgress;
+        
+        ScanningStatus temp = GetProgress(communicationObject);
+        
+        Console.WriteLine($"Total filtered:           {temp.TotalFiltered:n0}");
+        Console.WriteLine($"Total discarded:          {temp.TotalDiscarded:n0}");
+        Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}");
+        Console.WriteLine($"Total Ips left:           {temp.AmountOfIpv4Left:n0}");
+        Console.WriteLine($"Filtered DB size:         {temp.FilteredDbSize} Kb");
+        Console.WriteLine($"Discarded DB size:        {temp.DiscardedDbSize} Kb");
+        Console.WriteLine($"Mydb DB size:             {temp.MyDbSize} Kb");
+    }
+    
+    public static void StopServer()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.StopScanning;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void Vacuum()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.DbVacuum;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void ReIndex()
+    {
+        CommunicationObject communicationObject = new();
+        communicationObject.Command = CommunicationCommand.DbReindex;
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+
+    public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
+    {
+        CommunicationObject communicationObject = new()
+        {
+            Command = CommunicationCommand.ChangeRuntimeVariable,
+            Variable = runtimeVariable.ToString(),
+            VariableValue = value
+        };
+        
+        Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
+    }
+    
+    public static void GetHelp()
+    {
+        Console.WriteLine("Available commands:");
+        Console.WriteLine("  stop  - stops the server");
+        Console.WriteLine("  clear - clears the console");
+        Console.WriteLine("  q     - quits the program");
+        Console.WriteLine("  p     - print the progress information of the scanner");
+        Console.WriteLine("  g     - manual garbage collect on the server");
+        Console.WriteLine("  r     - manual reindex the databases");
+        Console.WriteLine("  v     - manual vacuum the databases");
+        Console.WriteLine("  R     - change runtime variable");
+        Console.WriteLine("  lR    - list runtime variables");
+        Console.WriteLine("  help  - shows this help");
+        Console.WriteLine();
+    }
+    
+    public static void PrintRuntimeVariables()
+    {
+        Console.WriteLine("Runtime variables:");
+        Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()}   - Sets the timeout in milliseconds for the scanner");
+        Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()}    - Sets the timeout in milliseconds for the content filter");
+        Console.WriteLine($"{RuntimeVariable.DbContent.ToString()}        - Sets the wait time in milliseconds for the content database if the queue is empty");
+        Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()}      - Sets the wait time in milliseconds for the discarded database if the queue is empty");
+    }
+
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
+    {
+        //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
+        
+        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+        
+        using RequestSocket client = new();
+        client.Connect("tcp://127.0.0.1:5556");
+        client.SendFrame(lol);
+        byte[] msg = client.ReceiveFrameBytes();
+        client.Close();
+        
+        return JsonSerializer.Deserialize<string>(msg);
+    }
+
+    [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
+    private static ScanningStatus GetProgress(CommunicationObject communicationObject)
+    {
+        byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+        
+        using RequestSocket client = new();
+        client.Connect("tcp://127.0.0.1:5556");
+        client.SendFrame(lol);
+        byte[] msg = client.ReceiveFrameBytes();
+        client.Close();
+        
+        return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    }
+}

Manager/Manager.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <!--<Platform>x64</Platform>
+        <Optimize>true</Optimize>
+        <PublishAot>true</PublishAot>
+        <PublishTrimmed>true</PublishTrimmed>
+        <TrimMode>link</TrimMode>
+        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Models\Models.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+
+</Project>

Manager/Program.cs

@@ -0,0 +1,90 @@
+using Manager;
+using Models.Model.Backend;
+
+bool stop = false;
+
+do
+{
+    string? input = Console.ReadLine();
+    
+    if (string.Equals(input, "stop"))
+    {
+        Commands.StopServer();
+    }
+    
+    else if (string.Equals(input, "q"))
+    {
+        stop = true;
+    }
+    
+    else if (string.Equals(input, "clear"))
+    {
+        Console.Clear();
+    }
+    
+    else if (string.Equals(input, "p"))
+    {
+        Commands.GetProgress();
+    }
+    
+    else if (string.Equals(input, "v"))
+    {
+        Commands.Vacuum();
+    }
+
+    else if (string.Equals(input, "r"))
+    {
+        Commands.ReIndex();
+    }
+    
+    else if (string.Equals(input, "R"))
+    {
+        Console.WriteLine("Variable name.");
+        string? variable = Console.ReadLine();
+        
+        Console.WriteLine("Variable value.");
+        string? value = Console.ReadLine();
+        
+        if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
+        {
+            Console.WriteLine("Please enter a value.");
+            return;
+        }
+
+        if (variable == RuntimeVariable.ScannerTimeout.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
+        }
+        
+        else if (variable == RuntimeVariable.ContentFilter.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
+        }
+        
+        else if (variable == RuntimeVariable.DbDiscarded.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
+        }
+        
+        else if (variable == RuntimeVariable.DbContent.ToString())
+        {
+            Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
+        }
+    }
+    
+    else if (string.Equals(input, "lR"))
+    {
+        Commands.PrintRuntimeVariables();
+    }
+    
+    else if (string.Equals(input, "help"))
+    {
+        Commands.GetHelp();
+    }
+    
+    else
+    {
+        Commands.GetHelp();
+    }
+    
+} while (!stop);

Models/Experimental/CustomPing.cs

@@ -1,39 +0,0 @@
-using System.Diagnostics;
-using System.Net;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
-using Models.Model.Backend;
-namespace Models.Experimental;
-public class CustomPing
-{
-    public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
-    {
-        SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
-        using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
-        int ipHeaderLength = 20;
-            
-        try
-        {
-            rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
-            byte[] numArray = new byte[136];
-            long timestamp = Stopwatch.GetTimestamp();
-            
-            // TODO: WTF ???
-            EndPoint lol = socketConfig.EndPoint;
-                
-            while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
-            {
-                int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
-                    
-                IPStatus status;
-                if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
-                    return status;
-            }
-        }
-        catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
-        {
-        }
-            
-        return IPStatus.TimedOut;
-    }
-}

Models/Experimental/MessageConstant.cs

@@ -1,59 +0,0 @@
-using System.Net.NetworkInformation;
-namespace Models.Experimental;
-public struct MessageConstant
-{
-  public static IPStatus MapV4TypeToIpStatus(int type, int code)
-  {
-    IPStatus ipStatus1;
-    switch ((IcmpV4MessageType) type)
-    {
-      case IcmpV4MessageType.EchoReply:
-        ipStatus1 = IPStatus.Success;
-        break;
-      case IcmpV4MessageType.DestinationUnreachable:
-        IPStatus ipStatus2;
-        switch ((byte) code)
-        {
-          case 0:
-            ipStatus2 = IPStatus.DestinationNetworkUnreachable;
-            break;
-          case 1:
-            ipStatus2 = IPStatus.DestinationHostUnreachable;
-            break;
-          case 2:
-            ipStatus2 = IPStatus.DestinationProtocolUnreachable;
-            break;
-          case 3:
-            ipStatus2 = IPStatus.DestinationPortUnreachable;
-            break;
-          default:
-            ipStatus2 = IPStatus.DestinationUnreachable;
-            break;
-        }
-        ipStatus1 = ipStatus2;
-        break;
-      case IcmpV4MessageType.SourceQuench:
-        ipStatus1 = IPStatus.SourceQuench;
-        break;
-      case IcmpV4MessageType.TimeExceeded:
-        ipStatus1 = IPStatus.TtlExpired;
-        break;
-      case IcmpV4MessageType.ParameterProblemBadIpHeader:
-        ipStatus1 = IPStatus.BadHeader;
-        break;
-      default:
-        ipStatus1 = IPStatus.Unknown;
-        break;
-    }
-    return ipStatus1;
-  }
-}
-
-internal enum IcmpV4MessageType : byte
-{
-  EchoReply = 0,
-  DestinationUnreachable = 3,
-  SourceQuench = 4,
-  TimeExceeded = 11, // 0x0B
-  ParameterProblemBadIpHeader = 12, // 0x0C
-}

Models/Experimental/RawSocket.cs

@@ -1,91 +0,0 @@
-using System.Net;
-using System.Net.NetworkInformation;
-using System.Net.Sockets;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using Models.Model.Backend;
-
-namespace Models.Experimental;
-
-public class RawSocket
-{
-    public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
-    {
-        Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
-        rawSocket.ReceiveTimeout = socketConfig.Timeout;
-        rawSocket.SendTimeout = socketConfig.Timeout;
-        rawSocket.Connect(socketConfig.EndPoint);
-        int num = 1;
-        rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
-        return rawSocket;
-    }
-    
-    public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
-    {
-        byte num = (byte) (receiveBuffer[0] & 15U);
-        ipHeaderLength = 4 * num;
-        
-        int start = ipHeaderLength;
-        int srcOffset = ipHeaderLength + 8;
-        
-        IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
-        byte[] numArray = new byte[bytesReceived - srcOffset];
-        Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
-        reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
-        return true;
-    }
-    
-    public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
-    {
-        int length = sizeof (IcmpHeader);
-        byte[] sendMessageBuffer = new byte[length];
-        new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
-        ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
-        sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
-        sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
-        return sendMessageBuffer;
-    }
-    
-    private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
-    {
-        uint num1 = 0;
-        for (int index = 0; index < buffer.Length; index += 2)
-        {
-            ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
-            num1 += num2;
-        }
-        while (num1 >> 16 != 0U)
-            num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
-        return (ushort) ~num1;
-    }
-}
-
-public class SocketConfig
-{
-    public EndPoint EndPoint;
-    public readonly int Timeout;
-    public readonly CustomProtocolType ProtocolType;
-    public readonly byte[] SendBuffer;
-
-    public SocketConfig(
-        EndPoint endPoint,
-        int timeout,
-        CustomProtocolType protocolType,
-        byte[] sendBuffer)
-    {
-        EndPoint = endPoint;
-        Timeout = timeout;
-        ProtocolType = protocolType;
-        SendBuffer = sendBuffer;
-    }
-}
-
-
-public struct IcmpHeader
-{
-    public byte Type;
-    public byte Code;
-    public ushort HeaderChecksum;
-    public ushort Identifier;
-    public ushort SequenceNumber;
-}

Models/Handler/DbHandler.cs

@@ -1,10 +1,8 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Text;
 using Microsoft.Data.Sqlite;
-using Models.Helper;
 using Models.Model.Backend;
+using Models.Model.External;
 
 namespace Models.Handler;
 
@@ -14,33 +12,27 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
-    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _preFilteredConnectionString;
+    private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
-    private readonly string _compressedConnectionString;
+    private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
                                            " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
-    private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                           " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
-                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
-    
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
-                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
-                                              " Url1, Url2, ServerType1, ServerType2," +
+                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
                                               " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
                                               " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
                                               " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
                                               " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
-                                              " @url1, @url2, " +
+                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
                                               " @robotsTXT1, @robotsTXT2," +
@@ -75,17 +67,13 @@ public class DbHandler
     
     private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                               " INSERT INTO Discarded (PackedData)" +
+                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
-                                               " VALUES (@packedData)";
+                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
-                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
-
-    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@@ -93,26 +81,20 @@ public class DbHandler
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
-    private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
+    private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
-    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
-    private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
-    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Id == @id ORDER BY Id ASC LIMIT 1;";
-    private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
-    private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
-    
     private const string ReIndexDatabasesStatement = "REINDEX;";
     
     private const string VacuumDatabasesStatement = "VACUUM;";
     
+    private readonly object _readFilteredLock = new();
     private readonly object _readAndDeleteResumeLock = new();
     
     private bool _stop;
     private bool _pause;
     private bool _paused;
-    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
@@ -122,15 +104,12 @@ public class DbHandler
     public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, 
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
-        ConcurrentQueue<Ip> preFilteredQueue,
-        string basePath)
     {
         _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
-        _preFilteredQueue = preFilteredQueue;
 
         SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
@@ -138,10 +117,9 @@ public class DbHandler
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
+        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
-        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
-        _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -204,24 +182,6 @@ public class DbHandler
         Console.WriteLine("Filtered DbHandler stopped.");
     }
     
-    public void PrefilteredDbHandler()
-    {
-        Console.WriteLine("PreFiltered Db handler started.");
-        
-        while (!_stop)
-        {
-            if (_preFilteredQueue.IsEmpty)
-            {
-                Thread.Sleep(4);
-                continue;
-            }
-            
-            _preFilteredQueue.TryDequeue(out Ip queueItem);
-            
-            InsertPrefiltered(queueItem);
-        }
-    }
-    
     public void ResumeDbHandler()
     {
         Console.WriteLine("Resume DB handler started");
@@ -235,10 +195,13 @@ public class DbHandler
                 continue;
             }
             
-            _resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
+            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
             
+            if (queueItem is not null)
+            {
                 InsertResumeObject(queueItem);
             }
+        }
         
         Console.WriteLine("Resume DbHandler stopped.");
     }
@@ -262,7 +225,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(150);
+            Thread.Sleep(1000);
         }
         
         return waitHandles;
@@ -273,20 +236,9 @@ public class DbHandler
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
         
-        int i = 0;
+        while (!_stop)
-        
-        SqliteConnection connection = new(connectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(InsertIntoDiscarded, connection);
-        
-        StringBuilder stringBuilder = new();
-        
-        bool running = true;
-        
-        while (!_stop && running)
         {
             if (_discardedQueue.IsEmpty || _pause)
             {
@@ -295,74 +247,16 @@ public class DbHandler
                 continue;
             }
             
-            if (_stop && _discardedQueue.IsEmpty)
+            _discardedQueue.TryDequeue(out Discarded queueItem);
-            {
+            
-                running = false;
+            InsertDiscarded(queueItem, connectionString);
         }
         
-            if (i == 2048)
-            {
-                command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
-                _ = command.ExecuteNonQuery();
-                
-                // Re-use the command object.
-                command.Parameters.Clear();
-                
-                stringBuilder.Clear();
-                
-                i = 0;
-            }
-            
-            if (_discardedQueue.TryDequeue(out Discarded queueItem))
-            {
-                stringBuilder.Append(queueItem.Ip.PackIp());
-                stringBuilder.Append(':');
-                stringBuilder.Append(queueItem.ResponseCode);
-                stringBuilder.Append(',');
-            }
-            
-            i++;
-        }
-        
-        if (stringBuilder.Length != 0)
-        {
-            command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
-            _ = command.ExecuteNonQuery();
-        }
-        
-        connection.Close();
-        connection.Dispose();
-        command.Dispose();
-        
         discardedDbHandlerSetting.Handle!.Set();
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
     
-    private void DropAndCreateDiscarded(int threadNumber)
-    {
-        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
-        
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-        const string dropStatement = "DROP TABLE Discarded;";
-        const string vacuum = "VACUUM;";
-        
-        using SqliteConnection connection = new(databaseName);
-        connection.Open();
-        
-        SqliteCommand command = new(dropStatement, connection);
-        command.ExecuteNonQuery();
-        
-        command = new(vacuum, connection);
-        command.ExecuteNonQuery();
-        
-        command = new(createStatement, connection);
-        command.ExecuteNonQuery();
-        command.Dispose();
-        
-        connection.Close();
-    }
-    
     private void InsertUnfiltered(Unfiltered unfiltered)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
@@ -382,6 +276,23 @@ public class DbHandler
         connection.Close();
     }
     
+    private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
+    {
+        using SqliteConnection connection = new(dbConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertIntoDiscarded, connection);
+        
+        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
+        command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
+        _ = command.ExecuteNonQuery();
+        
+        connection.Close();
+    }
+    
     private void InsertFiltered(Filtered filtered)
     {
         using SqliteConnection connection = new(_filteredConnectionString);
@@ -464,6 +375,10 @@ public class DbHandler
         command.Parameters.AddWithValue("@port2", filtered.Port2);
         command.Parameters.AddWithValue("@url1", filtered.Url1);
         command.Parameters.AddWithValue("@url2", filtered.Url2);
+        command.Parameters.AddWithValue("@title1", filtered.Title1);
+        command.Parameters.AddWithValue("@title2", filtered.Title2);
+        command.Parameters.AddWithValue("@description1", filtered.Description1);
+        command.Parameters.AddWithValue("@description2", filtered.Description2);
         command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
         command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
         command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
@@ -494,50 +409,19 @@ public class DbHandler
     {
         using SqliteConnection connection = new(_resumeConnectionString);
         connection.Open();
-        SqliteCommand command;
         
-        if (resumeObject.Operation == Operations.Insert)
+        using SqliteCommand command = new(InsertIntoResume, connection);
-        {
-            command = new(InsertIntoResume, connection);
-            command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
-            command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
-        }
-        else
-        {
-            command = new(UpdateResumeStatement, connection);
-        }
         
         command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
+        command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
+        command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
         command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
         command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
         command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
         command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
-        command.Parameters.AddWithValue("@paused", resumeObject.Paused);
-        command.Parameters.AddWithValue("@completed", resumeObject.Completed);
         
         _ = command.ExecuteNonQuery();
         connection.Close();
-        
-        command.Dispose();
-    }
-
-    private void InsertPrefiltered(Ip filterQueueItem)
-    {
-        using SqliteConnection connection = new(_preFilteredConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertPreFilteredStatement, connection);
-        
-        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip1);
-        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip2);
-        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip3);
-        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip4);
-        command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
-        command.Parameters.AddWithValue("@filtered", 0);
-        
-        _ = command.ExecuteNonQuery();
-        
-        connection.Close();
     }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
@@ -607,83 +491,51 @@ public class DbHandler
         return ids;
     }
     
-    public void GetPreFilterQueueItem()
+    public long GetFilteredIndexes()
     {
-        using SqliteConnection connection = new(_preFilteredConnectionString);
+        long rowId = 0;
+        
+        using SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
-        const string temp = "SELECT seq FROM sqlite_sequence;";
+        using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
-        
+        using SqliteDataReader reader = command.ExecuteReader();
-        SqliteCommand command = new(ReadPreFilteredStatement, connection);
-        SqliteCommand command2 = new(UpdatePreFilteredStatement, connection);
-        SqliteCommand command3 = new(temp, connection);
-        SqliteDataReader reader = command3.ExecuteReader();
-
-        long count = 0;
-        
-        while (reader.Read())
-        {
-            count = reader.GetInt64(0);
-        }
-        
-        reader.Close();
-        
-        for (long i = 0; i < count; i++)
-        {
-            if (_preFilteredQueue.Count > 1000)
-            {
-                Thread.Sleep(5);
-            }
-            
-            command.Parameters.AddWithValue("@id", i);
-            reader = command.ExecuteReader();
-            command.Parameters.Clear();
-            
-            Ip ip = new();
         
         if (!reader.HasRows)
         {
-                Console.WriteLine("Uuhhh");
+            return 0;
         }
         
         while (reader.Read())
         {
-                ip.Ip1 = reader.GetInt32(0);
+            rowId = reader.GetInt64(0);
-                ip.Ip2 = reader.GetInt32(1);
-                ip.Ip3 = reader.GetInt32(2);
-                ip.Ip4 = reader.GetInt32(3);
-                ip.ResponseCode = reader.GetInt32(4);
         }
         
-            reader.Close();
+        return rowId;
-            
-            _preFilteredQueue.Enqueue(ip);
-            
-            command2.Parameters.AddWithValue("@id", i);
-            command2.ExecuteNonQuery();
-            command2.Parameters.Clear();
-        }
     }
     
-    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    public long GetDiscardedIndexes()
     {
-        using SqliteConnection connection = new(connectionString);
+        long rowId = 0;
+        
+        for (int i = 0; i < _discardedConnectionStrings.Count; i++)
+        {
+            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
             connection.Open();
             
             using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
             using SqliteDataReader reader = command.ExecuteReader();
             
-        long rowId = 0;
-        
             if (!reader.HasRows)
             {
-            return rowId;
+                return 0;
             }
             
             while (reader.Read())
             {
                 rowId += reader.GetInt64(0);
             }
+        }
         
         return rowId;
     }
@@ -739,6 +591,36 @@ public class DbHandler
         return true;
     }
     
+    public List<SearchResult?> GetSearchResults()
+    {
+        lock (_readFilteredLock)
+        {
+            using SqliteConnection connection = new(_filteredConnectionString);
+            connection.Open();
+            
+            using SqliteCommand command = new(ReadFilteredStatement, connection);
+            using SqliteDataReader reader = command.ExecuteReader();
+            
+            if (!reader.HasRows)
+            {
+                return [];
+            }
+            
+            List<SearchResult?> results = [];
+            
+            while (reader.Read())
+            {
+                SearchResult result = new();
+                result.Title = reader.GetString(0);
+                result.Url = reader.GetString(1);
+                
+                results.Add(result);
+            }
+            
+            return results;
+        }
+    }
+    
     public ScannerResumeObject? GetResumeObject(int threadNumber)
     {
         lock (_readAndDeleteResumeLock)
@@ -746,7 +628,7 @@ public class DbHandler
             using SqliteConnection connection = new(_resumeConnectionString);
             connection.Open();
             
-            using SqliteCommand command = new(ReadResumeStatement, connection);
+            using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
             command.Parameters.AddWithValue("@threadNumber", threadNumber);
             
             using SqliteDataReader reader = command.ExecuteReader();
@@ -767,20 +649,96 @@ public class DbHandler
                 resumeObject.SecondByte = reader.GetInt32(4);
                 resumeObject.ThirdByte = reader.GetInt32(5);
                 resumeObject.FourthByte = reader.GetInt32(6);
-                resumeObject.Paused = reader.GetBoolean(7);
-                resumeObject.Completed = reader.GetBoolean(8);
             }
             
             return resumeObject;
         }
     }
     
-    private (string, string) CreateDiscardedDb(int threadNumber)
+    public void ReIndex()
+    {
+        _pause = true;
+        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
+        
+        if (!_paused)
+        {
+            Thread.Sleep(5000); // Just for safety.
+        }
+        
+        SqliteConnection connection = new(_discardedConnectionString);
+        connection.Open();
+        
+        SqliteCommand command = new(ReIndexDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection = new(_filteredConnectionString);
+        connection.Open();
+        
+        command = new(ReIndexDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection = new(_unfilteredConnectionString);
+        connection.Open();
+        
+        command = new(ReIndexDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection.Dispose();
+        command.Dispose();
+        
+        _pause = false;
+        _paused = false;
+    }
+    
+    public void Vacuum()
+    {
+        _pause = true;
+        
+        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
+
+        if (!_paused)
+        {
+            Thread.Sleep(5000); // Just for safety.
+        }
+        
+        SqliteConnection connection = new(_discardedConnectionString);
+        connection.Open();
+        
+        SqliteCommand command = new(VacuumDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection = new(_filteredConnectionString);
+        connection.Open();
+        
+        command = new(VacuumDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection = new(_unfilteredConnectionString);
+        connection.Open();
+        
+        command = new(VacuumDatabasesStatement, connection);
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        connection.Dispose();
+        command.Dispose();
+        
+        _pause = false;
+        _paused = false;
+    }
+
+    private string CreateDiscardedDb(int threadNumber)
     {
-        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        
+        _discardedConnectionStrings.Add(databaseName);
         
         using SqliteConnection connection = new(databaseName);
         connection.Open();
@@ -788,7 +746,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return (absolutePath, databaseName);
+        return databaseName;
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -1,50 +0,0 @@
-using System.IO.Compression;
-using System.Text;
-
-namespace Models.Helper;
-
-public static class CompressionHelper
-{
-    public static void CompressFile(string sourceFile, string targetFile)
-    {
-        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
-        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
-        using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize);
-        originalFileStream.CopyTo(compressor);
-    }
-    
-    public static void DecompressFile(string sourceFile, string targetFile)
-    {
-        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
-        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
-        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
-        decompressor.CopyTo(decompressedFileStream);
-    }
-    
-    public static string CompressString(string text)
-    {
-        byte[] byteArray = Encoding.UTF8.GetBytes(text);
-        
-        using MemoryStream memoryStream = new();
-        
-        using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize))
-        {
-            compressionStream.Write(byteArray, 0, byteArray.Length);
-        }
-        
-        return Convert.ToBase64String(memoryStream.ToArray());
-    }
-    
-    public static string DecompressString(string compressedText)
-    {
-        byte[] byteArray = Convert.FromBase64String(compressedText);
-        
-        using MemoryStream memoryStream = new(byteArray);
-        
-        using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress);
-        
-        using StreamReader reader = new(decompressionStream);
-        
-        return reader.ReadToEnd();
-    }
-}

Models/Helper/GetDatabasesHelper.cs

@@ -1,9 +0,0 @@
-namespace Models.Helper;
-
-public static class GetDatabasesHelper
-{
-    public static int GetTotalCompressedDatabases(string path)
-    {
-        return Directory.GetFiles(path, "*.gz").Length;
-    }
-}

Models/Model/Backend/CustomProtocolType.cs

@@ -1,6 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum CustomProtocolType
-{
-    Icmp = 1
-}

Models/Model/Backend/DatabaseSizes.cs

@@ -0,0 +1,10 @@
+namespace Models.Model.Backend;
+
+public struct DatabaseSizes
+{
+    public double DiscardedDbSize { get; set; }
+    
+    public double FilteredDbSize { get; set; }
+    
+    public double MyDbSize { get; set; }
+}

Models/Model/Backend/FilterQueueItem.cs

@@ -1,7 +0,0 @@
-namespace Models.Model.Backend;
-
-public struct FilterQueueItem
-{
-    public Ip Ip { get; set; }
-    public int ResponseCode { get; set; }
-}

Models/Model/Backend/Filtered.cs

@@ -3,6 +3,10 @@ namespace Models.Model.Backend;
 public class Filtered
 {
     public Ip Ip { get; set; }
+    public string Title1 { get; set; } = "";
+    public string Title2 { get; set; } = "";
+    public string Description1 { get; set; } = "";
+    public string Description2 { get; set; } = "";
     public string Url1 { get; set; } = "";
     public string Url2 { get; set; } = "";
     public int Port1 { get; set; }

Models/Model/Backend/IP.cs

@@ -10,15 +10,8 @@ public struct Ip
     
     public int Ip4 { get; set; }
     
-    public int ResponseCode { get; set; }
-    
     public override string ToString()
     {
         return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
     }
-    
-    public uint PackIp()
-    {
-        return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4;
-    }
 }

Models/Model/Backend/RuntimeVariable.cs

@@ -0,0 +1,9 @@
+namespace Models.Model.Backend;
+
+public enum RuntimeVariable
+{
+    DbContent,
+    DbDiscarded,
+    ContentFilter,
+    ScannerTimeout
+}

Models/Model/Backend/ScannerResumeObject.cs

@@ -9,7 +9,4 @@ public class ScannerResumeObject
     public int ThirdByte { get; set; }
     public int FourthByte { get; set; }
     public int ThreadNumber { get; set; }
-    public Operations Operation { get; set; }
-    public bool Paused { get; set; }
-    public bool Completed { get; set; }
 }

Models/Model/Backend/SizeUnits.cs

@@ -0,0 +1,8 @@
+namespace Models.Model.Backend;
+public enum SizeUnits
+{
+    Byte,
+    KB,
+    MB,
+    GB,
+}

Models/Model/External/CommunicationCommand.cs

@@ -0,0 +1,11 @@
+namespace Models.Model.External;
+
+public enum CommunicationCommand
+{
+    GetScanningProgress,
+    GetSearches,
+    StopScanning,
+    DbReindex,
+    DbVacuum,
+    ChangeRuntimeVariable,
+}

Models/Model/External/CommunicationObject.cs

@@ -0,0 +1,17 @@
+namespace Models.Model.External;
+
+//[MessagePackObject]
+public class CommunicationObject
+{
+    //[Key(0)]
+    public CommunicationCommand Command { get; set; }
+    
+    //[Key(1)]
+    public string? SearchTerm { get; set; } = "";
+    
+    //[Key(2)]
+    public string? Variable { get; set; } = "";
+    
+    //[Key(3)]
+    public string? VariableValue { get; set; } = "";
+}

Models/Model/External/CommunicationResult.cs

@@ -0,0 +1,8 @@
+namespace Models.Model.External;
+
+public class CommunicationResult
+{
+    public List<SearchResult?>? Result { get; set; }
+    
+    public ScanningStatus? Status { get; set; }
+}

Models/Model/External/ScanningStatus.cs

@@ -0,0 +1,18 @@
+namespace Models.Model.External;
+
+public struct ScanningStatus
+{
+    public float PercentageOfIpv4Scanned { get; set; }
+    
+    public long TotalFiltered { get; set; }
+    
+    public long AmountOfIpv4Left { get; set; }
+    
+    public long TotalDiscarded { get; set; }
+    
+    public double DiscardedDbSize { get; set; }
+    
+    public double FilteredDbSize { get; set; }
+    
+    public double MyDbSize { get; set; }
+}

Models/Model/External/SearchResult.cs

@@ -0,0 +1,8 @@
+namespace Models.Model.External;
+
+public class SearchResult
+{
+    public string? Title { get; set; } = "";
+    public string? Url { get; set; } = "";
+    public string? Description { get; set; } = "";
+}

Models/Model/External/SearchResults.cs

@@ -0,0 +1,6 @@
+namespace Models.Model.External;
+
+public class SearchResults
+{
+    public List<SearchResult?>? Results { get; set; }
+}

Models/Models.csproj

@@ -1,13 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
+      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
     </ItemGroup>
 </Project>

Proxy/Program.cs

@@ -0,0 +1,97 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using AspNetCoreRateLimit;
+using Microsoft.AspNetCore.HttpOverrides;
+using Models.Model.External;
+using NetMQ;
+using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
+
+WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
+
+builder.Services.ConfigureHttpJsonOptions(options =>
+{
+    options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
+});
+
+builder.Services.AddCors(options =>
+{
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
+});
+
+// Add memory cache and rate limiting services
+builder.Services.AddMemoryCache();
+builder.Services.Configure<IpRateLimitOptions>(options =>
+{
+    options.GeneralRules =
+    [
+        new()
+        {
+            Endpoint = "*", // Apply to all endpoints
+            Period = "10s", // Rate limiting window of 10 second
+            Limit = 5 // Maximum 5 requests per 10 seconds
+        }
+    ];
+});
+
+builder.Services.AddInMemoryRateLimiting();
+builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
+
+WebApplication app = builder.Build();
+
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+
+app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
+
+app.UseCors(myAllowSpecificOrigins);
+
+RouteGroupBuilder progressApi = app.MapGroup("/progress");
+progressApi.MapGet("/", () =>
+{
+    CommunicationObject communicationObject = new()
+    {
+        Command = CommunicationCommand.GetScanningProgress
+    };
+    
+    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+    
+    using RequestSocket client = new();
+    client.Connect("tcp://127.0.0.1:5556");
+    client.SendFrame(bytes);
+    byte[] msg = client.ReceiveFrameBytes();
+    client.Close();
+    
+    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+});
+
+/*
+RouteGroupBuilder searchApi = app.MapGroup("/search");
+searchApi.MapGet("/{term}", (string term) =>
+{
+    CommunicationObject communicationObject = new();
+    communicationObject.Command = CommunicationCommand.GetSearches;
+    communicationObject.SearchTerm = term;
+    
+    byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+    
+    using RequestSocket client = new();
+    client.Connect("tcp://127.0.0.1:5556");
+    client.SendFrame(bytes);
+    string msg = client.ReceiveFrameString();
+    client.Close();
+    
+    return JsonSerializer.Deserialize<SearchResults?>(msg);
+});
+*/
+app.Run();
+
+[JsonSerializable(typeof(ScanningStatus))]
+//[JsonSerializable(typeof(SearchResults))]
+[JsonSerializable(typeof(CommunicationObject))]
+internal partial class AppJsonSerializerContext : JsonSerializerContext
+{
+}

Proxy/Properties/launchSettings.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "",
+      "applicationUrl": "http://localhost:5224",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

Proxy/Proxy.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <Nullable>enable</Nullable>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <InvariantGlobalization>true</InvariantGlobalization>
+        <PublishAot>false</PublishAot>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
+        <PackageReference Include="NetMQ" Version="4.0.1.13" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Models\Models.csproj" />
+    </ItemGroup>
+    
+</Project>

Proxy/Proxy.http

@@ -0,0 +1,11 @@
+@Proxy_HostAddress = http://localhost:5224
+
+GET {{Proxy_HostAddress}}/progress/
+Accept: application/json
+
+###
+
+GET {{Proxy_HostAddress}}/search/asd
+Accept: application/json
+
+###

Proxy/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

Proxy/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}

README.md

@@ -3,7 +3,3 @@
 Rasmus Search Engine
 
 This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
-
-This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture.
-
-I'm just trying to minimize memory usage while maximizing performance.

RSE.sln

@@ -4,7 +4,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -20,9 +24,17 @@ Global
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,42 +1,7 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueueSegment_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4f72364deba8e6142d1f42386f1fd282d2e8c6bca41d389c28da4c3835d90_003FConcurrentQueueSegment_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F85_003Fa8fb60ce_003FConcurrentQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADnsEndPoint_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F3d_003F64a36162_003FDnsEndPoint_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIcmpV6MessageType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F4e_003Fec7b627f_003FIcmpV6MessageType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F6079bfa8f3c0474bb21ade75ce5d010318000_003F15_003F57afed32_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4682fe5857f946a98dcb2cd6f0a403983f200_003F3d_003Fca9070b2_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARawSocketPermissions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003Fc7_003Fb8dcfca9_003FRawSocketPermissions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteConnection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc1d827416891bc9ceaece7826dbd14f2ab1ef304d85bcc1f2187646a67c3c_003FSqliteConnection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F13_003Fbe8f056e_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F90_003F0588ec81_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -0,0 +1,5 @@
+namespace Shared;
+
+public class Class1
+{
+}

Shared/Shared.csproj

@@ -1,14 +1,9 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
-        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
-    <ItemGroup>
-      <Folder Include="CompressedDatabases\" />
-    </ItemGroup>
-
 </Project>

frontend/app.vue

@@ -4,17 +4,17 @@ useHead({
   meta: [
     { name: 'description', content: 'Just a search engine for fun' },
     { name: 'lang', content: 'en'}
-  ]
+  ],
+
+  link: [
+    { rel: 'icon', type: 'image/jpeg', href: '/favicon.jpg' }
+  ],
 })
 </script>
 
 <template>
-  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+  <NuxtLayout>
-    <ul>
-      <li><nuxt-link to="/">Search</nuxt-link></li>
-      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
-    </ul>
-  </div>
-
     <NuxtPage page-key="static" />
+  </NuxtLayout>
+
 </template>

frontend/nuxt.config.ts

@@ -13,9 +13,5 @@ export default defineNuxtConfig({
     },
   },
 
-  nitro: {
-    static: true,
-  },
-
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
 });

frontend/pages/index.vue

@@ -1,15 +1,11 @@
 <script setup lang="ts">
-const value = ref<string | null>(null)
+const route = useRoute()
-
-// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
 </script>
 
-
 <template>
-  <div class="h-screen flex items-center justify-center">
+  <div>
-    <div class="flex space-x-2">
+    <h1>Nuxt Routing set up successfully!</h1>
-      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+    <p>Current route: {{ route.path }}</p>
-      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
+    <a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
-    </div>
   </div>
 </template>

frontend/pages/progress.vue

@@ -21,7 +21,6 @@ const fetchMyData = async () => {
             },
             5 // Cache max age in seconds
         );
-
     } catch (err) {
         error.value = (err as Error).message;
     } finally {
@@ -39,6 +38,7 @@ fetchMyData();
 
 <template>
   <div class="h-screen grid place-items-center">
+    <div v-if="dict">
       <table class="table-auto border-gray-300">
         <thead>
           <tr>
@@ -47,44 +47,15 @@ fetchMyData();
           </tr>
         </thead>
 
-      <tbody v-if="dict">
+        <tbody>
           <tr v-for="d in dict">
             <td class="px-4 py-2">{{d.description}}</td>
             <td class="px-4 py-2">{{d.value}}</td>
           </tr>
         </tbody>
-      <tbody v-else>
-      <tr>
-        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Total filtered</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Total Discarded</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Amount of Ipv4 left</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Discarded db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Filtered db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Unfiltered db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      </tbody>
       </table>
     </div>
+  </div>
 </template>
 
 <style scoped>

frontend/pages/searchResult.vue

@@ -1,8 +0,0 @@
-<script setup lang="ts">
-const route = useRoute();
-  console.log(route.query);
-</script>
-
-<template>
-  
-</template>

frontend/tailwind.config.js

@@ -9,13 +9,7 @@ export default {
     "./error.vue",
   ],
   theme: {
-    extend: {
+    extend: {},
-      width: {
-        '128': '32rem',
-        '192' : '48rem',
-        '256': '64rem',
-      }
-    },
   },
   plugins: [],
 }

frontend/types/global.d.ts

@@ -6,22 +6,12 @@ interface ProgressType {
     discardedDbSize: bigint;
     filteredDbSize: bigint;
     myDbSize: bigint;
-};
+}
 
 interface ProgressDictionary {
     description: string;
     value: string;
-};
+}
-
-interface SearchResult {
-    title: string;
-    description: string;
-    url: string;
-};
-
-interface SearchResults {
-    results: SearchResult[];
-};
 
 type CacheEntry<T> = {
     data: T;

global.json

@@ -1,7 +0,0 @@
-{
-  "sdk": {
-    "version": "9.0.0",
-    "rollForward": "latestMajor",
-    "allowPrerelease": true
-  }
-}