Implement autosave

Reviewed-on: #45

.gitignore

@@ -7,6 +7,7 @@
 **/**/bin/*
 **/obj/*
 **/**/obj/*
+*.gz
 
 # User-specific stuff
 .idea/**/workspace.xml

Analyze/Analyze.csproj

@@ -1,9 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
+        <OutputType>Exe</OutputType>
         <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
+    <ItemGroup>
+      <Folder Include="CompressedDatabases\" />
+    </ItemGroup>
+
 </Project>

Analyze/Program.cs

@@ -0,0 +1,3 @@
+// See https://aka.ms/new-console-template for more information
+
+Console.WriteLine("Hello, World!");

Backend/Backend.csproj

@@ -16,6 +16,7 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/Communication.cs

@@ -87,6 +87,7 @@ public class Communication
                 }
                 else
                 {
+                    // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
                     status.PercentageOfIpv4Scanned = 0.0000000001f;
                 }
                 

Backend/Handler/ContentFilter.cs

@@ -6,10 +6,24 @@ using Models.Model.Backend;
 
 namespace Backend.Handler;
 
+public class Content
+{
+    public int Port1 { get; set; }
+    public int Port2 { get; set; }
+    public Ip Ip { get; set; }
+}
+
+public class ContentThread
+{
+    public int ThreadId { get; set; }
+    public EventWaitHandle EventWaitHandle { get; set; }
+}
+
 public class ContentFilter
 {
     private readonly ConcurrentQueue<Filtered> _queue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<Content?> _contentQueue = new();
     private readonly DbHandler _dbHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
@@ -27,6 +41,7 @@ public class ContentFilter
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
+        
         SetTimeout(3000);
     }
     
@@ -57,6 +72,11 @@ public class ContentFilter
             {
                 if (_stop) break;
                 
+                if (_contentQueue.Count >= 500)
+                {
+                    Thread.Sleep(1000);
+                }
+                
                 Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
                 
                 if (unfiltered.Filtered) continue;
@@ -78,12 +98,12 @@ public class ContentFilter
                     continue;
                 }
 
-                Filtered filtered = GetSiteData(ip);
+                Content content = new();
+                content.Ip = ip;
+                content.Port1 = unfiltered.Port1;
+                content.Port2 = unfiltered.Port2;
                 
-                filtered.Port1 = unfiltered.Port1;
+                _contentQueue.Enqueue(content);
-                filtered.Port2 = unfiltered.Port2;
-                
-                _queue.Enqueue(filtered);
             }
             
             Thread.Sleep(_timeOut);
@@ -92,17 +112,61 @@ public class ContentFilter
         ((EventWaitHandle) obj).Set();
     }
 
-    private Filtered GetSiteData(Ip ip)
+    public WaitHandle[] StartFilterThread(int threads)
     {
-        StartProcess(ip, 80);
+        WaitHandle[] waitHandle = new WaitHandle[threads];
-        StartProcess(ip, 443);
+
+        for (int i = 0; i < threads; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            ContentThread contentThread = new();
+            contentThread.ThreadId = i;
+            contentThread.EventWaitHandle = handle;
+            waitHandle[i] = handle;
+            
+            Thread thread = new(FilterThread!);
+            thread.Start(contentThread);
+        }
+        
+        return waitHandle;
+    }
+
+    private void FilterThread(object obj)
+    {
+        ContentThread thread = (ContentThread) obj;
+        
+        while (!_stop)
+        {
+            if (_contentQueue.IsEmpty)
+            {
+                Thread.Sleep(1000);
+            }
+            
+            _contentQueue.TryDequeue(out Content? content);
+
+            if (content is null)
+            {
+                continue;
+            }
+            
+            Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
+                
+            filtered.Port1 = content.Port1;
+            filtered.Port2 = content.Port2;
+                
+            _queue.Enqueue(filtered);
+        }
+        
+        thread.EventWaitHandle.Set();
+    }
+    
+    private Filtered GetSiteData(Ip ip, int threadId)
+    {
+        StartProcess(ip, 80, threadId);
+        StartProcess(ip, 443, threadId);
         
         string url1 = "";
         string url2 = "";
-        string title1 = "";
-        string title2 = "";
-        string description1 = "";
-        string description2 = "";
         bool robotsTxt1 = false;
         bool robotsTxt2 = false;
         string serverType1 = "";
@@ -128,7 +192,7 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
+            using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
             
             while (streamReader.Peek() != -1)
             {
@@ -161,41 +225,6 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html = "";
-            
-            if (ports[i] == 80)
-            {
-                if (string.IsNullOrWhiteSpace(url1)) continue;
-
-                try
-                {
-                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine(e);
-                }
-            }
-            else
-            {
-                if (string.IsNullOrWhiteSpace(url2)) continue;
-
-                try
-                {
-                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine(e);
-                }
-            }
-            
-            if (string.IsNullOrWhiteSpace(html)) continue;
-            
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
             if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
             if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }
@@ -205,10 +234,6 @@ public class ContentFilter
             Ip = ip,
             Url1 = url1,
             Url2 = url2,
-            Title1 = title1,
-            Title2 = title2,
-            Description1 = description1,
-            Description2 = description2,
             ServerType1 = serverType1,
             ServerType2 = serverType2,
             RobotsTXT1 = robotsTxt1,
@@ -234,7 +259,7 @@ public class ContentFilter
         return siteData;
     }
     
-    private void StartProcess(Ip ip, int port)
+    private void StartProcess(Ip ip, int port, int threadId)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
         
@@ -242,7 +267,7 @@ public class ContentFilter
         proc.StartInfo = new()
         {
             FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
+            Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,

Backend/Handler/IpFilterHandler.cs

@@ -0,0 +1,195 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using Backend.Helper;
+using Models.Model.Backend;
+
+namespace Backend.Handler;
+
+public class IpFilterHandler
+{
+    private readonly ConcurrentQueue<Discarded> _discardedQueue;
+    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private bool _stop;
+    private bool _stopAutoscaledThreads;
+    private int _timeout;
+    
+    public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
+        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
+        ConcurrentQueue<FilterQueueItem> filteredQueue)
+    {
+        _discardedQueue = discardedQueue;
+        _unfilteredQueue = unfilteredQueue;
+        _preFilteredQueue = filteredQueue;
+
+        _timeout = 16;
+    }
+    
+    public List<WaitHandle[]> Start(int threadCount)
+    {
+        WaitHandle[] waitHandle = new WaitHandle[64];
+        
+        int counter = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
+        
+        for (int i = 0; i < threadCount; i++)
+        {
+            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+            
+            if (counter < 64)
+            {
+                waitHandle[counter] = handle;
+                counter++;
+                
+                Thread f = new (Filter!);
+                f.Start(handle);
+                
+                Console.WriteLine($"Filter thread ({i}) started");
+                Thread.Sleep(128);
+                
+                continue;
+            }
+            
+            counter = 0;
+            
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
+        }
+        
+        return waitHandles;
+    }
+
+    public void AutoScaler()
+    {
+        int i = 0;
+        int j = 0;
+        
+        while (!_stop)
+        {
+            if (_preFilteredQueue.Count >= 2000)
+            {
+                if (i == 10)
+                {
+                    _stopAutoscaledThreads = false;
+                    Console.WriteLine("Autoscaler started");
+                    
+                    while (!_stopAutoscaledThreads)
+                    {
+                        if (_preFilteredQueue.Count <= 2000)
+                        {
+                            if (j == 1000)
+                            {
+                                _stopAutoscaledThreads = true;
+                            }
+
+                            j++;
+                            
+                            Thread.Sleep(128);
+                        }
+                        else
+                        {
+                            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
+                            Thread f = new (Filter_AutoScaler!);
+                            f.Start(handle);
+                            Thread.Sleep(16);
+                        }
+                    }
+                }
+
+                i++;
+            }
+            else
+            {
+                i = 0;
+                j = 0;
+            }
+            
+            Thread.Sleep(128);
+        }
+    }
+
+    private void Filter(object obj)
+    {
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(_timeout);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            
+            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+            
+            if (ports is { Item1: 0, Item2: 0 })
+            {
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                continue;
+            }
+            
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    private void Filter_AutoScaler(object obj)
+    {
+        while (!_stopAutoscaledThreads)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(_timeout);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            
+            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+            
+            if (ports is { Item1: 0, Item2: 0 })
+            {
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                continue;
+            }
+            
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+        }
+        
+        ((EventWaitHandle) obj).Set();
+    }
+    
+    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
+    {
+        return new()
+        {
+            Ip = ip,
+            ResponseCode = responseCode
+        };
+    }
+    
+    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    {
+        Unfiltered unfiltered = new()
+        {
+            Ip = ip,
+            Port1 = ports.Item1,
+            Port2 = ports.Item2,
+            Filtered = false
+        };
+        
+        return new()
+        {
+            Unfiltered = unfiltered,
+            Operations = Operations.Insert
+        };
+    }
+
+    public void Stop()
+    {
+        _stop = true;
+    }
+}

Backend/Handler/IpScanner.cs

@@ -18,22 +18,22 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
     private int _timeout;
     
-    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
+    public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
-        )
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue)
     {
         _dbHandler = dbHandler;
+        _preFilteredQueue = preFilteredQueue;
         _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
         
-        SetTimeout(64);
+        SetTimeout(16);
     }
     
     public void SetTimeout(int milliseconds)
@@ -49,22 +49,11 @@ public class IpScanner
             threadsAmount = 256 / threads;
         }
 
-        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle = new WaitHandle[64];
-        WaitHandle[] waitHandle2;
-        
-        if (threads <= 64)
-        {
-            waitHandle1 = new WaitHandle[threads];
-            waitHandle2 = new WaitHandle[threads];
-        }
-        else
-        {
-            waitHandle1 = new WaitHandle[64];
-            waitHandle2 = new WaitHandle[64];
-        }
         
         int counter = 0;
-        int counter2 = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -78,32 +67,27 @@ public class IpScanner
                 Handle = handle
             };
             
-            if (i < 64)
+            if (counter < 64)
             {
-                waitHandle1[counter] = handle;
+                waitHandle[counter] = handle;
                 counter++;
-            }
+                
-            else
+                Thread f = new (Scan!);
-            {
+                f.Start(scanSettings);
-                waitHandle2[counter2] = handle;
+                
-                counter2++;
+                Console.WriteLine($"Scanner thread ({i}) started");
+                Thread.Sleep(128);
+                
+                continue;
             }
             
-            Thread f = new (Scan!);
+            counter = 0;
-            f.Start(scanSettings);
             
-            Console.WriteLine($"Scanner thread ({i}) started");
+            waitHandles.Add(waitHandle);
-            Thread.Sleep(100);
+            
+            waitHandle = new WaitHandle[64];
         }
         
-        List<WaitHandle[]> waitHandles = new();
-        
-        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
-        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
-        
-        waitHandles.Add(waitHandle1);
-        waitHandles.Add(waitHandle2);
-        
         return waitHandles;
     }
     
@@ -124,15 +108,26 @@ public class IpScanner
         
         if (resumeNow is not null)
         {
+            if (resumeNow.Completed)
+            {
+                return;
+            }
+            
             scanSettings.Start = resumeNow.FirstByte;
             scanSettings.End = resumeNow.EndRange;
             secondByte = resumeNow.SecondByte;
             thirdByte = resumeNow.ThirdByte;
             fourthByte = resumeNow.FourthByte;
         }
+        else
+        {
+            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
+        }
         
+        // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
+        int x = 0;
         
         for (int i = scanSettings.Start; i < scanSettings.End; i++)
         {
@@ -153,11 +148,24 @@ public class IpScanner
                     
                     if (_discardedQueue.Count >= 2000)
                     {
-                        Thread.Sleep(500);
+                        Thread.Sleep(1000);
+                    }
+                    
+                    if (_preFilteredQueue.Count >= 2000)
+                    {
+                        Thread.Sleep(1000);
                     }
                     
                     for (int l = fourthByte; l < 256; l++)
                     {
+                        if (x == 75_000)
+                        {
+                            CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
+                            x = 0;
+                        }
+                        
+                        x++;
+                        
                         if (_stop)
                         {
                             resumeObject.FourthByte = l;
@@ -182,6 +190,7 @@ public class IpScanner
                             if (address is not null)
                             {
                                 responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                //Thread.Sleep(4);
                             }
                         }
                         catch
@@ -195,15 +204,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
+                        _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
-                        
-                        if (ports is { Item1: 0, Item2: 0 })
-                        {
-                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
-                            continue;
-                        }
-                        
-                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
                     }
                     
                     if (_stop)
@@ -229,11 +230,39 @@ public class IpScanner
             Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
 
+        if (_stop)
+        {
+            resumeObject.Paused = true;
+        }
+        else
+        {
+            resumeObject.Completed = true;
+        }
+
+        resumeObject.Operation = Operations.Update;
+
         _resumeQueue.Enqueue(resumeObject);
         
         scanSettings.Handle!.Set();
     }
 
+    private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
+    {
+        ScannerResumeObject resumeObject = new();
+        resumeObject.ThreadNumber = threadNumber;
+        resumeObject.StartRange = startRange;
+        resumeObject.EndRange = endRange;
+        resumeObject.FirstByte = firstByte;
+        resumeObject.SecondByte = secondByte;
+        resumeObject.ThirdByte = thirdByte;
+        resumeObject.FourthByte = fourthByte;
+        resumeObject.Paused = paused;
+        resumeObject.Completed = completed;
+        resumeObject.Operation = operation;
+        
+        _resumeQueue.Enqueue(resumeObject);
+    }
+    
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
         return new()
@@ -243,21 +272,15 @@ public class IpScanner
         };
     }
     
-    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
     {
-        Unfiltered unfiltered = new()
+        FilterQueueItem filterQueueItem = new()
         {
             Ip = ip,
-            Port1 = ports.Item1,
+            ResponseCode = responseCode
-            Port2 = ports.Item2,
-            Filtered = false
         };
         
-        return new()
+        return filterQueueItem;
-        {
-            Unfiltered = unfiltered,
-            Operations = Operations.Insert
-        };
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -10,10 +10,12 @@ public class ThreadHandler
     private readonly Communication _communication;
     private readonly IpScanner _ipScanner;
     private readonly ContentFilter _contentFilter;
+    private readonly IpFilterHandler _ipFilterHandler;
     
     private bool _communicationStopped;
     private bool _ipScannerStopped;
     private bool _contentFilterStopped;
+    private bool _ipFilterStopped;
     
     public ThreadHandler(string path)
     {
@@ -21,45 +23,56 @@ public class ThreadHandler
         ConcurrentQueue<Discarded> discardedQueue = new();
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
         ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
         
         _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
+        _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
         _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
         _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue);
     }
     
     public void Start()
     {
         Thread scanner = new(StartScanner);
+        Thread ipFilter = new(StartIpFilter);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
         Thread communication = new(StartCommunicationHandler);
+        Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
+        Thread contentFilterThread = new(StartContentFilterThread);
         
+        ipFilter.Start();
         scanner.Start();
+        ipFilterAutoScaler.Start();
         indexer.Start();
         database.Start();
         discarded.Start();
         filtered.Start();
         resume.Start();
         communication.Start();
+        contentFilterThread.Start();
         
         scanner.Join();
+        ipFilter.Join();
         indexer.Join();
         database.Join();
         discarded.Join();
         filtered.Join();
         resume.Join();
         communication.Join();
+        ipFilterAutoScaler.Join();
+        contentFilterThread.Join();
     }
     
     private void StartScanner()
     {
-        Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
+        Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(128);
+        List<WaitHandle[]> wait = _ipScanner.Start(256);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -84,6 +97,34 @@ public class ThreadHandler
         _contentFilterStopped = true;
     }
 
+    private void StartContentFilterThread()
+    {
+        WaitHandle[] wait = _contentFilter.StartFilterThread(4);
+        
+        WaitHandle.WaitAll(wait);
+    }
+
+    private void StartIpFilterAutoScaler()
+    {
+        _ipFilterHandler.AutoScaler();
+    }
+
+    private void StartIpFilter()
+    {
+        Thread.Sleep(1000);
+        
+        List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
+
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
+        
+        Console.WriteLine("Ip filter finished");
+        
+        _ipFilterStopped = true;
+    }
+    
     private void StartDbHandler()
     {
         _dbHandler.UnfilteredDbHandler();
@@ -130,7 +171,7 @@ public class ThreadHandler
         
         while (stopping)
         {
-            if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
+            if (_ipScannerStopped && _contentFilterStopped && _ipFilterStopped)
             {
                 _dbHandler.Stop();
                 stopping = false;

Backend/Helper/HttpClientHelper.cs

@@ -1,24 +1,47 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
     {
         using HttpClient client = new();
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response;
         
@@ -28,15 +51,48 @@ public static class HttpClientHelper
         }
         catch
         {
-            return "";
+            return ("", "");
         }
         
         if (!response.IsSuccessStatusCode)
         {
-            return "";
+            return ("", "");
         }
 
-        return await response.Content.ReadAsStringAsync();
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
+        }
+        
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
     }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -45,20 +101,35 @@ public static class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {//
+        {
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
@@ -68,4 +139,9 @@ public static class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }

Backend/Helper/TcpClientHelper.cs

@@ -1,20 +1,23 @@
-using System.Net;
 using System.Net.Sockets;
+using Models.Model.Backend;
 
 namespace Backend.Helper;
 
 public static class TcpClientHelper
 {
-    public static (int, int) CheckPort(string ip, params int[] ports)
+    public static (int, int) CheckPort(Ip ip, params int[] ports)
     {
         // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached,
         // and it would just return a result.error, for example.
         for (int i = 0; i < ports.Length; i++) {
+            using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+            socket.SendTimeout = 250;
+            
             try
             {
-                using TcpClient client = new();
+                socket.Connect(ip.ToString(), ports[i]);
-                client.Connect(ip, ports[i]);
+                socket.Close();
-                // If the connection is successful, update the result array with the port number
+                // If the connection is not successful, update the ports array with 0.
             }
             catch
             {

Backend/Scripts/443Header.txt

@@ -0,0 +1,14 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 1.0.0.95:443...
+* Connected to 1.0.0.95 (1.0.0.95) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS alert, handshake failure (552):
+{ [2 bytes data]
+* OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* closing connection #0
+curl: (35) OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure

Backend/Scripts/443Header0.txt

@@ -0,0 +1,76 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.72.3:443...
+* Connected to 192.0.72.3 (192.0.72.3) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2063 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [79 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=files.wordpress.com
+*  start date: Dec 16 09:49:37 2024 GMT
+*  expire date: Mar 16 09:49:36 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E6
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.72.3/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.72.3]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.72.3
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 302 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:34 GMT
+< content-type: text/html; charset=utf-8
+< location: https://developer.wordpress.com
+< vary: Cookie
+< x-nc: MISS hhn 3
+< x-content-type-options: nosniff
+< alt-svc: h3=":443"; ma=86400
+< 
+{ [0 bytes data]
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.72.3 left intact
+HTTP/2 302 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:34 GMT
+content-type: text/html; charset=utf-8
+location: https://developer.wordpress.com
+vary: Cookie
+x-nc: MISS hhn 3
+x-content-type-options: nosniff
+alt-svc: h3=":443"; ma=86400
+

Backend/Scripts/443Header1.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.251:443...
+* Connected to 192.0.66.251 (192.0.66.251) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [79 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.251/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.251]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.251
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:27 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+␍  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.251 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:27 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/443Header2.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.254:443...
+* Connected to 192.0.66.254 (192.0.66.254) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [78 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.254/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.254]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.254
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:28 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+␍  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.254 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:28 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/443Header3.txt

@@ -0,0 +1,70 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.66.253:443...
+* Connected to 192.0.66.253 (192.0.66.253) port 443
+* ALPN: curl offers h2,http/1.1
+} [5 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Client hello (1):
+} [512 bytes data]
+* TLSv1.3 (IN), TLS handshake, Server hello (2):
+{ [122 bytes data]
+* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
+{ [15 bytes data]
+* TLSv1.3 (IN), TLS handshake, Certificate (11):
+{ [2033 bytes data]
+* TLSv1.3 (IN), TLS handshake, CERT verify (15):
+{ [78 bytes data]
+* TLSv1.3 (IN), TLS handshake, Finished (20):
+{ [52 bytes data]
+* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
+} [1 bytes data]
+* TLSv1.3 (OUT), TLS handshake, Finished (20):
+} [52 bytes data]
+* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
+* ALPN: server accepted h2
+* Server certificate:
+*  subject: CN=go-vip.co
+*  start date: Jan 18 19:43:58 2025 GMT
+*  expire date: Apr 18 19:43:57 2025 GMT
+*  issuer: C=US; O=Let's Encrypt; CN=E5
+*  SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
+*   Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
+*   Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
+} [5 bytes data]
+* using HTTP/2
+* [HTTP/2] [1] OPENED stream for https://192.0.66.253/
+* [HTTP/2] [1] [:method: HEAD]
+* [HTTP/2] [1] [:scheme: https]
+* [HTTP/2] [1] [:authority: 192.0.66.253]
+* [HTTP/2] [1] [:path: /]
+* [HTTP/2] [1] [user-agent: curl/8.9.1]
+* [HTTP/2] [1] [accept: */*]
+} [5 bytes data]
+> HEAD / HTTP/2
+> Host: 192.0.66.253
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+{ [5 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
+{ [249 bytes data]
+< HTTP/2 404 
+< server: nginx
+< date: Fri, 07 Feb 2025 22:58:28 GMT
+< content-type: text/html
+< content-length: 146
+< x-rq: hhn2 
+< 
+{ [0 bytes data]
+␍  0   146    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.66.253 left intact
+HTTP/2 404 
+server: nginx
+date: Fri, 07 Feb 2025 22:58:28 GMT
+content-type: text/html
+content-length: 146
+x-rq: hhn2 
+

Backend/Scripts/80Header.txt

@@ -0,0 +1,36 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 1.0.0.95:80...
+* Connected to 1.0.0.95 (1.0.0.95) port 80
+> HEAD / HTTP/1.1
+> Host: 1.0.0.95
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+< HTTP/1.1 403 Forbidden
+< Date: Fri, 07 Feb 2025 21:37:37 GMT
+< Content-Type: text/plain; charset=UTF-8
+< Content-Length: 16
+< Connection: close
+< X-Frame-Options: SAMEORIGIN
+< Referrer-Policy: same-origin
+< Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+< Expires: Thu, 01 Jan 1970 00:00:01 GMT
+< Server: cloudflare
+< CF-RAY: 90e685af4a6b930d-CPH
+< 
+␍  0    16    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* shutting down connection #0
+HTTP/1.1 403 Forbidden
+Date: Fri, 07 Feb 2025 21:37:37 GMT
+Content-Type: text/plain; charset=UTF-8
+Content-Length: 16
+Connection: close
+X-Frame-Options: SAMEORIGIN
+Referrer-Policy: same-origin
+Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0
+Expires: Thu, 01 Jan 1970 00:00:01 GMT
+Server: cloudflare
+CF-RAY: 90e685af4a6b930d-CPH
+

Backend/Scripts/80Header0.txt

@@ -0,0 +1,28 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 192.0.72.3:80...
+* Connected to 192.0.72.3 (192.0.72.3) port 80
+> HEAD / HTTP/1.1
+> Host: 192.0.72.3
+> User-Agent: curl/8.9.1
+> Accept: */*
+> 
+* Request completely sent off
+< HTTP/1.1 301 Moved Permanently
+< Server: nginx
+< Date: Fri, 07 Feb 2025 22:58:33 GMT
+< Content-Type: text/html
+< Content-Length: 162
+< Connection: keep-alive
+< Location: https://192.0.72.3/
+< 
+␍  0   162    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+* Connection #0 to host 192.0.72.3 left intact
+HTTP/1.1 301 Moved Permanently
+Server: nginx
+Date: Fri, 07 Feb 2025 22:58:33 GMT
+Content-Type: text/html
+Content-Length: 162
+Connection: keep-alive
+Location: https://192.0.72.3/
+

Backend/Scripts/80Header1.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.64.106:80...
+␍  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Backend/Scripts/80Header2.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.68.86:80...
+␍  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Backend/Scripts/80Header3.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+␍  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0*   Trying 141.0.68.78:80...
+␍  0     0    0     0    0     0      0      0 --:--:--  0:00:01 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:02 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:03 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:04 --:--:--     0␍  0     0    0     0    0     0      0      0 --:--:--  0:00:05 --:--:--     0

Models/Handler/DbHandler.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using Microsoft.Data.Sqlite;
+using Models.Helper;
 using Models.Model.Backend;
 using Models.Model.External;
 
@@ -14,9 +15,9 @@ public class DbHandler
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
+    private readonly string _compressedConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
@@ -26,13 +27,13 @@ public class DbHandler
     
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
-                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
+                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
-                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
+                                              " Url1, Url2, ServerType1, ServerType2," +
                                               " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
                                               " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
                                               " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
                                               " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
-                                              " @title1, @title2, @description1, @description2, @url1, @url2, " +
+                                              " @url1, @url2, " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
                                               " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
                                               " @robotsTXT1, @robotsTXT2," +
@@ -72,8 +73,12 @@ public class DbHandler
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
+                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
-                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
+
+    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@@ -81,10 +86,13 @@ public class DbHandler
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
-    private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
+    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
+    private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
+    
     private const string ReIndexDatabasesStatement = "REINDEX;";
     
     private const string VacuumDatabasesStatement = "VACUUM;";
@@ -95,6 +103,7 @@ public class DbHandler
     private bool _stop;
     private bool _pause;
     private bool _paused;
+    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
@@ -117,9 +126,9 @@ public class DbHandler
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
-        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
+        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -225,7 +234,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(1000);
+            Thread.Sleep(5000);
         }
         
         return waitHandles;
@@ -236,7 +245,9 @@ public class DbHandler
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        
+        int i = 0;
         
         while (!_stop)
         {
@@ -247,9 +258,28 @@ public class DbHandler
                 continue;
             }
             
+            if (i >= 5_000_000 && !_compressing)
+            {
+                _compressing = true;
+                
+                i = 0;
+                
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                
+                _compressing = false;
+            }
+            
             _discardedQueue.TryDequeue(out Discarded queueItem);
             
             InsertDiscarded(queueItem, connectionString);
+            
+            i++;
         }
         
         discardedDbHandlerSetting.Handle!.Set();
@@ -257,6 +287,30 @@ public class DbHandler
         Console.WriteLine("Discarded DbHandler stopped.");
     }
     
+    private void DropAndCreateDiscarded(int threadNumber)
+    {
+        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
+        
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string dropStatement = "DROP TABLE Discarded;";
+        const string vacuum = "VACUUM;";
+        
+        using SqliteConnection connection = new(databaseName);
+        connection.Open();
+        
+        SqliteCommand command = new(dropStatement, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(vacuum, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(createStatement, connection);
+        command.ExecuteNonQuery();
+        command.Dispose();
+        
+        connection.Close();
+    }
+    
     private void InsertUnfiltered(Unfiltered unfiltered)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
@@ -375,10 +429,6 @@ public class DbHandler
         command.Parameters.AddWithValue("@port2", filtered.Port2);
         command.Parameters.AddWithValue("@url1", filtered.Url1);
         command.Parameters.AddWithValue("@url2", filtered.Url2);
-        command.Parameters.AddWithValue("@title1", filtered.Title1);
-        command.Parameters.AddWithValue("@title2", filtered.Title2);
-        command.Parameters.AddWithValue("@description1", filtered.Description1);
-        command.Parameters.AddWithValue("@description2", filtered.Description2);
         command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
         command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
         command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
@@ -409,16 +459,42 @@ public class DbHandler
     {
         using SqliteConnection connection = new(_resumeConnectionString);
         connection.Open();
+        SqliteCommand command;
 
-        using SqliteCommand command = new(InsertIntoResume, connection);
+        if (resumeObject.Operation == Operations.Insert)
+        {
+            command = new(InsertIntoResume, connection);
+            command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
+            command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
+        }
+        else
+        {
+            command = new(UpdateResumeStatement, connection);
+        }
         
         command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
-        command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
-        command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
         command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
         command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
         command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
         command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
+        command.Parameters.AddWithValue("@paused", resumeObject.Paused);
+        command.Parameters.AddWithValue("@completed", resumeObject.Completed);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+        
+        command.Dispose();
+    }
+
+    private void InsertCompressedDatabase(int threadNumber, long rows)
+    {
+        using SqliteConnection connection = new(_compressedConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
+        
+        command.Parameters.AddWithValue("@dbNumber", threadNumber);
+        command.Parameters.AddWithValue("@rows", rows);
         
         _ = command.ExecuteNonQuery();
         connection.Close();
@@ -518,23 +594,72 @@ public class DbHandler
     {
         long rowId = 0;
 
+        SqliteConnection connection;
+        SqliteCommand command;
+        SqliteDataReader reader;
+        
         for (int i = 0; i < _discardedConnectionStrings.Count; i++)
         {
-            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
+            connection = new(_discardedConnectionStrings[i]);
             connection.Open();
             
-            using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+            command = new(ReadDiscardedSeqIdsStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
+            reader = command.ExecuteReader();
             
             if (!reader.HasRows)
             {
-                return 0;
+                return rowId;
             }
             
             while (reader.Read())
             {
                 rowId += reader.GetInt64(0);
             }
+            
+            connection.Close();
+        }
+        
+        connection = new(_compressedConnectionString);
+        connection.Open();
+        command = new(ReadCompressedDbRowsStatement, connection);
+        reader = command.ExecuteReader();
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        reader.Dispose();
+        
+        return rowId;
+    }
+
+    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    {
+        using SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+        using SqliteDataReader reader = command.ExecuteReader();
+        
+        long rowId = 0;
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
         }
 
         return rowId;
@@ -628,7 +753,7 @@ public class DbHandler
             using SqliteConnection connection = new(_resumeConnectionString);
             connection.Open();
             
-            using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
+            using SqliteCommand command = new(ReadResumeStatement, connection);
             command.Parameters.AddWithValue("@threadNumber", threadNumber);
             
             using SqliteDataReader reader = command.ExecuteReader();
@@ -649,6 +774,8 @@ public class DbHandler
                 resumeObject.SecondByte = reader.GetInt32(4);
                 resumeObject.ThirdByte = reader.GetInt32(5);
                 resumeObject.FourthByte = reader.GetInt32(6);
+                resumeObject.Paused = reader.GetBoolean(7);
+                resumeObject.Completed = reader.GetBoolean(8);
             }
             
             return resumeObject;
@@ -665,20 +792,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(ReIndexDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -704,20 +824,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(VacuumDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -732,8 +845,9 @@ public class DbHandler
         _paused = false;
     }
 
-    private string CreateDiscardedDb(int threadNumber)
+    private (string, string) CreateDiscardedDb(int threadNumber)
     {
+        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
         const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
@@ -746,7 +860,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return databaseName;
+        return (absolutePath, databaseName);
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -0,0 +1,22 @@
+using System.IO.Compression;
+
+namespace Models.Helper;
+
+public static class CompressionHelper
+{
+    public static void CompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
+        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize);
+        originalFileStream.CopyTo(compressor);
+    }
+    
+    public static void DecompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
+        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
+        decompressor.CopyTo(decompressedFileStream);
+    }
+}

Models/Helper/GetDatabasesHelper.cs

@@ -0,0 +1,9 @@
+namespace Models.Helper;
+
+public static class GetDatabasesHelper
+{
+    public static int GetTotalCompressedDatabases(string path)
+    {
+        return Directory.GetFiles(path, "*.gz").Length;
+    }
+}

Models/Model/Backend/FilterQueueItem.cs

@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public struct FilterQueueItem
+{
+    public Ip Ip { get; init; }
+    public int ResponseCode { get; init; }
+}

Models/Model/Backend/Filtered.cs

@@ -3,10 +3,6 @@ namespace Models.Model.Backend;
 public class Filtered
 {
     public Ip Ip { get; set; }
-    public string Title1 { get; set; } = "";
-    public string Title2 { get; set; } = "";
-    public string Description1 { get; set; } = "";
-    public string Description2 { get; set; } = "";
     public string Url1 { get; set; } = "";
     public string Url2 { get; set; } = "";
     public int Port1 { get; set; }

Models/Model/Backend/ScannerResumeObject.cs

@@ -9,4 +9,7 @@ public class ScannerResumeObject
     public int ThirdByte { get; set; }
     public int FourthByte { get; set; }
     public int ThreadNumber { get; set; }
+    public Operations Operation { get; set; }
+    public bool Paused { get; set; }
+    public bool Completed { get; set; }
 }

Proxy/Program.cs

@@ -1,10 +1,12 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using MessagePack;
+using AspNetCoreRateLimit;
-using Microsoft.AspNetCore.Cors.Infrastructure;
+using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
+const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
 
 WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
 
@@ -15,18 +17,29 @@ builder.Services.ConfigureHttpJsonOptions(options =>
 
 builder.Services.AddCors(options =>
 {
-    options.AddPolicy("CorsPolicy", x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader().Build());
+    options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
+
 WebApplication app = builder.Build();
 
-app.UseCors();
+app.UseForwardedHeaders(new()
+{
+    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+});
+
+app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.AllowAnonymous();
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
-progressApi.DisableAntiforgery();
-progressApi.MapGet("/", () =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -40,11 +53,15 @@ progressApi.MapGet("/", () =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });
 
+
 RouteGroupBuilder searchApi = app.MapGroup("/search");
-progressApi.AllowAnonymous();
 searchApi.MapGet("/{term}", (string term) =>
 {
     CommunicationObject communicationObject = new();
@@ -65,7 +82,7 @@ searchApi.MapGet("/{term}", (string term) =>
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]
-[JsonSerializable(typeof(SearchResults))]
+//[JsonSerializable(typeof(SearchResults))]
 [JsonSerializable(typeof(CommunicationObject))]
 internal partial class AppJsonSerializerContext : JsonSerializerContext
 {

Proxy/Proxy.csproj

@@ -9,6 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
         <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
 

Proxy/appsettings.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Warning",
       "Microsoft.AspNetCore": "Warning"
     }
   },

RSE.sln

@@ -8,7 +8,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -32,9 +32,9 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,6 +1,24 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
-	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}

frontend/.gitignore

@@ -0,0 +1,24 @@
+# Nuxt dev/build outputs
+.output
+.data
+.nuxt
+.nitro
+.cache
+dist
+
+# Node dependencies
+node_modules
+
+# Logs
+logs
+*.log
+
+# Misc
+.DS_Store
+.fleet
+.idea
+
+# Local env files
+.env
+.env.*
+!.env.example

frontend/README.md

@@ -0,0 +1,75 @@
+# Nuxt Minimal Starter
+
+Look at the [Nuxt documentation](https://nuxt.com/docs/getting-started/introduction) to learn more.
+
+## Setup
+
+Make sure to install dependencies:
+
+```bash
+# npm
+npm install
+
+# pnpm
+pnpm install
+
+# yarn
+yarn install
+
+# bun
+bun install
+```
+
+## Development Server
+
+Start the development server on `http://localhost:3000`:
+
+```bash
+# npm
+npm run dev
+
+# pnpm
+pnpm dev
+
+# yarn
+yarn dev
+
+# bun
+bun run dev
+```
+
+## Production
+
+Build the application for production:
+
+```bash
+# npm
+npm run build
+
+# pnpm
+pnpm build
+
+# yarn
+yarn build
+
+# bun
+bun run build
+```
+
+Locally preview production build:
+
+```bash
+# npm
+npm run preview
+
+# pnpm
+pnpm preview
+
+# yarn
+yarn preview
+
+# bun
+bun run preview
+```
+
+Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.

frontend/app.vue

@@ -0,0 +1,20 @@
+<script setup lang="ts">
+useHead({
+  title: 'Rasmus Search Engine',
+  meta: [
+    { name: 'description', content: 'Just a search engine for fun' },
+    { name: 'lang', content: 'en'}
+  ]
+})
+</script>
+
+<template>
+  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+    <ul>
+      <li><nuxt-link to="/">Search</nuxt-link></li>
+      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
+    </ul>
+  </div>
+
+  <NuxtPage page-key="static" />
+</template>

frontend/assets/css/main.css

@@ -0,0 +1,3 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;

frontend/nuxt.config.ts

@@ -0,0 +1,21 @@
+// https://nuxt.com/docs/api/configuration/nuxt-config
+export default defineNuxtConfig({
+  compatibilityDate: '2024-12-16',
+  ssr: true,
+
+  devtools: { enabled: true },
+  css: ['@/assets/css/main.css'],
+
+  postcss: {
+    plugins: {
+      tailwindcss: {},
+      autoprefixer: {},
+    },
+  },
+
+  nitro: {
+    static: true,
+  },
+
+  modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
+});

frontend/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "nuxt-app",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "nuxt build",
+    "dev": "nuxt dev",
+    "generate": "nuxt generate",
+    "preview": "nuxt preview",
+    "postinstall": "nuxt prepare"
+  },
+  "dependencies": {
+    "@nuxtjs/partytown": "^1.5.0",
+    "nuxt": "^3.14.1592",
+    "vue": "latest",
+    "vue-router": "latest"
+  },
+  "devDependencies": {
+    "@nuxtjs/tailwindcss": "^6.12.2",
+    "autoprefixer": "^10.4.20",
+    "nuxt-purgecss": "^2.0.0",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.16"
+  }
+}

frontend/pages/index.vue

@@ -0,0 +1,15 @@
+<script setup lang="ts">
+const value = ref<string | null>(null)
+
+// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
+</script>
+
+
+<template>
+  <div class="h-screen flex items-center justify-center">
+    <div class="flex space-x-2">
+      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
+    </div>
+  </div>
+</template>

frontend/pages/progress.vue

@@ -0,0 +1,92 @@
+<script setup lang="ts">
+import { fetchWithCache } from '~/utils/cacheUtil';
+import { progressTypeToDictionary } from '~/utils/convertProgressTypeToDictionary';
+
+const data = ref<ProgressType | null>(null);
+const loading = ref<boolean>(true);
+const error = ref<string | null>(null);
+let dict = ref<ProgressDictionary[] | null>(null);
+
+const fetchMyData = async () => {
+    try {
+        loading.value = true;
+
+        // Use the caching utility
+        data.value = await fetchWithCache<ProgressType>(
+              'Progress',
+              async () => {
+                const response = await fetch('https://proxy.rbwr.dk/progress');
+                if (!response.ok) throw new Error('API fetch failed');
+                return (await response.json()) as ProgressType;
+            },
+            5 // Cache max age in seconds
+        );
+
+    } catch (err) {
+        error.value = (err as Error).message;
+    } finally {
+        loading.value = false;
+
+      if (data.value !== null) {
+        dict.value = progressTypeToDictionary(data.value);
+      }
+    }
+};
+
+fetchMyData();
+
+</script>
+
+<template>
+  <div class="h-screen grid place-items-center">
+    <table class="table-auto border-gray-300">
+      <thead>
+        <tr>
+          <th class="px-4 py-2">Metric</th>
+          <th class="px-4 py-2">Value</th>
+        </tr>
+      </thead>
+
+      <tbody v-if="dict">
+        <tr v-for="d in dict">
+          <td class="px-4 py-2">{{d.description}}</td>
+          <td class="px-4 py-2">{{d.value}}</td>
+        </tr>
+      </tbody>
+      <tbody v-else>
+      <tr>
+        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total filtered</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total Discarded</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Amount of Ipv4 left</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Discarded db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Filtered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Unfiltered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      </tbody>
+    </table>
+  </div>
+</template>
+
+<style scoped>
+
+</style>

frontend/pages/searchResult.vue

@@ -0,0 +1,8 @@
+<script setup lang="ts">
+const route = useRoute();
+  console.log(route.query);
+</script>
+
+<template>
+  
+</template>

frontend/public/robots.txt

@@ -0,0 +1 @@
+

frontend/server/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../.nuxt/tsconfig.server.json"
+}

frontend/tailwind.config.js

@@ -0,0 +1,22 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  content: [
+    "./components/**/*.{js,vue,ts}",
+    "./layouts/**/*.vue",
+    "./pages/**/*.vue",
+    "./plugins/**/*.{js,ts}",
+    "./app.vue",
+    "./error.vue",
+  ],
+  theme: {
+    extend: {
+      width: {
+        '128': '32rem',
+        '192' : '48rem',
+        '256': '64rem',
+      }
+    },
+  },
+  plugins: [],
+}
+

frontend/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  // https://nuxt.com/docs/guide/concepts/typescript
+  "extends": "./.nuxt/tsconfig.json"
+}

frontend/types/global.d.ts

@@ -0,0 +1,29 @@
+interface ProgressType {
+    percentageOfIpv4Scanned: number; // assuming this is a number
+    totalFiltered: bigint;
+    amountOfIpv4Left: bigint;
+    totalDiscarded: bigint;
+    discardedDbSize: bigint;
+    filteredDbSize: bigint;
+    myDbSize: bigint;
+};
+
+interface ProgressDictionary {
+    description: string;
+    value: string;
+};
+
+interface SearchResult {
+    title: string;
+    description: string;
+    url: string;
+};
+
+interface SearchResults {
+    results: SearchResult[];
+};
+
+type CacheEntry<T> = {
+    data: T;
+    timestamp: number;
+};

frontend/utils/cacheUtil.ts

@@ -0,0 +1,34 @@
+export function fetchWithCache<Type>(
+    cacheKey: string,
+    fetchFunction: () => Promise<Type>,
+    maxAge: number = 5 // 30 seconds
+): Promise<Type> {
+    return new Promise(async (resolve, reject) => {
+        try {
+            const cachedData = localStorage.getItem(cacheKey);
+
+            if (cachedData) {
+                const { data, timestamp } = JSON.parse(cachedData) as CacheEntry<Type>;
+                const now = Date.now();
+                const ageInSeconds = (now - timestamp) / 1000;
+
+                // Check if cache is still valid
+                if (ageInSeconds < maxAge) {
+                    return resolve(data);
+                }
+            }
+
+            // Cache is missing or expired, fetch fresh data
+            const freshData = await fetchFunction();
+            const cacheEntry: CacheEntry<Type> = {
+                data: freshData,
+                timestamp: Date.now(),
+            };
+
+            localStorage.setItem(cacheKey, JSON.stringify(cacheEntry));
+            resolve(freshData);
+        } catch (error) {
+            reject(error);
+        }
+    });
+}

frontend/utils/convertProgressTypeToDictionary.ts

@@ -0,0 +1,11 @@
+export function progressTypeToDictionary(progress: ProgressType) {
+    return [
+    { description: "Percentage of Ipv4 scanned", value: progress.percentageOfIpv4Scanned.toString() },
+    { description: "Total filtered", value: progress.totalFiltered.toString() },
+    { description: "Total Discarded", value: progress.totalDiscarded.toString() },
+    { description: "Amount of Ipv4 left", value: progress.amountOfIpv4Left.toString() },
+    { description: "Discarded db size", value: progress.discardedDbSize.toString() },
+    { description: "Filtered db size", value: progress.filteredDbSize.toString() },
+    { description: "Unfiltered db size", value: progress.myDbSize.toString() },
+    ]
+}