Merge pull request 'SearchFunction' (#45) from SearchFunction into main

Reviewed-on: #45

Backend/Backend.csproj

@@ -16,6 +16,7 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/Communication.cs

@@ -87,6 +87,7 @@ public class Communication
                 }
                 else
                 {
+                    // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
                     status.PercentageOfIpv4Scanned = 0.0000000001f;
                 }
                 
@@ -96,7 +97,7 @@ public class Communication
                 status.MyDbSize = databaseSizes.MyDbSize;
                 status.FilteredDbSize = databaseSizes.FilteredDbSize;
                 status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
-
+                
                 byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
                 
                 rep.SendFrame(serializedResult);

Backend/Handler/ContentFilter.cs

@@ -161,41 +161,37 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html = "";
-            
             if (ports[i] == 80)
             {
                 if (string.IsNullOrWhiteSpace(url1)) continue;
-
+                
                 try
                 {
-                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
+                    title1 = temp.Item1;
+                    description1 = temp.Item2;
                 }
-                catch (Exception e)
+                catch
                 {
-                    Console.WriteLine(e);
+                    //
                 }
             }
             else
             {
                 if (string.IsNullOrWhiteSpace(url2)) continue;
-
+                
                 try
                 {
-                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                    (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
+                    title2 = temp.Item1;
+                    description2 = temp.Item2;
                 }
-                catch (Exception e)
+                catch
                 {
-                    Console.WriteLine(e);
+                    //
                 }
             }
             
-            if (string.IsNullOrWhiteSpace(html)) continue;
-            
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
-            if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
-            if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
             if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
             if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }

Backend/Handler/IpScanner.cs

@@ -32,7 +32,7 @@ public class IpScanner
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
-
+        
         SetTimeout(64);
     }
     
@@ -49,22 +49,11 @@ public class IpScanner
             threadsAmount = 256 / threads;
         }
 
-        WaitHandle[] waitHandle1;
-        WaitHandle[] waitHandle2;
-        
-        if (threads <= 64)
-        {
-            waitHandle1 = new WaitHandle[threads];
-            waitHandle2 = new WaitHandle[threads];
-        }
-        else
-        {
-            waitHandle1 = new WaitHandle[64];
-            waitHandle2 = new WaitHandle[64];
-        }
+        WaitHandle[] waitHandle = new WaitHandle[64];
         
         int counter = 0;
-        int counter2 = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -77,33 +66,28 @@ public class IpScanner
                 ThreadNumber = i,
                 Handle = handle
             };
-
-            if (i < 64)
+            
+            if (counter < 64)
             {
-                waitHandle1[counter] = handle;
+                waitHandle[counter] = handle;
                 counter++;
-            }
-            else
-            {
-                waitHandle2[counter2] = handle;
-                counter2++;
+                
+                Thread f = new (Scan!);
+                f.Start(scanSettings);
+                
+                Console.WriteLine($"Scanner thread ({i}) started");
+                Thread.Sleep(100);
+                
+                continue;
             }
             
-            Thread f = new (Scan!);
-            f.Start(scanSettings);
+            counter = 0;
             
-            Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(100);
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
         }
         
-        List<WaitHandle[]> waitHandles = new();
-        
-        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
-        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
-        
-        waitHandles.Add(waitHandle1);
-        waitHandles.Add(waitHandle2);
-        
         return waitHandles;
     }
     
@@ -131,6 +115,7 @@ public class IpScanner
             fourthByte = resumeNow.FourthByte;
         }
         
+        // Empty buffer so we use the lowest abstracted ping.Send() method.
         byte[] buf = [];
         using Ping ping = new();
         
@@ -181,7 +166,8 @@ public class IpScanner
                             _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             if (address is not null)
                             {
-                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
+                                Thread.Sleep(_timeout);
                             }
                         }
                         catch

Backend/Handler/ThreadHandler.cs

@@ -59,7 +59,7 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(128);
+        List<WaitHandle[]> wait = _ipScanner.Start(256);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -101,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(4);
+        WaitHandle[] wait = _dbHandler.Start(5);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -1,24 +1,47 @@
-using System.Diagnostics;
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
-    public static async Task<string> GetHtml(string url, int port)
+    // Reddit, for example, will block the GET request if you don't have a user agent.
+    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
+    
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
     {
         using HttpClient client = new();
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response;
         
@@ -28,15 +51,48 @@ public static class HttpClientHelper
         }
         catch
         {
-            return "";
+            return ("", "");
         }
         
         if (!response.IsSuccessStatusCode)
         {
-            return "";
+            return ("", "");
+        }
+
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
         }
         
-        return await response.Content.ReadAsStringAsync();
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
     }
     
     public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -45,20 +101,35 @@ public static class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(1);
+        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
         try
-        {//
+        {
             response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
@@ -68,4 +139,9 @@ public static class HttpClientHelper
         
         return response is not null && response.IsSuccessStatusCode;
     }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }

Models/Handler/DbHandler.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using Microsoft.Data.Sqlite;
+using Models.Helper;
 using Models.Model.Backend;
 using Models.Model.External;
 
@@ -14,9 +15,9 @@ public class DbHandler
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
+    private readonly string _compressedConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
@@ -74,6 +75,10 @@ public class DbHandler
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
                                             " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
                                             " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+
+    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@@ -82,6 +87,7 @@ public class DbHandler
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
@@ -95,6 +101,7 @@ public class DbHandler
     private bool _stop;
     private bool _pause;
     private bool _paused;
+    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
@@ -117,9 +124,9 @@ public class DbHandler
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
-        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
+        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -213,7 +220,7 @@ public class DbHandler
         for (int i = 0; i < threads; i++)
         {
             EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-
+            
             DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
             {
                 Handle = handle,
@@ -225,7 +232,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(1000);
+            Thread.Sleep(5000);
         }
         
         return waitHandles;
@@ -236,7 +243,9 @@ public class DbHandler
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        
+        int i = 0;
         
         while (!_stop)
         {
@@ -247,9 +256,28 @@ public class DbHandler
                 continue;
             }
             
+            if (i >= 50_000_000 && !_compressing)
+            {
+                _compressing = true;
+                
+                i = 0;
+                
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                
+                _compressing = false;
+            }
+            
             _discardedQueue.TryDequeue(out Discarded queueItem);
             
             InsertDiscarded(queueItem, connectionString);
+            
+            i++;
         }
         
         discardedDbHandlerSetting.Handle!.Set();
@@ -257,6 +285,30 @@ public class DbHandler
         Console.WriteLine("Discarded DbHandler stopped.");
     }
     
+    private void DropAndCreateDiscarded(int threadNumber)
+    {
+        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
+        
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string dropStatement = "DROP TABLE Discarded;";
+        const string vacuum = "VACUUM;";
+        
+        using SqliteConnection connection = new(databaseName);
+        connection.Open();
+        
+        SqliteCommand command = new(dropStatement, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(vacuum, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(createStatement, connection);
+        command.ExecuteNonQuery();
+        command.Dispose();
+        
+        connection.Close();
+    }
+    
     private void InsertUnfiltered(Unfiltered unfiltered)
     {
         using SqliteConnection connection = new(_unfilteredConnectionString);
@@ -423,6 +475,20 @@ public class DbHandler
         _ = command.ExecuteNonQuery();
         connection.Close();
     }
+
+    private void InsertCompressedDatabase(int threadNumber, long rows)
+    {
+        using SqliteConnection connection = new(_compressedConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
+        
+        command.Parameters.AddWithValue("@dbNumber", threadNumber);
+        command.Parameters.AddWithValue("@rows", rows);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+    }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
@@ -517,26 +583,75 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
+
+        SqliteConnection connection;
+        SqliteCommand command;
+        SqliteDataReader reader;
         
         for (int i = 0; i < _discardedConnectionStrings.Count; i++)
         {
-            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
+            connection = new(_discardedConnectionStrings[i]);
             connection.Open();
             
-            using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
+            command = new(ReadDiscardedSeqIdsStatement, connection);
+            reader = command.ExecuteReader();
             
             if (!reader.HasRows)
             {
-                return 0;
+                return rowId;
             }
             
             while (reader.Read())
             {
                 rowId += reader.GetInt64(0);
             }
+            
+            connection.Close();
         }
         
+        connection = new(_compressedConnectionString);
+        connection.Open();
+        command = new(ReadCompressedDbRowsStatement, connection);
+        reader = command.ExecuteReader();
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        reader.Dispose();
+        
+        return rowId;
+    }
+
+    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    {
+        using SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+        using SqliteDataReader reader = command.ExecuteReader();
+        
+        long rowId = 0;
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+
         return rowId;
     }
 
@@ -665,20 +780,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(ReIndexDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -704,20 +812,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(VacuumDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -732,8 +833,9 @@ public class DbHandler
         _paused = false;
     }
 
-    private string CreateDiscardedDb(int threadNumber)
+    private (string, string) CreateDiscardedDb(int threadNumber)
     {
+        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
         const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
@@ -746,7 +848,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return databaseName;
+        return (absolutePath, databaseName);
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -0,0 +1,22 @@
+using System.IO.Compression;
+
+namespace Models.Helper;
+
+public static class CompressionHelper
+{
+    public static void CompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
+        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
+        originalFileStream.CopyTo(compressor);
+    }
+    
+    public static void DecompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
+        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
+        decompressor.CopyTo(decompressedFileStream);
+    }
+}

Models/Helper/GetDatabasesHelper.cs

@@ -0,0 +1,9 @@
+namespace Models.Helper;
+
+public static class GetDatabasesHelper
+{
+    public static int GetTotalCompressedDatabases(string path)
+    {
+        return Directory.GetFiles(path, "*.gz").Length;
+    }
+}

Proxy/Program.cs

@@ -2,6 +2,7 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using AspNetCoreRateLimit;
 using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.Extensions.Caching.Memory;
 using Models.Model.External;
 using NetMQ;
 using NetMQ.Sockets;
@@ -19,23 +20,7 @@ builder.Services.AddCors(options =>
     options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
 });
 
-// Add memory cache and rate limiting services
-builder.Services.AddMemoryCache();
-builder.Services.Configure<IpRateLimitOptions>(options =>
-{
-    options.GeneralRules =
-    [
-        new()
-        {
-            Endpoint = "*", // Apply to all endpoints
-            Period = "10s", // Rate limiting window of 10 second
-            Limit = 5 // Maximum 5 requests per 10 seconds
-        }
-    ];
-});
-
-builder.Services.AddInMemoryRateLimiting();
-builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
+builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
 
 WebApplication app = builder.Build();
 
@@ -44,14 +29,17 @@ app.UseForwardedHeaders(new()
     ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
 });
 
-
-app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
-
 app.UseCors(myAllowSpecificOrigins);
 
 RouteGroupBuilder progressApi = app.MapGroup("/progress");
-progressApi.MapGet("/", () =>
+progressApi.MapGet("/", (IMemoryCache memoryCache) =>
 {
+    const string cacheKey = "progress_status";
+    if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
+    {
+        return scanningStatus;
+    }
+    
     CommunicationObject communicationObject = new()
     {
         Command = CommunicationCommand.GetScanningProgress
@@ -65,10 +53,14 @@ progressApi.MapGet("/", () =>
     byte[] msg = client.ReceiveFrameBytes();
     client.Close();
     
-    return JsonSerializer.Deserialize<ScanningStatus>(msg);
+    scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
+    
+    memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
+    
+    return scanningStatus;
 });
 
-/*
+
 RouteGroupBuilder searchApi = app.MapGroup("/search");
 searchApi.MapGet("/{term}", (string term) =>
 {
@@ -86,7 +78,7 @@ searchApi.MapGet("/{term}", (string term) =>
     
     return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-*/
+
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]

RSE.sln

@@ -8,8 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -32,9 +30,5 @@ Global
 		{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

RSE.sln.DotSettings.user

@@ -1,7 +1,15 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

Shared/Class1.cs

@@ -1,5 +0,0 @@
-namespace Shared;
-
-public class Class1
-{
-}

Shared/Shared.csproj

@@ -1,9 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-</Project>

frontend/app.vue

@@ -4,17 +4,17 @@ useHead({
   meta: [
     { name: 'description', content: 'Just a search engine for fun' },
     { name: 'lang', content: 'en'}
-  ],
-
-  link: [
-    { rel: 'icon', type: 'image/jpeg', href: '/favicon.jpg' }
-  ],
+  ]
 })
 </script>
 
 <template>
-  <NuxtLayout>
-    <NuxtPage page-key="static" />
-  </NuxtLayout>
+  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+    <ul>
+      <li><nuxt-link to="/">Search</nuxt-link></li>
+      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
+    </ul>
+  </div>
 
+  <NuxtPage page-key="static" />
 </template>

frontend/nuxt.config.ts

@@ -13,5 +13,9 @@ export default defineNuxtConfig({
     },
   },
 
+  nitro: {
+    static: true,
+  },
+
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
 });

frontend/pages/index.vue

@@ -1,11 +1,15 @@
 <script setup lang="ts">
-const route = useRoute()
+const value = ref<string | null>(null)
+
+// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
 </script>
 
+
 <template>
-  <div>
-    <h1>Nuxt Routing set up successfully!</h1>
-    <p>Current route: {{ route.path }}</p>
-    <a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
+  <div class="h-screen flex items-center justify-center">
+    <div class="flex space-x-2">
+      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
+    </div>
   </div>
 </template>

frontend/pages/progress.vue

@@ -21,6 +21,7 @@ const fetchMyData = async () => {
             },
             5 // Cache max age in seconds
         );
+
     } catch (err) {
         error.value = (err as Error).message;
     } finally {
@@ -38,23 +39,51 @@ fetchMyData();
 
 <template>
   <div class="h-screen grid place-items-center">
-    <div v-if="dict">
-      <table class="table-auto border-gray-300">
-        <thead>
-          <tr>
-            <th class="px-4 py-2">Metric</th>
-            <th class="px-4 py-2">Value</th>
-          </tr>
-        </thead>
+    <table class="table-auto border-gray-300">
+      <thead>
+        <tr>
+          <th class="px-4 py-2">Metric</th>
+          <th class="px-4 py-2">Value</th>
+        </tr>
+      </thead>
 
-        <tbody>
-          <tr v-for="d in dict">
-            <td class="px-4 py-2">{{d.description}}</td>
-            <td class="px-4 py-2">{{d.value}}</td>
-          </tr>
-        </tbody>
-      </table>
-    </div>
+      <tbody v-if="dict">
+        <tr v-for="d in dict">
+          <td class="px-4 py-2">{{d.description}}</td>
+          <td class="px-4 py-2">{{d.value}}</td>
+        </tr>
+      </tbody>
+      <tbody v-else>
+      <tr>
+        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total filtered</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total Discarded</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Amount of Ipv4 left</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Discarded db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Filtered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Unfiltered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      </tbody>
+    </table>
   </div>
 </template>
 

frontend/pages/searchResult.vue

@@ -0,0 +1,8 @@
+<script setup lang="ts">
+const route = useRoute();
+  console.log(route.query);
+</script>
+
+<template>
+  
+</template>

frontend/tailwind.config.js

@@ -9,7 +9,13 @@ export default {
     "./error.vue",
   ],
   theme: {
-    extend: {},
+    extend: {
+      width: {
+        '128': '32rem',
+        '192' : '48rem',
+        '256': '64rem',
+      }
+    },
   },
   plugins: [],
 }

frontend/types/global.d.ts

@@ -6,12 +6,22 @@ interface ProgressType {
     discardedDbSize: bigint;
     filteredDbSize: bigint;
     myDbSize: bigint;
-}
+};
 
 interface ProgressDictionary {
     description: string;
     value: string;
-}
+};
+
+interface SearchResult {
+    title: string;
+    description: string;
+    url: string;
+};
+
+interface SearchResults {
+    results: SearchResult[];
+};
 
 type CacheEntry<T> = {
     data: T;