Merge branch 'main' into Analyzer

Reviewed-on: #46

Backend/Handler/IpFilterHandler.cs

@@ -1,7 +1,5 @@
 using System.Collections.Concurrent;
-using System.Diagnostics;
 using Backend.Helper;
-using Models.Handler;
 using Models.Model.Backend;
 
 namespace Backend.Handler;
@@ -10,23 +8,20 @@ public class IpFilterHandler
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
-    private DbHandler _dbHandler;
     private ThreadHandler _threadHandler;
     private bool _stop;
-    private bool _fillerStop;
+    private bool _done;
     private bool _stopAutoscaledThreads;
     private int _timeout;
     
     public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        ConcurrentQueue<Ip> preFilteredQueue, ThreadHandler threadHandler)
-        DbHandler dbHandler, ThreadHandler threadHandler)
     {
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _preFilteredQueue = preFilteredQueue;
-        _dbHandler = dbHandler;
         _threadHandler = threadHandler;
 
         _timeout = 16;
@@ -67,64 +62,15 @@ public class IpFilterHandler
         
         return waitHandles;
     }
-
+    
-    public void AutoScaler()
-    {
-        int i = 0;
-        int j = 0;
-        
-        while (!_stop)
-        {
-            if (_preFilteredQueue.Count >= 2000)
-            {
-                if (i == 10)
-                {
-                    _stopAutoscaledThreads = false;
-                    Console.WriteLine("Autoscaler started");
-                    
-                    while (!_stopAutoscaledThreads)
-                    {
-                        if (_preFilteredQueue.Count <= 2000)
-                        {
-                            if (j == 1000)
-                            {
-                                _stopAutoscaledThreads = true;
-                            }
-
-                            j++;
-                            
-                            Thread.Sleep(128);
-                        }
-                        else
-                        {
-                            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-                            Thread f = new (Filter_AutoScaler!);
-                            f.Start(handle);
-                            Thread.Sleep(16);
-                        }
-                    }
-                }
-
-                i++;
-            }
-            else
-            {
-                i = 0;
-                j = 0;
-            }
-            
-            Thread.Sleep(128);
-        }
-    }
-
     private void Filter(object obj)
     {
         int counter = 0;
         while (!_stop)
         {
-            if (_preFilteredQueue.IsEmpty && _fillerStop)
+            if (_preFilteredQueue.IsEmpty)
             {
-                if (counter == 100)
+                if (counter == 30_000)
                 {
                     _threadHandler.Stop();
                     _stop = true;
@@ -132,63 +78,21 @@ public class IpFilterHandler
                 
                 counter++;
                 Thread.Sleep(128);
-            }
+                
-            
-            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
-            
-            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
-        
-            if (ports is { Item1: 0, Item2: 0 })
-            {
-                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
-                continue;
-            }
-        
-            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
-        }
-        
-        ((EventWaitHandle) obj).Set();
-    }
-
-    public void FillFilterQueue()
-    {
-        Console.WriteLine("Fill FilterQueue started.");
-        while (!_stop)
-        {
-            if (_preFilteredQueue.Count > 500) continue;
-            
-            if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
-            {
-                _preFilteredQueue.Enqueue(item);
-            }
-            else
-            {
-                _fillerStop = true;
-            }
-        }
-    }
-    
-    private void Filter_AutoScaler(object obj)
-    {
-        while (!_stopAutoscaledThreads)
-        {
-            if (_preFilteredQueue.IsEmpty)
-            {
-                Thread.Sleep(_timeout);
                 continue;
             }
             
-            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            _preFilteredQueue.TryDequeue(out Ip item);
             
-            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+            (int, int) ports = TcpClientHelper.CheckPort(item, 80, 443);
             
             if (ports is { Item1: 0, Item2: 0 })
             {
-                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item, item.ResponseCode));
                 continue;
             }
             
-            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item, ports));
         }
         
         ((EventWaitHandle) obj).Set();

Backend/Handler/IpScanner.cs

@@ -22,7 +22,7 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
@@ -30,7 +30,7 @@ public class IpScanner
     
     public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue)
+        ConcurrentQueue<Ip> preFilteredQueue)
     {
         _dbHandler = dbHandler;
         _preFilteredQueue = preFilteredQueue;
@@ -145,12 +145,12 @@ public class IpScanner
                     if (i == 192 && k == 2) continue;
                     if (i == 192 && j == 88 && k == 99) continue;
                     
-                    if (_discardedQueue.Count >= 2000)
+                    if (_discardedQueue.Count >= 20_000)
                     {
                         Thread.Sleep(1000);
                     }
                     
-                    if (_preFilteredQueue.Count >= 2000)
+                    if (_preFilteredQueue.Count >= 20_000)
                     {
                         Thread.Sleep(1000);
                     }
@@ -170,7 +170,9 @@ public class IpScanner
                             resumeObject.FourthByte = l;
                             break;
                         }
-
+                        
+                        
+                        
                         Ip ip = new()
                         {
                             Ip1 = i,
@@ -186,19 +188,21 @@ public class IpScanner
                             // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
                             //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
-
+                            
-                            if (i % 2 == 0)
+                            responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
+                            
+                            /*if (l % 2 == 0)
                             {
                                 responseCode = IPStatus.Success;
                             }
                             else
                             {
                                 responseCode = IPStatus.TimedOut;
-                            }
+                            }*/
                             
-                            //CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
+                            //responseCode = IPStatus.TimedOut;
-                            Thread.Sleep(16);
                             
+                            //Thread.Sleep(0);
                         }
                         catch
                         {
@@ -233,10 +237,10 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            
-            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
+        Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped.");
+        
         if (_stop)
         {
             resumeObject.Paused = true;
@@ -279,15 +283,11 @@ public class IpScanner
         };
     }
     
-    private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
+    private static Ip CreateUnfilteredQueueItem(Ip ip, int responseCode)
     {
-        FilterQueueItem filterQueueItem = new()
+        ip.ResponseCode = responseCode;
-        {
-            Ip = ip,
-            ResponseCode = responseCode
-        };
         
-        return filterQueueItem;
+        return ip;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -15,22 +15,22 @@ public class ThreadHandler
     private bool _contentFilterStopped;
     private bool _ipFilterStopped;
 
-    private bool _stage1;
+    private bool _stage1 = false;
     private bool _stage2 = true;
-    private bool _stage3;
+    private bool _stage3 = false;
     
     ConcurrentQueue<Filtered> filteredQueue = new();
     ConcurrentQueue<Discarded> discardedQueue = new();
     ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
     ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
-    ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
+    ConcurrentQueue<Ip> preFilteredQueue = new();
     
     public ThreadHandler(string path)
     {
         _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
         _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
         _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
-        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, this);
     }
     
     public void Start()
@@ -42,12 +42,13 @@ public class ThreadHandler
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
-        Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
         Thread contentFilterThread = new(StartContentFilterThread);
         Thread prefilterDb = new(StartPreFilterDbHandler);
         Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
         //Thread check = new(CheckQueue);
-
+        
+        //check.Start();
+        
         if (_stage1)
         {
             discarded.Start(); // de-queues from discardedQueue
@@ -55,27 +56,28 @@ public class ThreadHandler
             scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
             resume.Start(); // de-queues from resumeQueue
             
+            scanner.Join();
+            Stop();
+            
             discarded.Join();
             prefilterDb.Join();
-            scanner.Join();
             resume.Join();
         }
-
+        
         if (_stage2)
         {
             database.Start(); // de-queues from unfilteredQueue
             discarded.Start(); // de-queues from discardedQueue
             ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
-            ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
             fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
-            
+
+            ipFilter.Join();
+            Stop();
             database.Join();
             discarded.Join();
-            ipFilter.Join();
-            ipFilterAutoScaler.Join();
             fillIpFilterQueue.Join();
         }
-
+        
         if (_stage3)
         {
             filtered.Start(); // de-queues from filteredQueue
@@ -89,7 +91,7 @@ public class ThreadHandler
             indexer.Join();
         }
     }
-
+    
     private void CheckQueue()
     {
         while (true)
@@ -103,13 +105,14 @@ public class ThreadHandler
             Thread.Sleep(5);
         }
     }
+
     
     private void StartScanner()
     {
         Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
         
         List<WaitHandle[]> wait = _ipScanner.Start(256);
-
+        
         for (int i = 0; i < wait.Count; i++)
         {
             WaitHandle.WaitAll(wait[i]);
@@ -133,26 +136,21 @@ public class ThreadHandler
 
     private void StartContentFilterThread()
     {
-        WaitHandle[] wait = _contentFilter.StartFilterThread(64);
+        WaitHandle[] wait = _contentFilter.StartFilterThread(8);
         
         WaitHandle.WaitAll(wait);
     }
 
-    private void StartIpFilterAutoScaler()
-    {
-        _ipFilterHandler.AutoScaler();
-    }
-
     private void StartFillIpFilterQueue()
     {
-        _ipFilterHandler.FillFilterQueue();
+        _dbHandler.GetPreFilterQueueItem();
     }
 
     private void StartIpFilter()
     {
         Thread.Sleep(1000);
         
-        List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
+        List<WaitHandle[]> wait = _ipFilterHandler.Start(1024);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -205,6 +203,7 @@ public class ThreadHandler
         
         Console.WriteLine("Stopping Super Extra...");
         _dbHandler.Stop();
+        
         Console.WriteLine("Stopped.");
     }
 }

Models/Handler/DbHandler.cs

@@ -1,9 +1,10 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
 using Microsoft.Data.Sqlite;
 using Models.Helper;
 using Models.Model.Backend;
-using Models.Model.External;
 
 namespace Models.Handler;
 
@@ -13,14 +14,13 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
     private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
     private readonly string _compressedConnectionString;
-    private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -75,8 +75,8 @@ public class DbHandler
     
     private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
+                                               " INSERT INTO Discarded (PackedData)" +
-                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+                                               " VALUES (@packedData)";
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -96,7 +96,7 @@ public class DbHandler
     private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
     private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
-    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
+    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Id == @id ORDER BY Id ASC LIMIT 1;";
     private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
@@ -107,7 +107,6 @@ public class DbHandler
     
     private const string VacuumDatabasesStatement = "VACUUM;";
     
-    private readonly object _readFilteredLock = new();
     private readonly object _readAndDeleteResumeLock = new();
     
     private bool _stop;
@@ -124,7 +123,7 @@ public class DbHandler
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
         ConcurrentQueue<ScannerResumeObject> resumeQueue, 
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        ConcurrentQueue<Ip> preFilteredQueue,
         string basePath)
     {
         _filteredQueue = filteredQueue;
@@ -217,7 +216,7 @@ public class DbHandler
                 continue;
             }
             
-            _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
+            _preFilteredQueue.TryDequeue(out Ip queueItem);
             
             InsertPrefiltered(queueItem);
         }
@@ -263,7 +262,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(5000);
+            Thread.Sleep(150);
         }
         
         return waitHandles;
@@ -278,7 +277,16 @@ public class DbHandler
         
         int i = 0;
         
-        while (!_stop)
+        SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        SqliteCommand command = new(InsertIntoDiscarded, connection);
+        
+        StringBuilder stringBuilder = new();
+        
+        bool running = true;
+        
+        while (!_stop && running)
         {
             if (_discardedQueue.IsEmpty || _pause)
             {
@@ -286,31 +294,46 @@ public class DbHandler
                 _paused = true;
                 continue;
             }
-            
+
-            if (i >= 5_000_000 && !_compressing)
+            if (_stop && _discardedQueue.IsEmpty)
             {
-                _compressing = true;
+                running = false;
-                
-                i = 0;
-                
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
-                
-                _compressing = false;
             }
             
-            _discardedQueue.TryDequeue(out Discarded queueItem);
+            if (i == 2048)
+            {
+                command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+                _ = command.ExecuteNonQuery();
+                
+                // Re-use the command object.
+                command.Parameters.Clear();
+                
+                stringBuilder.Clear();
+                
+                i = 0;
+            }
             
-            InsertDiscarded(queueItem, connectionString);
+            if (_discardedQueue.TryDequeue(out Discarded queueItem))
+            {
+                stringBuilder.Append(queueItem.Ip.PackIp());
+                stringBuilder.Append(':');
+                stringBuilder.Append(queueItem.ResponseCode);
+                stringBuilder.Append(',');
+            }
             
             i++;
         }
         
+        if (stringBuilder.Length != 0)
+        {
+            command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+            _ = command.ExecuteNonQuery();
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        
         discardedDbHandlerSetting.Handle!.Set();
         
         Console.WriteLine("Discarded DbHandler stopped.");
@@ -359,23 +382,6 @@ public class DbHandler
         connection.Close();
     }
     
-    private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
-    {
-        using SqliteConnection connection = new(dbConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoDiscarded, connection);
-        
-        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
-        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
-        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
-        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
-        command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
-        _ = command.ExecuteNonQuery();
-        
-        connection.Close();
-    }
-    
     private void InsertFiltered(Filtered filtered)
     {
         using SqliteConnection connection = new(_filteredConnectionString);
@@ -515,35 +521,22 @@ public class DbHandler
         command.Dispose();
     }
 
-    private void InsertCompressedDatabase(int threadNumber, long rows)
+    private void InsertPrefiltered(Ip filterQueueItem)
-    {
-        using SqliteConnection connection = new(_compressedConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
-        
-        command.Parameters.AddWithValue("@dbNumber", threadNumber);
-        command.Parameters.AddWithValue("@rows", rows);
-        
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-    }
-
-    private void InsertPrefiltered(FilterQueueItem filterQueueItem)
     {
         using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
         
         using SqliteCommand command = new(InsertPreFilteredStatement, connection);
         
-        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip1);
-        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip2);
-        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip3);
-        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
+        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip4);
         command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
         command.Parameters.AddWithValue("@filtered", 0);
         
         _ = command.ExecuteNonQuery();
+        
         connection.Close();
     }
     
@@ -614,42 +607,62 @@ public class DbHandler
         return ids;
     }
     
-    public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
+    public void GetPreFilterQueueItem()
     {
         using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
+
+        const string temp = "SELECT seq FROM sqlite_sequence;";
         
         SqliteCommand command = new(ReadPreFilteredStatement, connection);
-        using SqliteDataReader reader = command.ExecuteReader();
+        SqliteCommand command2 = new(UpdatePreFilteredStatement, connection);
-        
+        SqliteCommand command3 = new(temp, connection);
-        filterQueueItem = new();
+        SqliteDataReader reader = command3.ExecuteReader();
-        Ip ip = new();
+
-        long id = 0;
+        long count = 0;
-        
-        if (!reader.HasRows)
-        {
-            return false;
-        }
         
         while (reader.Read())
         {
-            ip.Ip1 = reader.GetInt32(0);
+            count = reader.GetInt64(0);
-            ip.Ip2 = reader.GetInt32(1);
-            ip.Ip3 = reader.GetInt32(2);
-            ip.Ip4 = reader.GetInt32(3);
-            filterQueueItem.ResponseCode = reader.GetInt32(4);
-            id = reader.GetInt64(5);
         }
         
-        filterQueueItem.Ip = ip;
+        reader.Close();
         
-        command = new(UpdatePreFilteredStatement, connection);
+        for (long i = 0; i < count; i++)
-        command.Parameters.AddWithValue("@id", id);
+        {
+            if (_preFilteredQueue.Count > 1000)
+            {
+                Thread.Sleep(5);
+            }
+            
+            command.Parameters.AddWithValue("@id", i);
+            reader = command.ExecuteReader();
+            command.Parameters.Clear();
+            
+            Ip ip = new();
 
-        command.ExecuteNonQuery();
+            if (!reader.HasRows)
-        command.Dispose();
+            {
-        
+                Console.WriteLine("Uuhhh");
-        return true;
+            }
+             
+            while (reader.Read())
+            {
+                ip.Ip1 = reader.GetInt32(0);
+                ip.Ip2 = reader.GetInt32(1);
+                ip.Ip3 = reader.GetInt32(2);
+                ip.Ip4 = reader.GetInt32(3);
+                ip.ResponseCode = reader.GetInt32(4);
+            }
+            
+            reader.Close();
+            
+            _preFilteredQueue.Enqueue(ip);
+            
+            command2.Parameters.AddWithValue("@id", i);
+            command2.ExecuteNonQuery();
+            command2.Parameters.Clear();
+        }
     }
 
     private static long GetDiscardedIndexesForSpecificDb(string connectionString)
@@ -726,36 +739,6 @@ public class DbHandler
         return true;
     }
     
-    public List<SearchResult?> GetSearchResults()
-    {
-        lock (_readFilteredLock)
-        {
-            using SqliteConnection connection = new(_filteredConnectionString);
-            connection.Open();
-            
-            using SqliteCommand command = new(ReadFilteredStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return [];
-            }
-            
-            List<SearchResult?> results = [];
-            
-            while (reader.Read())
-            {
-                SearchResult result = new();
-                result.Title = reader.GetString(0);
-                result.Url = reader.GetString(1);
-                
-                results.Add(result);
-            }
-            
-            return results;
-        }
-    }
-    
     public ScannerResumeObject? GetResumeObject(int threadNumber)
     {
         lock (_readAndDeleteResumeLock)
@@ -791,78 +774,13 @@ public class DbHandler
             return resumeObject;
         }
     }
-    
-    public void ReIndex()
-    {
-        _pause = true;
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
-        
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
-    
-    public void Vacuum()
-    {
-        _pause = true;
-        
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
-
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
 
     private (string, string) CreateDiscardedDb(int threadNumber)
     {
         string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-        
-        _discardedConnectionStrings.Add(databaseName);
         
         using SqliteConnection connection = new(databaseName);
         connection.Open();

Models/Helper/CompressionHelper.cs

@@ -1,4 +1,5 @@
 using System.IO.Compression;
+using System.Text;
 
 namespace Models.Helper;
 
@@ -19,4 +20,31 @@ public static class CompressionHelper
         using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
         decompressor.CopyTo(decompressedFileStream);
     }
+    
+    public static string CompressString(string text)
+    {
+        byte[] byteArray = Encoding.UTF8.GetBytes(text);
+        
+        using MemoryStream memoryStream = new();
+        
+        using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize))
+        {
+            compressionStream.Write(byteArray, 0, byteArray.Length);
+        }
+        
+        return Convert.ToBase64String(memoryStream.ToArray());
+    }
+    
+    public static string DecompressString(string compressedText)
+    {
+        byte[] byteArray = Convert.FromBase64String(compressedText);
+        
+        using MemoryStream memoryStream = new(byteArray);
+        
+        using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress);
+        
+        using StreamReader reader = new(decompressionStream);
+        
+        return reader.ReadToEnd();
+    }
 }

Models/Model/Backend/IP.cs

@@ -10,8 +10,15 @@ public struct Ip
     
     public int Ip4 { get; set; }
     
+    public int ResponseCode { get; set; }
+    
     public override string ToString()
     {
         return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
     }
+    
+    public uint PackIp()
+    {
+        return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4;
+    }
 }

Models/Model/External/SearchResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResult
-{
-    public string? Title { get; set; } = "";
-    public string? Url { get; set; } = "";
-    public string? Description { get; set; } = "";
-}

RSE.sln.DotSettings.user

@@ -16,6 +16,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F6079bfa8f3c0474bb21ade75ce5d010318000_003F15_003F57afed32_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -25,14 +26,17 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteConnection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc1d827416891bc9ceaece7826dbd14f2ab1ef304d85bcc1f2187646a67c3c_003FSqliteConnection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F13_003Fbe8f056e_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F90_003F0588ec81_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>