Merge pull request 'Fixed a new memory leak' (#47) from Analyzer into main

Reviewed-on: #47

Backend/Handler/IpFilterHandler.cs

@@ -1,7 +1,5 @@
 using System.Collections.Concurrent;
-using System.Diagnostics;
 using Backend.Helper;
-using Models.Handler;
 using Models.Model.Backend;
 
 namespace Backend.Handler;
@@ -10,23 +8,20 @@ public class IpFilterHandler
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
-    private DbHandler _dbHandler;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     private ThreadHandler _threadHandler;
     private bool _stop;
-    private bool _fillerStop;
+    private bool _done;
     private bool _stopAutoscaledThreads;
     private int _timeout;
     
     public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
-        DbHandler dbHandler, ThreadHandler threadHandler)
+        ConcurrentQueue<Ip> preFilteredQueue, ThreadHandler threadHandler)
     {
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _preFilteredQueue = preFilteredQueue;
-        _dbHandler = dbHandler;
         _threadHandler = threadHandler;
 
         _timeout = 16;
@@ -68,63 +63,14 @@ public class IpFilterHandler
         return waitHandles;
     }
     
-    public void AutoScaler()
-    {
-        int i = 0;
-        int j = 0;
-        
-        while (!_stop)
-        {
-            if (_preFilteredQueue.Count >= 2000)
-            {
-                if (i == 10)
-                {
-                    _stopAutoscaledThreads = false;
-                    Console.WriteLine("Autoscaler started");
-                    
-                    while (!_stopAutoscaledThreads)
-                    {
-                        if (_preFilteredQueue.Count <= 2000)
-                        {
-                            if (j == 1000)
-                            {
-                                _stopAutoscaledThreads = true;
-                            }
-
-                            j++;
-                            
-                            Thread.Sleep(128);
-                        }
-                        else
-                        {
-                            EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-                            Thread f = new (Filter_AutoScaler!);
-                            f.Start(handle);
-                            Thread.Sleep(16);
-                        }
-                    }
-                }
-
-                i++;
-            }
-            else
-            {
-                i = 0;
-                j = 0;
-            }
-            
-            Thread.Sleep(128);
-        }
-    }
-
     private void Filter(object obj)
     {
         int counter = 0;
         while (!_stop)
         {
-            if (_preFilteredQueue.IsEmpty && _fillerStop)
+            if (_preFilteredQueue.IsEmpty)
             {
-                if (counter == 100)
+                if (counter == 30_000)
                 {
                     _threadHandler.Stop();
                     _stop = true;
@@ -132,63 +78,21 @@ public class IpFilterHandler
                 
                 counter++;
                 Thread.Sleep(128);
+                
+                continue;
             }
             
-            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
+            _preFilteredQueue.TryDequeue(out Ip item);
             
-            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
+            (int, int) ports = TcpClientHelper.CheckPort(item, 80, 443);
             
             if (ports is { Item1: 0, Item2: 0 })
             {
-                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
+                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item, item.ResponseCode));
                 continue;
             }
             
-            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
-        }
-        
-        ((EventWaitHandle) obj).Set();
-    }
-
-    public void FillFilterQueue()
-    {
-        Console.WriteLine("Fill FilterQueue started.");
-        while (!_stop)
-        {
-            if (_preFilteredQueue.Count > 500) continue;
-            
-            if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
-            {
-                _preFilteredQueue.Enqueue(item);
-            }
-            else
-            {
-                _fillerStop = true;
-            }
-        }
-    }
-    
-    private void Filter_AutoScaler(object obj)
-    {
-        while (!_stopAutoscaledThreads)
-        {
-            if (_preFilteredQueue.IsEmpty)
-            {
-                Thread.Sleep(_timeout);
-                continue;
-            }
-            
-            _preFilteredQueue.TryDequeue(out FilterQueueItem item);
-            
-            (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
-            
-            if (ports is { Item1: 0, Item2: 0 })
-            {
-                _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
-                continue;
-            }
-            
-            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
+            _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item, ports));
         }
         
         ((EventWaitHandle) obj).Set();

Backend/Handler/IpScanner.cs

@@ -22,7 +22,7 @@ public class ScanSettings
 public class IpScanner
 {
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
@@ -30,7 +30,7 @@ public class IpScanner
     
     public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
         ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler, 
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue)
+        ConcurrentQueue<Ip> preFilteredQueue)
     {
         _dbHandler = dbHandler;
         _preFilteredQueue = preFilteredQueue;
@@ -145,12 +145,12 @@ public class IpScanner
                     if (i == 192 && k == 2) continue;
                     if (i == 192 && j == 88 && k == 99) continue;
                     
-                    if (_discardedQueue.Count >= 2000)
+                    if (_discardedQueue.Count >= 20_000)
                     {
                         Thread.Sleep(1000);
                     }
                     
-                    if (_preFilteredQueue.Count >= 2000)
+                    if (_preFilteredQueue.Count >= 20_000)
                     {
                         Thread.Sleep(1000);
                     }
@@ -171,6 +171,8 @@ public class IpScanner
                             break;
                         }
                         
+                        
+                        
                         Ip ip = new()
                         {
                             Ip1 = i,
@@ -187,18 +189,20 @@ public class IpScanner
                             // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
                             //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             
-                            if (i % 2 == 0)
+                            responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
+                            
+                            /*if (l % 2 == 0)
                             {
                                 responseCode = IPStatus.Success;
                             }
                             else
                             {
                                 responseCode = IPStatus.TimedOut;
-                            }
+                            }*/
                             
-                            //CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
-                            Thread.Sleep(16);
+                            //responseCode = IPStatus.TimedOut;
                             
+                            //Thread.Sleep(0);
                         }
                         catch
                         {
@@ -233,10 +237,10 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            
-            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
+        Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped.");
+        
         if (_stop)
         {
             resumeObject.Paused = true;
@@ -279,15 +283,11 @@ public class IpScanner
         };
     }
     
-    private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
+    private static Ip CreateUnfilteredQueueItem(Ip ip, int responseCode)
     {
-        FilterQueueItem filterQueueItem = new()
-        {
-            Ip = ip,
-            ResponseCode = responseCode
-        };
+        ip.ResponseCode = responseCode;
         
-        return filterQueueItem;
+        return ip;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -15,22 +15,22 @@ public class ThreadHandler
     private bool _contentFilterStopped;
     private bool _ipFilterStopped;
 
-    private bool _stage1;
+    private bool _stage1 = false;
     private bool _stage2 = true;
-    private bool _stage3;
+    private bool _stage3 = false;
     
     ConcurrentQueue<Filtered> filteredQueue = new();
     ConcurrentQueue<Discarded> discardedQueue = new();
     ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
     ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
-    ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
+    ConcurrentQueue<Ip> preFilteredQueue = new();
     
     public ThreadHandler(string path)
     {
         _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
         _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
         _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
-        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
+        _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, this);
     }
     
     public void Start()
@@ -42,12 +42,13 @@ public class ThreadHandler
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
         Thread resume = new(StartResumeDbHandler);
-        Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
         Thread contentFilterThread = new(StartContentFilterThread);
         Thread prefilterDb = new(StartPreFilterDbHandler);
         Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
         //Thread check = new(CheckQueue);
         
+        //check.Start();
+        
         if (_stage1)
         {
             discarded.Start(); // de-queues from discardedQueue
@@ -55,9 +56,11 @@ public class ThreadHandler
             scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
             resume.Start(); // de-queues from resumeQueue
             
+            scanner.Join();
+            Stop();
+            
             discarded.Join();
             prefilterDb.Join();
-            scanner.Join();
             resume.Join();
         }
         
@@ -66,13 +69,12 @@ public class ThreadHandler
             database.Start(); // de-queues from unfilteredQueue
             discarded.Start(); // de-queues from discardedQueue
             ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
-            ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
             fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
 
+            ipFilter.Join();
+            Stop();
             database.Join();
             discarded.Join();
-            ipFilter.Join();
-            ipFilterAutoScaler.Join();
             fillIpFilterQueue.Join();
         }
         
@@ -104,6 +106,7 @@ public class ThreadHandler
         }
     }
 
+    
     private void StartScanner()
     {
         Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
@@ -133,26 +136,21 @@ public class ThreadHandler
 
     private void StartContentFilterThread()
     {
-        WaitHandle[] wait = _contentFilter.StartFilterThread(64);
+        WaitHandle[] wait = _contentFilter.StartFilterThread(8);
         
         WaitHandle.WaitAll(wait);
     }
 
-    private void StartIpFilterAutoScaler()
-    {
-        _ipFilterHandler.AutoScaler();
-    }
-
     private void StartFillIpFilterQueue()
     {
-        _ipFilterHandler.FillFilterQueue();
+        _dbHandler.GetPreFilterQueueItem();
     }
 
     private void StartIpFilter()
     {
         Thread.Sleep(1000);
         
-        List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
+        List<WaitHandle[]> wait = _ipFilterHandler.Start(1024);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -205,6 +203,7 @@ public class ThreadHandler
         
         Console.WriteLine("Stopping Super Extra...");
         _dbHandler.Stop();
+        
         Console.WriteLine("Stopped.");
     }
 }

Models/Handler/DbHandler.cs

@@ -1,9 +1,10 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
 using Microsoft.Data.Sqlite;
 using Models.Helper;
 using Models.Model.Backend;
-using Models.Model.External;
 
 namespace Models.Handler;
 
@@ -13,14 +14,13 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
-    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
+    private readonly ConcurrentQueue<Ip> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
     private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
     private readonly string _compressedConnectionString;
-    private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -75,8 +75,8 @@ public class DbHandler
     
     private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                                " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
-                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+                                               " INSERT INTO Discarded (PackedData)" +
+                                               " VALUES (@packedData)";
     
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -96,7 +96,7 @@ public class DbHandler
     private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
     private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
-    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
+    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Id == @id ORDER BY Id ASC LIMIT 1;";
     private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
@@ -107,7 +107,6 @@ public class DbHandler
     
     private const string VacuumDatabasesStatement = "VACUUM;";
     
-    private readonly object _readFilteredLock = new();
     private readonly object _readAndDeleteResumeLock = new();
     
     private bool _stop;
@@ -124,7 +123,7 @@ public class DbHandler
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
         ConcurrentQueue<ScannerResumeObject> resumeQueue, 
-        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        ConcurrentQueue<Ip> preFilteredQueue,
         string basePath)
     {
         _filteredQueue = filteredQueue;
@@ -217,7 +216,7 @@ public class DbHandler
                 continue;
             }
             
-            _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
+            _preFilteredQueue.TryDequeue(out Ip queueItem);
             
             InsertPrefiltered(queueItem);
         }
@@ -263,7 +262,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(5000);
+            Thread.Sleep(150);
         }
         
         return waitHandles;
@@ -278,7 +277,16 @@ public class DbHandler
         
         int i = 0;
         
-        while (!_stop)
+        SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        SqliteCommand command = new(InsertIntoDiscarded, connection);
+        
+        StringBuilder stringBuilder = new();
+        
+        bool running = true;
+        
+        while (!_stop && running)
         {
             if (_discardedQueue.IsEmpty || _pause)
             {
@@ -287,30 +295,45 @@ public class DbHandler
                 continue;
             }
 
-            if (i >= 5_000_000 && !_compressing)
+            if (_stop && _discardedQueue.IsEmpty)
             {
-                _compressing = true;
-                
-                i = 0;
-                
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
-                
-                _compressing = false;
+                running = false;
             }
             
-            _discardedQueue.TryDequeue(out Discarded queueItem);
+            if (i == 2048)
+            {
+                command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+                _ = command.ExecuteNonQuery();
                 
-            InsertDiscarded(queueItem, connectionString);
+                // Re-use the command object.
+                command.Parameters.Clear();
+                
+                stringBuilder.Clear();
+                
+                i = 0;
+            }
+            
+            if (_discardedQueue.TryDequeue(out Discarded queueItem))
+            {
+                stringBuilder.Append(queueItem.Ip.PackIp());
+                stringBuilder.Append(':');
+                stringBuilder.Append(queueItem.ResponseCode);
+                stringBuilder.Append(',');
+            }
             
             i++;
         }
         
+        if (stringBuilder.Length != 0)
+        {
+            command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
+            _ = command.ExecuteNonQuery();
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        
         discardedDbHandlerSetting.Handle!.Set();
         
         Console.WriteLine("Discarded DbHandler stopped.");
@@ -359,23 +382,6 @@ public class DbHandler
         connection.Close();
     }
     
-    private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
-    {
-        using SqliteConnection connection = new(dbConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoDiscarded, connection);
-        
-        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
-        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
-        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
-        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
-        command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
-        _ = command.ExecuteNonQuery();
-        
-        connection.Close();
-    }
-    
     private void InsertFiltered(Filtered filtered)
     {
         using SqliteConnection connection = new(_filteredConnectionString);
@@ -515,35 +521,22 @@ public class DbHandler
         command.Dispose();
     }
 
-    private void InsertCompressedDatabase(int threadNumber, long rows)
-    {
-        using SqliteConnection connection = new(_compressedConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
-        
-        command.Parameters.AddWithValue("@dbNumber", threadNumber);
-        command.Parameters.AddWithValue("@rows", rows);
-        
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-    }
-
-    private void InsertPrefiltered(FilterQueueItem filterQueueItem)
+    private void InsertPrefiltered(Ip filterQueueItem)
     {
         using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
         
         using SqliteCommand command = new(InsertPreFilteredStatement, connection);
         
-        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
-        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
-        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
-        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
+        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip1);
+        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip2);
+        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip3);
+        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip4);
         command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
         command.Parameters.AddWithValue("@filtered", 0);
         
         _ = command.ExecuteNonQuery();
+        
         connection.Close();
     }
     
@@ -614,21 +607,43 @@ public class DbHandler
         return ids;
     }
     
-    public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
+    public void GetPreFilterQueueItem()
     {
         using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
 
-        SqliteCommand command = new(ReadPreFilteredStatement, connection);
-        using SqliteDataReader reader = command.ExecuteReader();
+        const string temp = "SELECT seq FROM sqlite_sequence;";
+        
+        SqliteCommand command = new(ReadPreFilteredStatement, connection);
+        SqliteCommand command2 = new(UpdatePreFilteredStatement, connection);
+        SqliteCommand command3 = new(temp, connection);
+        SqliteDataReader reader = command3.ExecuteReader();
+
+        long count = 0;
+        
+        while (reader.Read())
+        {
+            count = reader.GetInt64(0);
+        }
+        
+        reader.Close();
+        
+        for (long i = 0; i < count; i++)
+        {
+            if (_preFilteredQueue.Count > 1000)
+            {
+                Thread.Sleep(5);
+            }
+            
+            command.Parameters.AddWithValue("@id", i);
+            reader = command.ExecuteReader();
+            command.Parameters.Clear();
             
-        filterQueueItem = new();
             Ip ip = new();
-        long id = 0;
 
             if (!reader.HasRows)
             {
-            return false;
+                Console.WriteLine("Uuhhh");
             }
              
             while (reader.Read())
@@ -637,19 +652,17 @@ public class DbHandler
                 ip.Ip2 = reader.GetInt32(1);
                 ip.Ip3 = reader.GetInt32(2);
                 ip.Ip4 = reader.GetInt32(3);
-            filterQueueItem.ResponseCode = reader.GetInt32(4);
-            id = reader.GetInt64(5);
+                ip.ResponseCode = reader.GetInt32(4);
             }
             
-        filterQueueItem.Ip = ip;
+            reader.Close();
             
-        command = new(UpdatePreFilteredStatement, connection);
-        command.Parameters.AddWithValue("@id", id);
+            _preFilteredQueue.Enqueue(ip);
             
-        command.ExecuteNonQuery();
-        command.Dispose();
-        
-        return true;
+            command2.Parameters.AddWithValue("@id", i);
+            command2.ExecuteNonQuery();
+            command2.Parameters.Clear();
+        }
     }
 
     private static long GetDiscardedIndexesForSpecificDb(string connectionString)
@@ -726,36 +739,6 @@ public class DbHandler
         return true;
     }
     
-    public List<SearchResult?> GetSearchResults()
-    {
-        lock (_readFilteredLock)
-        {
-            using SqliteConnection connection = new(_filteredConnectionString);
-            connection.Open();
-            
-            using SqliteCommand command = new(ReadFilteredStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return [];
-            }
-            
-            List<SearchResult?> results = [];
-            
-            while (reader.Read())
-            {
-                SearchResult result = new();
-                result.Title = reader.GetString(0);
-                result.Url = reader.GetString(1);
-                
-                results.Add(result);
-            }
-            
-            return results;
-        }
-    }
-    
     public ScannerResumeObject? GetResumeObject(int threadNumber)
     {
         lock (_readAndDeleteResumeLock)
@@ -792,77 +775,12 @@ public class DbHandler
         }
     }
 
-    public void ReIndex()
-    {
-        _pause = true;
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
-        
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
-    
-    public void Vacuum()
-    {
-        _pause = true;
-        
-        Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
-
-        if (!_paused)
-        {
-            Thread.Sleep(5000); // Just for safety.
-        }
-        
-        SqliteConnection connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        SqliteCommand command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection = new(_unfilteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
-        connection.Dispose();
-        command.Dispose();
-        
-        _pause = false;
-        _paused = false;
-    }
-
     private (string, string) CreateDiscardedDb(int threadNumber)
     {
         string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
-        
-        _discardedConnectionStrings.Add(databaseName);
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
         
         using SqliteConnection connection = new(databaseName);
         connection.Open();

Models/Helper/CompressionHelper.cs

@@ -1,4 +1,5 @@
 using System.IO.Compression;
+using System.Text;
 
 namespace Models.Helper;
 
@@ -19,4 +20,31 @@ public static class CompressionHelper
         using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
         decompressor.CopyTo(decompressedFileStream);
     }
+    
+    public static string CompressString(string text)
+    {
+        byte[] byteArray = Encoding.UTF8.GetBytes(text);
+        
+        using MemoryStream memoryStream = new();
+        
+        using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize))
+        {
+            compressionStream.Write(byteArray, 0, byteArray.Length);
+        }
+        
+        return Convert.ToBase64String(memoryStream.ToArray());
+    }
+    
+    public static string DecompressString(string compressedText)
+    {
+        byte[] byteArray = Convert.FromBase64String(compressedText);
+        
+        using MemoryStream memoryStream = new(byteArray);
+        
+        using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress);
+        
+        using StreamReader reader = new(decompressionStream);
+        
+        return reader.ReadToEnd();
+    }
 }

Models/Model/Backend/IP.cs

@@ -10,8 +10,15 @@ public struct Ip
     
     public int Ip4 { get; set; }
     
+    public int ResponseCode { get; set; }
+    
     public override string ToString()
     {
         return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
     }
+    
+    public uint PackIp()
+    {
+        return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4;
+    }
 }

Models/Model/External/SearchResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResult
-{
-    public string? Title { get; set; } = "";
-    public string? Url { get; set; } = "";
-    public string? Description { get; set; } = "";
-}

RSE.sln.DotSettings.user

@@ -16,6 +16,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F6079bfa8f3c0474bb21ade75ce5d010318000_003F15_003F57afed32_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -25,14 +26,17 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteConnection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc1d827416891bc9ceaece7826dbd14f2ab1ef304d85bcc1f2187646a67c3c_003FSqliteConnection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStackFrameIterator_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F13_003Fbe8f056e_003FStackFrameIterator_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F1d0213738b064ff59eef90be1f87fbbbd19c00_003F90_003F0588ec81_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>