Backend/Backend.csproj

@@ -16,7 +16,6 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
-      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/ThreadHandler.cs

@@ -60,7 +60,7 @@ public class ThreadHandler
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
         List<WaitHandle[]> wait = _ipScanner.Start(128);
-
+        
         for (int i = 0; i < wait.Count; i++)
         {
             WaitHandle.WaitAll(wait[i]);

Backend/Helper/HttpClientHelper.cs

@@ -1,5 +1,4 @@
 using System.Text.RegularExpressions;
-using HtmlAgilityPack;
 
 namespace Backend.Helper;
 

Models/Handler/DbHandler.cs

@@ -244,6 +244,16 @@ public class DbHandler
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
         (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+
+        // Compress the databases if they're already filled.
+        if (GetDiscardedIndexesForSpecificDb(connectionString) >= 50_000_000 && !_compressing)
+        {
+            _compressing = true;
+            
+            Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
+            
+            _compressing = false;
+        }
         
         int i = 0;
         
@@ -262,13 +272,7 @@ public class DbHandler
                 
                 i = 0;
                 
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
                 
                 _compressing = false;
             }
@@ -284,6 +288,18 @@ public class DbHandler
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
+
+    private void Compress(int threadNumber, string absolutePath, string connectionString)
+    {
+        InsertCompressedDatabase(threadNumber, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+        int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+        CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+        DropAndCreateDiscarded(threadNumber);
+    }
+    
     
     private void DropAndCreateDiscarded(int threadNumber)
     {
@@ -583,7 +599,7 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
-
+        
         SqliteConnection connection;
         SqliteCommand command;
         SqliteDataReader reader;