From 2dc414fc64959ad0a47da970e7eb528e365b642b Mon Sep 17 00:00:00 2001
From: Rasmus <rasmusbwr.prof@gmail.com>
Date: Tue, 31 Dec 2024 12:59:01 +0100
Subject: [PATCH] Removed Agility pack and changed a few things

---
 Backend/Backend.csproj             |  1 -
 Backend/Handler/ThreadHandler.cs   |  2 +-
 Backend/Helper/HttpClientHelper.cs |  1 -
 Models/Handler/DbHandler.cs        | 32 ++++++++++++++++++++++--------
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj
index 95c5c1e..5c26c51 100644
--- a/Backend/Backend.csproj
+++ b/Backend/Backend.csproj
@@ -16,7 +16,6 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
-      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     
diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs
index 784c270..7c8da07 100644
--- a/Backend/Handler/ThreadHandler.cs
+++ b/Backend/Handler/ThreadHandler.cs
@@ -60,7 +60,7 @@ public class ThreadHandler
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
         List<WaitHandle[]> wait = _ipScanner.Start(128);
-
+        
         for (int i = 0; i < wait.Count; i++)
         {
             WaitHandle.WaitAll(wait[i]);
diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs
index 444c16c..d264100 100644
--- a/Backend/Helper/HttpClientHelper.cs
+++ b/Backend/Helper/HttpClientHelper.cs
@@ -1,5 +1,4 @@
 using System.Text.RegularExpressions;
-using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs
index 282eaf8..c3de072 100644
--- a/Models/Handler/DbHandler.cs
+++ b/Models/Handler/DbHandler.cs
@@ -244,6 +244,16 @@ public class DbHandler
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
         (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+
+        // Compress the databases if they're already filled.
+        if (GetDiscardedIndexesForSpecificDb(connectionString) >= 50_000_000 && !_compressing)
+        {
+            _compressing = true;
+            
+            Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
+            
+            _compressing = false;
+        }
         
         int i = 0;
         
@@ -262,13 +272,7 @@ public class DbHandler
                 
                 i = 0;
                 
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
                 
                 _compressing = false;
             }
@@ -284,6 +288,18 @@ public class DbHandler
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
+
+    private void Compress(int threadNumber, string absolutePath, string connectionString)
+    {
+        InsertCompressedDatabase(threadNumber, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+        int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+        CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+        DropAndCreateDiscarded(threadNumber);
+    }
+    
     
     private void DropAndCreateDiscarded(int threadNumber)
     {
@@ -583,7 +599,7 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
-
+        
         SqliteConnection connection;
         SqliteCommand command;
         SqliteDataReader reader;