Implement database compression

Models/Handler/DbHandler.cs

@@ -15,10 +15,9 @@ public class DbHandler
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     
     private readonly string _unfilteredConnectionString;
-    private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
-    private readonly string _CompressedConnectionString;
+    private readonly string _compressedConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
@@ -88,6 +87,7 @@ public class DbHandler
     private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
+    private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
@@ -124,10 +124,9 @@ public class DbHandler
         _basePath = basePath;
         
         _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
-        _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
-        _CompressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
+        _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -221,7 +220,7 @@ public class DbHandler
         for (int i = 0; i < threads; i++)
         {
             EventWaitHandle handle = new(false, EventResetMode.ManualReset);
-
+            
             DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
             {
                 Handle = handle,
@@ -233,7 +232,7 @@ public class DbHandler
             Thread f = new (DiscardedDbHandler!);
             f.Start(discardedDbHandlerSetting);
             
-            Thread.Sleep(1000);
+            Thread.Sleep(5000);
         }
         
         return waitHandles;
@@ -257,19 +256,17 @@ public class DbHandler
                 continue;
             }
             
-            if (i == Random.Shared.Next(1_000_000, 10_000_000) || i >= 10_000_000 && !_compressing)
+            if (i >= 50_000_000 && !_compressing)
             {
                 _compressing = true;
-
+                
                 i = 0;
                 
-                Console.WriteLine("Compressing");
-                
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexes());
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
                 
                 int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
                 
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}.gz");
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
                 
                 DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
                 
@@ -287,7 +284,7 @@ public class DbHandler
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
-
+    
     private void DropAndCreateDiscarded(int threadNumber)
     {
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
@@ -481,7 +478,7 @@ public class DbHandler
 
     private void InsertCompressedDatabase(int threadNumber, long rows)
     {
-        using SqliteConnection connection = new(_CompressedConnectionString);
+        using SqliteConnection connection = new(_compressedConnectionString);
         connection.Open();
         
         using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
@@ -586,26 +583,75 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
+
+        SqliteConnection connection;
+        SqliteCommand command;
+        SqliteDataReader reader;
         
         for (int i = 0; i < _discardedConnectionStrings.Count; i++)
         {
-            using SqliteConnection connection = new(_discardedConnectionStrings[i]);
+            connection = new(_discardedConnectionStrings[i]);
             connection.Open();
             
-            using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
-            using SqliteDataReader reader = command.ExecuteReader();
+            command = new(ReadDiscardedSeqIdsStatement, connection);
+            reader = command.ExecuteReader();
             
             if (!reader.HasRows)
             {
-                return 0;
+                return rowId;
             }
             
             while (reader.Read())
             {
                 rowId += reader.GetInt64(0);
             }
+            
+            connection.Close();
         }
         
+        connection = new(_compressedConnectionString);
+        connection.Open();
+        command = new(ReadCompressedDbRowsStatement, connection);
+        reader = command.ExecuteReader();
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+        
+        connection.Close();
+        connection.Dispose();
+        command.Dispose();
+        reader.Dispose();
+        
+        return rowId;
+    }
+
+    private static long GetDiscardedIndexesForSpecificDb(string connectionString)
+    {
+        using SqliteConnection connection = new(connectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
+        using SqliteDataReader reader = command.ExecuteReader();
+        
+        long rowId = 0;
+        
+        if (!reader.HasRows)
+        {
+            return rowId;
+        }
+        
+        while (reader.Read())
+        {
+            rowId += reader.GetInt64(0);
+        }
+
         return rowId;
     }
 
@@ -734,20 +780,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(ReIndexDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(ReIndexDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();
         
@@ -773,20 +812,13 @@ public class DbHandler
             Thread.Sleep(5000); // Just for safety.
         }
         
-        SqliteConnection connection = new(_discardedConnectionString);
+        SqliteConnection connection = new(_filteredConnectionString);
         connection.Open();
         
         SqliteCommand command = new(VacuumDatabasesStatement, connection);
         _ = command.ExecuteNonQuery();
         connection.Close();
         
-        connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        command = new(VacuumDatabasesStatement, connection);
-        _ = command.ExecuteNonQuery();
-        connection.Close();
-        
         connection = new(_unfilteredConnectionString);
         connection.Open();