Backend/Handler/IpScanner.cs

@@ -181,7 +181,8 @@ public class IpScanner
                             _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             if (address is not null)
                             {
-                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
+                                Thread.Sleep(1);
                             }
                         }
                         catch

Backend/Handler/ThreadHandler.cs

@@ -31,7 +31,7 @@ public class ThreadHandler
     public void Start()
     {
         Thread scanner = new(StartScanner);
-        Thread indexer = new(StartContentFilter);
+        //Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
         Thread discarded = new(StartDiscardedDbHandler);
         Thread filtered = new(StartFilteredDbHandler);
@@ -39,7 +39,7 @@ public class ThreadHandler
         Thread communication = new(StartCommunicationHandler);
         
         scanner.Start();
-        indexer.Start();
+        //indexer.Start();
         database.Start();
         discarded.Start();
         filtered.Start();
@@ -47,7 +47,7 @@ public class ThreadHandler
         communication.Start();
         
         scanner.Join();
-        indexer.Join();
+        //indexer.Join();
         database.Join();
         discarded.Join();
         filtered.Join();
@@ -101,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(4);
+        WaitHandle[] wait = _dbHandler.Start(3);
         
         WaitHandle.WaitAll(wait);
         

Models/Handler/DbHandler.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using Microsoft.Data.Sqlite;
+using Models.Helper;
 using Models.Model.Backend;
 using Models.Model.External;
 
@@ -17,6 +18,7 @@ public class DbHandler
     private readonly string _discardedConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
+    private readonly string _CompressedConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
@@ -74,6 +76,10 @@ public class DbHandler
                                             " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
                                             " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
                                             " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+
+    private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                                                   " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                                                   " INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@@ -95,6 +101,7 @@ public class DbHandler
     private bool _stop;
     private bool _pause;
     private bool _paused;
+    private bool _compressing;
 
     private int _contentWaitTime;
     private int _discardedWaitTime;
@@ -120,6 +127,7 @@ public class DbHandler
         _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
+        _CompressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -236,7 +244,9 @@ public class DbHandler
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
-        string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+        
+        int i = 0;
         
         while (!_stop)
         {
@@ -247,15 +257,60 @@ public class DbHandler
                 continue;
             }
             
+            if (i == Random.Shared.Next(1_000_000, 10_000_000) || i >= 10_000_000 && !_compressing)
+            {
+                _compressing = true;
+
+                i = 0;
+                
+                Console.WriteLine("Compressing");
+                
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexes());
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}.gz");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
+                
+                _compressing = false;
+            }
+            
             _discardedQueue.TryDequeue(out Discarded queueItem);
             
             InsertDiscarded(queueItem, connectionString);
+            
+            i++;
         }
         
         discardedDbHandlerSetting.Handle!.Set();
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
+
+    private void DropAndCreateDiscarded(int threadNumber)
+    {
+        string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
+        
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string dropStatement = "DROP TABLE Discarded;";
+        const string vacuum = "VACUUM;";
+        
+        using SqliteConnection connection = new(databaseName);
+        connection.Open();
+        
+        SqliteCommand command = new(dropStatement, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(vacuum, connection);
+        command.ExecuteNonQuery();
+        
+        command = new(createStatement, connection);
+        command.ExecuteNonQuery();
+        command.Dispose();
+        
+        connection.Close();
+    }
     
     private void InsertUnfiltered(Unfiltered unfiltered)
     {
@@ -423,6 +478,20 @@ public class DbHandler
         _ = command.ExecuteNonQuery();
         connection.Close();
     }
+
+    private void InsertCompressedDatabase(int threadNumber, long rows)
+    {
+        using SqliteConnection connection = new(_CompressedConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
+        
+        command.Parameters.AddWithValue("@dbNumber", threadNumber);
+        command.Parameters.AddWithValue("@rows", rows);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+    }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
@@ -732,8 +801,9 @@ public class DbHandler
         _paused = false;
     }
 
-    private string CreateDiscardedDb(int threadNumber)
+    private (string, string) CreateDiscardedDb(int threadNumber)
     {
+        string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
         const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
@@ -746,7 +816,7 @@ public class DbHandler
         using SqliteCommand command = new(createStatement, connection);
         command.ExecuteNonQuery();
         
-        return databaseName;
+        return (absolutePath, databaseName);
     }
     
     public void Stop()

Models/Helper/CompressionHelper.cs

@@ -0,0 +1,22 @@
+using System.IO.Compression;
+
+namespace Models.Helper;
+
+public static class CompressionHelper
+{
+    public static void CompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream originalFileStream = new(sourceFile, FileMode.Open);
+        using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
+        originalFileStream.CopyTo(compressor);
+    }
+    
+    public static void DecompressFile(string sourceFile, string targetFile)
+    {
+        using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
+        using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
+        using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
+        decompressor.CopyTo(decompressedFileStream);
+    }
+}

Models/Helper/GetDatabasesHelper.cs

@@ -0,0 +1,9 @@
+namespace Models.Helper;
+
+public static class GetDatabasesHelper
+{
+    public static int GetTotalCompressedDatabases(string path)
+    {
+        return Directory.GetFiles(path, "*.gz").Length;
+    }
+}

RSE.sln.DotSettings.user

@@ -4,8 +4,10 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>