Backend/Handler/ContentFilter.cs

@@ -8,8 +8,7 @@ namespace Backend.Handler;
 
 public class ContentFilter
 {
-    private readonly ConcurrentQueue<Filtered> _queue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<QueueItem> _queue;
     private readonly DbHandler _dbHandler;
     private readonly string _getDomainPort80;
     private readonly string _getDomainPort443;
@@ -17,13 +16,12 @@ public class ContentFilter
     private int _timeOut;
     private readonly string _basePath;
     
-    public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
+    public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, string basePath)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
-        _unfilteredQueue = unfilteredQueue;
-
+        
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
@@ -65,13 +63,14 @@ public class ContentFilter
                 
                 unfiltered.Filtered = true;
                 
-                UnfilteredQueueItem superUnfilteredObject = new()
+                QueueItem superUnfilteredObject = new()
                 {
                     Unfiltered = unfiltered,
-                    Operations = Operations.Update
+                    Operations = Operations.Update,
+                    DbType = DbType.Unfiltered
                 };
                 
-                _unfilteredQueue.Enqueue(superUnfilteredObject);
+                _queue.Enqueue(superUnfilteredObject);
                 
                 if (_dbHandler.FilteredIpExists(unfiltered.Ip))
                 {
@@ -83,7 +82,14 @@ public class ContentFilter
                 filtered.Port1 = unfiltered.Port1;
                 filtered.Port2 = unfiltered.Port2;
                 
-                _queue.Enqueue(filtered);
+                QueueItem superFilteredObject = new()
+                {
+                    Filtered = filtered,
+                    Operations = Operations.Insert,
+                    DbType = DbType.Filtered
+                };
+                
+                _queue.Enqueue(superFilteredObject);
             }
             
             Thread.Sleep(_timeOut);

Backend/Handler/IpScanner.cs

@@ -17,22 +17,18 @@ public class ScanSettings
 
 public class IpScanner
 {
+    private readonly ConcurrentQueue<QueueItem> _queue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
-    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     private readonly DbHandler _dbHandler;
     private bool _stop;
     private int _timeout;
     
-    public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
-        )
+    public IpScanner(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, ConcurrentQueue<Discarded> discardedQueue)
     {
+        _queue = queue;
         _dbHandler = dbHandler;
         _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
-        _resumeQueue = resumeQueue;
-
+        
         SetTimeout(128);
     }
     
@@ -171,7 +167,7 @@ public class IpScanner
                             continue;
                         }
                         
-                        _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
+                        _queue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
                     }
                     
                     if (_stop)
@@ -196,7 +192,13 @@ public class IpScanner
             //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
-        _resumeQueue.Enqueue(resumeObject);
+        QueueItem resume = new()
+        {
+            ResumeObject = resumeObject,
+            Operations = Operations.Insert
+        };
+        
+        _queue.Enqueue(resume);
         
         scanSettings.Handle!.Set();
     }
@@ -210,7 +212,7 @@ public class IpScanner
         };
     }
     
-    private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
+    private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
     {
         Unfiltered unfiltered = new()
         {
@@ -223,7 +225,8 @@ public class IpScanner
         return new()
         {
             Unfiltered = unfiltered,
-            Operations = Operations.Insert
+            Operations = Operations.Insert,
+            DbType = DbType.Unfiltered
         };
     }
     

Backend/Handler/ThreadHandler.cs

@@ -17,41 +17,33 @@ public class ThreadHandler
     
     public ThreadHandler(string path)
     {
-        ConcurrentQueue<Filtered> filteredQueue = new();
+        ConcurrentQueue<QueueItem> contentQueue = new();
         ConcurrentQueue<Discarded> discardedQueue = new();
-        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
-        ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
         
-        _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
-        _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
-        _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
+        _dbHandler = new(contentQueue, discardedQueue, path);
+        _ipScanner = new(contentQueue, _dbHandler, discardedQueue);
+        _contentFilter = new(contentQueue, _dbHandler, path);
         _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
     }
     
     public void Start()
     {
-        Thread scanner = new(StartScanner);
+        //Thread scanner = new(StartScanner);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
-        Thread discarded = new(StartDiscardedDbHandler);
-        Thread filtered = new(StartFilteredDbHandler);
-        Thread resume = new(StartResumeDbHandler);
+        //Thread discarded = new(StartDiscardedDbHandler);
         Thread communication = new(StartCommunicationHandler);
         
-        scanner.Start();
+        //scanner.Start();
         indexer.Start();
         database.Start();
-        discarded.Start();
-        filtered.Start();
-        resume.Start();
+        //discarded.Start();
         communication.Start();
         
-        scanner.Join();
+        //scanner.Join();
         indexer.Join();
         database.Join();
-        discarded.Join();
-        filtered.Join();
-        resume.Join();
+        //discarded.Join();
         communication.Join();
     }
     
@@ -81,17 +73,7 @@ public class ThreadHandler
     
     private void StartDbHandler()
     {
-        _dbHandler.UnfilteredDbHandler();
-    }
-
-    private void StartFilteredDbHandler()
-    {
-        _dbHandler.FilteredDbHandler();
-    }
-
-    private void StartResumeDbHandler()
-    {
-        _dbHandler.ResumeDbHandler();
+        _dbHandler.StartContent();
     }
     
     private void StartDiscardedDbHandler()

Models/Handler/DbHandler.cs

@@ -1,3 +1,4 @@
+using System.Collections;
 using System.Collections.Concurrent;
 using Microsoft.Data.Sqlite;
 using Models.Model.Backend;
@@ -7,10 +8,8 @@ namespace Models.Handler;
 
 public class DbHandler
 {
-    private readonly ConcurrentQueue<Filtered> _filteredQueue;
-    private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
+    private readonly ConcurrentQueue<QueueItem> _contentQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
-    private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
     
     private readonly string _unfilteredConnectionString;
     private readonly string _discardedConnectionString;
@@ -18,34 +17,10 @@ public class DbHandler
     private readonly string _resumeConnectionString;
     private readonly List<string> _discardedConnectionStrings = [];
     
-    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                           " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
-                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
-    
-    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                              " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                              " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
-                                              " Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
-                                              " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
-                                              " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
-                                              " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
-                                              " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2," +
-                                              " @title1, @title2, @description1, @description2, @url1, @url2, @serverType1," +
-                                              " @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2," +
-                                              " @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion," +
-                                              " @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2," +
-                                              " @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
-    
-    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                               " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
-                                               " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
-    
-    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
-                                            " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
-                                            " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
-                                            " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
+    private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
+    private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
+    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode) VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
+    private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
@@ -73,17 +48,12 @@ public class DbHandler
 
     private readonly string _basePath;
     
-    public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
-        ConcurrentQueue<Discarded> discardedQueue, 
-        ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
+    public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
     {
-        _filteredQueue = filteredQueue;
+        _contentQueue = contentQueue;
         _discardedQueue = discardedQueue;
-        _unfilteredQueue = unfilteredQueue;
-        _resumeQueue = resumeQueue;
-
-        SetContentWaitTime(100);
+        
+        SetContentWaitTime(10);
         SetDiscardedWaitTime(10);
         
         _basePath = basePath;
@@ -104,78 +74,45 @@ public class DbHandler
         _discardedWaitTime = waitTime;
     }
     
-    public void UnfilteredDbHandler()
+    public void StartContent()
     {
-        Console.WriteLine("Unfiltered DbHandler started");
+        Console.WriteLine("Content DbHandler started");
         
         while (!_stop)
         {
-            if (_unfilteredQueue.IsEmpty || _pause)
+            if (_contentQueue.IsEmpty || _pause)
             {
                 Thread.Sleep(_contentWaitTime);
                 _paused = true;
                 continue;
             }
             
-            _unfilteredQueue.TryDequeue(out UnfilteredQueueItem queueItem);
+            _contentQueue.TryDequeue(out QueueItem? queueItem);
             
-            if (queueItem.Operations == Operations.Insert)
+            if (queueItem is null) { continue; }
+            
+            if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Unfiltered)
             {
                 InsertUnfiltered(queueItem.Unfiltered);
             }
             
-            else if (queueItem.Operations == Operations.Update)
+            else if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Filtered)
+            {
+                InsertFiltered(queueItem.Filtered!);
+            }
+            
+            else if (queueItem.Operations == Operations.Insert && queueItem.ResumeObject is not null)
+            {
+                InsertResumeObject(queueItem.ResumeObject);
+            }
+            
+            else if (queueItem.Operations == Operations.Update && queueItem.DbType == DbType.Unfiltered)
             {
                 UpdateUnfiltered(queueItem.Unfiltered);
             }
         }
         
-        Console.WriteLine("Unfiltered DbHandler stopped.");
-    }
-    
-    public void FilteredDbHandler()
-    {
-        Console.WriteLine("Filtered DB handler started");
-        
-        while (!_stop)
-        {
-            if (_filteredQueue.IsEmpty || _pause)
-            {
-                Thread.Sleep(_contentWaitTime);
-                _paused = true;
-                continue;
-            }
-            
-            _filteredQueue.TryDequeue(out Filtered? queueItem);
-            
-            InsertFiltered(queueItem!);
-        }
-        
-        Console.WriteLine("Filtered DbHandler stopped.");
-    }
-    
-    public void ResumeDbHandler()
-    {
-        Console.WriteLine("Resume DB handler started");
-        
-        while (!_stop)
-        {
-            if (_resumeQueue.IsEmpty || _pause)
-            {
-                Thread.Sleep(_contentWaitTime);
-                _paused = true;
-                continue;
-            }
-            
-            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
-            
-            if (queueItem is not null)
-            {
-                InsertResumeObject(queueItem);
-            }
-        }
-        
-        Console.WriteLine("Resume DbHandler stopped.");
+        Console.WriteLine("Content DbHandler stopped.");
     }
     
     public WaitHandle[] Start(int threads)
@@ -194,7 +131,7 @@ public class DbHandler
             
             waitHandles[i] = handle;
             
-            Thread f = new (DiscardedDbHandler!);
+            Thread f = new (RunDiscarded!);
             f.Start(discardedDbHandlerSetting);
             
             Thread.Sleep(1000);
@@ -203,7 +140,7 @@ public class DbHandler
         return waitHandles;
     }
     
-    private void DiscardedDbHandler(object obj)
+    private void RunDiscarded(object obj)
     {
         DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
@@ -226,7 +163,7 @@ public class DbHandler
         
         discardedDbHandlerSetting.Handle!.Set();
         
-        Console.WriteLine("Discarded DbHandler stopped.");
+        Console.WriteLine("Content DbHandler stopped.");
     }
     
     private void InsertUnfiltered(Unfiltered unfiltered)

Models/Model/Backend/DbType.cs

@@ -0,0 +1,7 @@
+namespace Models.Model.Backend;
+
+public enum DbType
+{
+    Unfiltered,
+    Filtered,
+}

Models/Model/Backend/Operations.cs

@@ -4,4 +4,5 @@ public enum Operations
 {
     Insert,
     Update,
+    Optimize,
 }

Models/Model/Backend/QueueItem.cs

@@ -0,0 +1,10 @@
+namespace Models.Model.Backend;
+
+public class QueueItem
+{
+    public Unfiltered Unfiltered { get; init; }
+    public Filtered? Filtered { get; init; }
+    public ScannerResumeObject? ResumeObject { get; init; }
+    public Operations Operations { get; init; }
+    public DbType DbType { get; init; }
+}

Models/Model/Backend/UnfilteredQueueItem.cs

@@ -1,7 +0,0 @@
-namespace Models.Model.Backend;
-
-public struct UnfilteredQueueItem
-{
-    public Unfiltered Unfiltered { get; init; }
-    public Operations Operations { get; init; }
-}