Made a lot of changes. Enhanced memory usage.

Models/Experimental/CustomPing.cs

@@ -0,0 +1,39 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using Models.Model.Backend;
+namespace Models.Experimental;
+public class CustomPing
+{
+    public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
+    {
+        SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
+        using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
+        int ipHeaderLength = 20;
+            
+        try
+        {
+            rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
+            byte[] numArray = new byte[136];
+            long timestamp = Stopwatch.GetTimestamp();
+            
+            // TODO: WTF ???
+            EndPoint lol = socketConfig.EndPoint;
+                
+            while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
+            {
+                int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
+                    
+                IPStatus status;
+                if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
+                    return status;
+            }
+        }
+        catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
+        {
+        }
+            
+        return IPStatus.TimedOut;
+    }
+}

Models/Experimental/MessageConstant.cs

@@ -0,0 +1,59 @@
+using System.Net.NetworkInformation;
+namespace Models.Experimental;
+public struct MessageConstant
+{
+  public static IPStatus MapV4TypeToIpStatus(int type, int code)
+  {
+    IPStatus ipStatus1;
+    switch ((IcmpV4MessageType) type)
+    {
+      case IcmpV4MessageType.EchoReply:
+        ipStatus1 = IPStatus.Success;
+        break;
+      case IcmpV4MessageType.DestinationUnreachable:
+        IPStatus ipStatus2;
+        switch ((byte) code)
+        {
+          case 0:
+            ipStatus2 = IPStatus.DestinationNetworkUnreachable;
+            break;
+          case 1:
+            ipStatus2 = IPStatus.DestinationHostUnreachable;
+            break;
+          case 2:
+            ipStatus2 = IPStatus.DestinationProtocolUnreachable;
+            break;
+          case 3:
+            ipStatus2 = IPStatus.DestinationPortUnreachable;
+            break;
+          default:
+            ipStatus2 = IPStatus.DestinationUnreachable;
+            break;
+        }
+        ipStatus1 = ipStatus2;
+        break;
+      case IcmpV4MessageType.SourceQuench:
+        ipStatus1 = IPStatus.SourceQuench;
+        break;
+      case IcmpV4MessageType.TimeExceeded:
+        ipStatus1 = IPStatus.TtlExpired;
+        break;
+      case IcmpV4MessageType.ParameterProblemBadIpHeader:
+        ipStatus1 = IPStatus.BadHeader;
+        break;
+      default:
+        ipStatus1 = IPStatus.Unknown;
+        break;
+    }
+    return ipStatus1;
+  }
+}
+
+internal enum IcmpV4MessageType : byte
+{
+  EchoReply = 0,
+  DestinationUnreachable = 3,
+  SourceQuench = 4,
+  TimeExceeded = 11, // 0x0B
+  ParameterProblemBadIpHeader = 12, // 0x0C
+}

Models/Experimental/RawSocket.cs

@@ -0,0 +1,91 @@
+using System.Net;
+using System.Net.NetworkInformation;
+using System.Net.Sockets;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using Models.Model.Backend;
+
+namespace Models.Experimental;
+
+public class RawSocket
+{
+    public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
+    {
+        Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
+        rawSocket.ReceiveTimeout = socketConfig.Timeout;
+        rawSocket.SendTimeout = socketConfig.Timeout;
+        rawSocket.Connect(socketConfig.EndPoint);
+        int num = 1;
+        rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
+        return rawSocket;
+    }
+    
+    public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
+    {
+        byte num = (byte) (receiveBuffer[0] & 15U);
+        ipHeaderLength = 4 * num;
+        
+        int start = ipHeaderLength;
+        int srcOffset = ipHeaderLength + 8;
+        
+        IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
+        byte[] numArray = new byte[bytesReceived - srcOffset];
+        Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
+        reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
+        return true;
+    }
+    
+    public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
+    {
+        int length = sizeof (IcmpHeader);
+        byte[] sendMessageBuffer = new byte[length];
+        new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
+        ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
+        sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
+        sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
+        return sendMessageBuffer;
+    }
+    
+    private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
+    {
+        uint num1 = 0;
+        for (int index = 0; index < buffer.Length; index += 2)
+        {
+            ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
+            num1 += num2;
+        }
+        while (num1 >> 16 != 0U)
+            num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
+        return (ushort) ~num1;
+    }
+}
+
+public class SocketConfig
+{
+    public EndPoint EndPoint;
+    public readonly int Timeout;
+    public readonly CustomProtocolType ProtocolType;
+    public readonly byte[] SendBuffer;
+
+    public SocketConfig(
+        EndPoint endPoint,
+        int timeout,
+        CustomProtocolType protocolType,
+        byte[] sendBuffer)
+    {
+        EndPoint = endPoint;
+        Timeout = timeout;
+        ProtocolType = protocolType;
+        SendBuffer = sendBuffer;
+    }
+}
+
+
+public struct IcmpHeader
+{
+    public byte Type;
+    public byte Code;
+    public ushort HeaderChecksum;
+    public ushort Identifier;
+    public ushort SequenceNumber;
+}

Models/Handler/DbHandler.cs

@@ -13,8 +13,10 @@ public class DbHandler
     private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
     private readonly ConcurrentQueue<Discarded> _discardedQueue;
     private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
+    private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
     
     private readonly string _unfilteredConnectionString;
+    private readonly string _preFilteredConnectionString;
     private readonly string _filteredConnectionString;
     private readonly string _resumeConnectionString;
     private readonly string _compressedConnectionString;
@@ -25,6 +27,11 @@ public class DbHandler
                                            " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
                                            " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     
+    private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
+                                           " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
+                                           " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
+                                           " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
+    
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
                                               " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
                                               " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
@@ -88,6 +95,9 @@ public class DbHandler
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
     private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
+    private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
+    private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
+    private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
     
     private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
     
@@ -113,12 +123,15 @@ public class DbHandler
     public DbHandler(ConcurrentQueue<Filtered> filteredQueue, 
         ConcurrentQueue<Discarded> discardedQueue, 
         ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
-        ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
+        ConcurrentQueue<ScannerResumeObject> resumeQueue, 
+        ConcurrentQueue<FilterQueueItem> preFilteredQueue,
+        string basePath)
     {
         _filteredQueue = filteredQueue;
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
+        _preFilteredQueue = preFilteredQueue;
 
         SetContentWaitTime(100);
         SetDiscardedWaitTime(10);
@@ -129,6 +142,7 @@ public class DbHandler
         _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
         _resumeConnectionString  = $"Data Source={basePath}/Models/ScannerResume.db";
         _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
+        _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
     }
     
     public void SetContentWaitTime(int waitTime)
@@ -190,6 +204,24 @@ public class DbHandler
         
         Console.WriteLine("Filtered DbHandler stopped.");
     }
+
+    public void PrefilteredDbHandler()
+    {
+        Console.WriteLine("PreFiltered Db handler started.");
+        
+        while (!_stop)
+        {
+            if (_preFilteredQueue.IsEmpty)
+            {
+                Thread.Sleep(4);
+                continue;
+            }
+            
+            _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
+            
+            InsertPrefiltered(queueItem);
+        }
+    }
     
     public void ResumeDbHandler()
     {
@@ -204,12 +236,9 @@ public class DbHandler
                 continue;
             }
             
-            _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
+            _resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
             
-            if (queueItem is not null)
-            {
-                InsertResumeObject(queueItem);
-            }
+            InsertResumeObject(queueItem);
         }
         
         Console.WriteLine("Resume DbHandler stopped.");
@@ -499,6 +528,24 @@ public class DbHandler
         _ = command.ExecuteNonQuery();
         connection.Close();
     }
+
+    private void InsertPrefiltered(FilterQueueItem filterQueueItem)
+    {
+        using SqliteConnection connection = new(_preFilteredConnectionString);
+        connection.Open();
+        
+        using SqliteCommand command = new(InsertPreFilteredStatement, connection);
+        
+        command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
+        command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
+        command.Parameters.AddWithValue("@filtered", 0);
+        
+        _ = command.ExecuteNonQuery();
+        connection.Close();
+    }
     
     private void UpdateUnfiltered(Unfiltered unfiltered)
     {
@@ -567,79 +614,42 @@ public class DbHandler
         return ids;
     }
     
-    public long GetFilteredIndexes()
+    public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
     {
-        long rowId = 0;
-        
-        using SqliteConnection connection = new(_filteredConnectionString);
+        using SqliteConnection connection = new(_preFilteredConnectionString);
         connection.Open();
         
-        using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
+        SqliteCommand command = new(ReadPreFilteredStatement, connection);
         using SqliteDataReader reader = command.ExecuteReader();
         
+        filterQueueItem = new();
+        Ip ip = new();
+        long id = 0;
+        
         if (!reader.HasRows)
         {
-            return 0;
+            return false;
         }
         
         while (reader.Read())
         {
-            rowId = reader.GetInt64(0);
+            ip.Ip1 = reader.GetInt32(0);
+            ip.Ip2 = reader.GetInt32(1);
+            ip.Ip3 = reader.GetInt32(2);
+            ip.Ip4 = reader.GetInt32(3);
+            filterQueueItem.ResponseCode = reader.GetInt32(4);
+            id = reader.GetInt64(5);
         }
         
-        return rowId;
-    }
-    
-    public long GetDiscardedIndexes()
-    {
-        long rowId = 0;
+        filterQueueItem.Ip = ip;
+        
+        command = new(UpdatePreFilteredStatement, connection);
+        command.Parameters.AddWithValue("@id", id);
 
-        SqliteConnection connection;
-        SqliteCommand command;
-        SqliteDataReader reader;
-        
-        for (int i = 0; i < _discardedConnectionStrings.Count; i++)
-        {
-            connection = new(_discardedConnectionStrings[i]);
-            connection.Open();
-            
-            command = new(ReadDiscardedSeqIdsStatement, connection);
-            reader = command.ExecuteReader();
-            
-            if (!reader.HasRows)
-            {
-                return rowId;
-            }
-            
-            while (reader.Read())
-            {
-                rowId += reader.GetInt64(0);
-            }
-            
-            connection.Close();
-        }
-        
-        connection = new(_compressedConnectionString);
-        connection.Open();
-        command = new(ReadCompressedDbRowsStatement, connection);
-        reader = command.ExecuteReader();
-        
-        if (!reader.HasRows)
-        {
-            return rowId;
-        }
-        
-        while (reader.Read())
-        {
-            rowId += reader.GetInt64(0);
-        }
-        
-        connection.Close();
-        connection.Dispose();
+        command.ExecuteNonQuery();
         command.Dispose();
-        reader.Dispose();
         
-        return rowId;
+        return true;
     }
 
     private static long GetDiscardedIndexesForSpecificDb(string connectionString)

Models/Model/Backend/CustomProtocolType.cs

@@ -0,0 +1,6 @@
+namespace Models.Model.Backend;
+
+public enum CustomProtocolType
+{
+    Icmp = 1
+}

Models/Model/Backend/DatabaseSizes.cs

@@ -1,10 +0,0 @@
-namespace Models.Model.Backend;
-
-public struct DatabaseSizes
-{
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/Backend/FilterQueueItem.cs

@@ -2,6 +2,6 @@ namespace Models.Model.Backend;
 
 public struct FilterQueueItem
 {
-    public Ip Ip { get; init; }
-    public int ResponseCode { get; init; }
+    public Ip Ip { get; set; }
+    public int ResponseCode { get; set; }
 }

Models/Model/Backend/RuntimeVariable.cs

@@ -1,9 +0,0 @@
-namespace Models.Model.Backend;
-
-public enum RuntimeVariable
-{
-    DbContent,
-    DbDiscarded,
-    ContentFilter,
-    ScannerTimeout
-}

Models/Model/Backend/SizeUnits.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.Backend;
-public enum SizeUnits
-{
-    Byte,
-    KB,
-    MB,
-    GB,
-}

Models/Model/External/CommunicationCommand.cs

@@ -1,11 +0,0 @@
-namespace Models.Model.External;
-
-public enum CommunicationCommand
-{
-    GetScanningProgress,
-    GetSearches,
-    StopScanning,
-    DbReindex,
-    DbVacuum,
-    ChangeRuntimeVariable,
-}

Models/Model/External/CommunicationObject.cs

@@ -1,17 +0,0 @@
-namespace Models.Model.External;
-
-//[MessagePackObject]
-public class CommunicationObject
-{
-    //[Key(0)]
-    public CommunicationCommand Command { get; set; }
-    
-    //[Key(1)]
-    public string? SearchTerm { get; set; } = "";
-    
-    //[Key(2)]
-    public string? Variable { get; set; } = "";
-    
-    //[Key(3)]
-    public string? VariableValue { get; set; } = "";
-}

Models/Model/External/CommunicationResult.cs

@@ -1,8 +0,0 @@
-namespace Models.Model.External;
-
-public class CommunicationResult
-{
-    public List<SearchResult?>? Result { get; set; }
-    
-    public ScanningStatus? Status { get; set; }
-}

Models/Model/External/ScanningStatus.cs

@@ -1,18 +0,0 @@
-namespace Models.Model.External;
-
-public struct ScanningStatus
-{
-    public float PercentageOfIpv4Scanned { get; set; }
-    
-    public long TotalFiltered { get; set; }
-    
-    public long AmountOfIpv4Left { get; set; }
-    
-    public long TotalDiscarded { get; set; }
-    
-    public double DiscardedDbSize { get; set; }
-    
-    public double FilteredDbSize { get; set; }
-    
-    public double MyDbSize { get; set; }
-}

Models/Model/External/SearchResults.cs

@@ -1,6 +0,0 @@
-namespace Models.Model.External;
-
-public class SearchResults
-{
-    public List<SearchResult?>? Results { get; set; }
-}

Models/Models.csproj

@@ -1,12 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
+      <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
     </ItemGroup>
 </Project>