Create runtime variables and remove MessagePack.

This commit is contained in:
2024-11-26 19:40:27 +01:00
parent 28c5918a83
commit be07cdb13e
28 changed files with 336 additions and 200 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
+57 -32
View File
@@ -9,11 +9,11 @@ public class DbHandler
{
private readonly ConcurrentQueue<QueueItem> _contentQueue;
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private const string UnfilteredConnectionString = "Data Source=../../../../Models/mydb.db";
private const string DiscardedConnectionString = "Data Source=../../../../Models/Discarded.db";
private const string FilteredConnectionString = "Data Source=../../../../Models/Filtered.db";
private const string ResumeConnectionString = "Data Source=../../../../Models/ScannerResume.db";
private readonly string _unfilteredConnectionString;
private readonly string _discardedConnectionString;
private readonly string _filteredConnectionString;
private readonly string _resumeConnectionString;
private readonly List<string> _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)";
@@ -33,18 +33,43 @@ public class DbHandler
private const string ReIndexDatabasesStatement = "REINDEX;";
private const string VacuumDatabasesStatement = "VACUUM;";
private readonly object _readFilteredLock = new();
private readonly object _readAndDeleteResumeLock = new();
private bool _stop;
private bool _pause;
private bool _paused;
private int _contentWaitTime;
private int _discardedWaitTime;
private readonly string _basePath;
public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue)
public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
{
_contentQueue = contentQueue;
_discardedQueue = discardedQueue;
SetContentWaitTime(10);
SetDiscardedWaitTime(10);
_basePath = basePath;
_unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
_discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
_filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
_resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
}
public void SetContentWaitTime(int waitTime)
{
_contentWaitTime = waitTime;
}
public void SetDiscardedWaitTime(int waitTime)
{
_discardedWaitTime = waitTime;
}
public void StartContent()
@@ -55,7 +80,7 @@ public class DbHandler
{
if (_contentQueue.IsEmpty || _pause)
{
Thread.Sleep(10);
Thread.Sleep(_contentWaitTime);
_paused = true;
continue;
}
@@ -120,7 +145,7 @@ public class DbHandler
{
if (_discardedQueue.IsEmpty || _pause)
{
Thread.Sleep(10);
Thread.Sleep(_discardedWaitTime);
_paused = true;
continue;
}
@@ -137,9 +162,9 @@ public class DbHandler
Console.WriteLine("Content DbHandler stopped.");
}
private static void InsertUnfiltered(Unfiltered unfiltered)
private void InsertUnfiltered(Unfiltered unfiltered)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertStatement, connection);
@@ -168,9 +193,9 @@ public class DbHandler
connection.Close();
}
private static void InsertFiltered(Filtered filtered)
private void InsertFiltered(Filtered filtered)
{
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoFiltered, connection);
@@ -209,9 +234,9 @@ public class DbHandler
connection.Close();
}
private static void InsertResumeObject(ScannerResumeObject resumeObject)
private void InsertResumeObject(ScannerResumeObject resumeObject)
{
using SqliteConnection connection = new(ResumeConnectionString);
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoResume, connection);
@@ -228,9 +253,9 @@ public class DbHandler
connection.Close();
}
private static void UpdateUnfiltered(Unfiltered unfiltered)
private void UpdateUnfiltered(Unfiltered unfiltered)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(UpdateUnfilteredStatement, connection);
@@ -241,9 +266,9 @@ public class DbHandler
connection.Close();
}
public static Unfiltered? ReadUnfilteredWithId(long id)
public Unfiltered? ReadUnfilteredWithId(long id)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredStatement, connection);
@@ -266,11 +291,11 @@ public class DbHandler
return unfiltered;
}
public static long GetUnfilteredIndexes()
public long GetUnfilteredIndexes()
{
long rowId = 0;
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredIdsStatement, connection);
@@ -289,11 +314,11 @@ public class DbHandler
return rowId;
}
public static long GetFilteredIndexes()
public long GetFilteredIndexes()
{
long rowId = 0;
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
@@ -347,7 +372,7 @@ public class DbHandler
{
lock (_readFilteredLock)
{
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredStatement, connection);
@@ -377,7 +402,7 @@ public class DbHandler
{
lock (_readAndDeleteResumeLock)
{
using SqliteConnection connection = new(ResumeConnectionString);
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
@@ -417,21 +442,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(DiscardedConnectionString);
SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(FilteredConnectionString);
connection = new(_filteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(UnfilteredConnectionString);
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
@@ -456,21 +481,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(DiscardedConnectionString);
SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(FilteredConnectionString);
connection = new(_filteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(UnfilteredConnectionString);
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
@@ -486,7 +511,7 @@ public class DbHandler
private string CreateDiscardedDb(int threadNumber)
{
string databaseName = $"Data Source=../../../../Models/Discarded{threadNumber}.db";
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+2 -8
View File
@@ -1,16 +1,10 @@
using MessagePack;
namespace Models.Model.Backend;
[MessagePackObject]
public struct DatabaseSizes
{
[Key(0)]
public double DiscardedDbSize { get; set; }
[Key(1)]
public double FilteredDbSize { get; set; }
[Key(2)]
public double MyDbSize { get; set; }
}
+9
View File
@@ -0,0 +1,9 @@
namespace Models.Model.Backend;
public enum RuntimeVariable
{
DbContent,
DbDiscarded,
ContentFilter,
ScannerTimeout
}
+1
View File
@@ -7,4 +7,5 @@ public enum CommunicationCommand
StopScanning,
DbReindex,
DbVacuum,
ChangeRuntimeVariable,
}
+10 -6
View File
@@ -1,13 +1,17 @@
using MessagePack;
namespace Models.Model.External;
[MessagePackObject]
//[MessagePackObject]
public class CommunicationObject
{
[Key(0)]
//[Key(0)]
public CommunicationCommand Command { get; set; }
[Key(1)]
public string? SearchTerm { get; set; }
//[Key(1)]
public string? SearchTerm { get; set; } = "";
//[Key(2)]
public string? Variable { get; set; } = "";
//[Key(3)]
public string? VariableValue { get; set; } = "";
}
-5
View File
@@ -1,13 +1,8 @@
using MessagePack;
namespace Models.Model.External;
[MessagePackObject]
public class CommunicationResult
{
[Key(0)]
public List<SearchResult?>? Result { get; set; }
[Key(1)]
public ScanningStatus? Status { get; set; }
}
+2 -13
View File
@@ -1,29 +1,18 @@
using MessagePack;
using Models.Model.Backend;
namespace Models.Model.External;
[MessagePackObject]
public struct ScanningStatus
{
[Key(0)]
public float PercentageOfIpv4Scanned { get; set; }
[Key(1)]
public long TotalFiltered { get; set; }
[Key(2)]
public long AmountOfIpv4Left { get; set; }
[Key(3)]
public long TotalDiscarded { get; set; }
[Key(4)]
public double DiscardedDbSize { get; set; }
[Key(5)]
public double FilteredDbSize { get; set; }
[Key(6)]
public double MyDbSize { get; set; }
}
-6
View File
@@ -7,12 +7,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MessagePack" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePack.Annotations" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePackAnalyzer" Version="3.0.238-rc.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
<PackageReference Include="SQLite" Version="3.13.0" />
</ItemGroup>
Binary file not shown.
BIN
View File
Binary file not shown.