diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj
index f445ed4..87db91a 100644
--- a/Backend/Backend.csproj
+++ b/Backend/Backend.csproj
@@ -16,12 +16,6 @@
-
-
-
- all
- runtime; build; native; contentfiles; analyzers; buildtransitive
-
diff --git a/Backend/Handler/Communication.cs b/Backend/Handler/Communication.cs
index a22971f..6086039 100644
--- a/Backend/Handler/Communication.cs
+++ b/Backend/Handler/Communication.cs
@@ -1,7 +1,6 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Backend.Helper;
-using MessagePack;
using Models.Handler;
using Models.Model.Backend;
using Models.Model.External;
@@ -14,12 +13,18 @@ public class Communication
{
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
+ private readonly IpScanner _ipScanner;
+ private readonly ContentFilter _contentFilter;
private bool _isRunning = true;
+ private string _basePath;
- public Communication(DbHandler dbHandler, ThreadHandler threadHandler)
+ public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
{
_dbHandler = dbHandler;
_threadHandler = threadHandler;
+ _ipScanner = ipScanner;
+ _contentFilter = contentFilter;
+ _basePath = basePath;
}
public WaitHandle[] Start()
@@ -43,15 +48,20 @@ public class Communication
while (_isRunning)
{
byte[] message = rep.ReceiveFrameBytes();
+
+ CommunicationObject? communicationObject = JsonSerializer.Deserialize(message);
- CommunicationObject communicationObject = MessagePackSerializer.Deserialize(message);
-
+ if (communicationObject is null)
+ {
+ continue;
+ }
+
OnServerOnReceiveReady(communicationObject, rep);
}
((EventWaitHandle) obj).Set();
}
-
+
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")]
private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
@@ -60,8 +70,8 @@ public class Communication
{
case CommunicationCommand.GetScanningProgress:
{
- DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes();
-
+ DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
+
long discardedIndexes = _dbHandler.GetDiscardedIndexes();
ScanningStatus status = new();
@@ -77,13 +87,13 @@ public class Communication
}
status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
- status.TotalFiltered = DbHandler.GetFilteredIndexes();
+ status.TotalFiltered = _dbHandler.GetFilteredIndexes();
status.TotalDiscarded = discardedIndexes;
status.MyDbSize = databaseSizes.MyDbSize;
status.FilteredDbSize = databaseSizes.FilteredDbSize;
status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
-
- byte[] serializedResult = MessagePackSerializer.Serialize(status, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
+
+ byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
rep.SendFrame(serializedResult);
@@ -93,7 +103,7 @@ public class Communication
case CommunicationCommand.DbReindex:
{
_dbHandler.ReIndex();
-
+
SendStringResponse(rep, "All Dbs have been reindexed.");
break;
}
@@ -101,7 +111,7 @@ public class Communication
case CommunicationCommand.DbVacuum:
{
_dbHandler.Vacuum();
-
+
SendStringResponse(rep, "All Dbs have been vacuumed.");
break;
}
@@ -121,13 +131,48 @@ public class Communication
_isRunning = false;
break;
}
+
+ case CommunicationCommand.ChangeRuntimeVariable:
+ {
+ Console.WriteLine("lmao");
+
+ if (string.IsNullOrWhiteSpace(communicationObject.VariableValue))
+ {
+ break;
+ }
+
+ if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
+ {
+ int value = int.Parse(communicationObject.VariableValue);
+ _dbHandler.SetContentWaitTime(value);
+ }
+
+ if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
+ {
+ int value = int.Parse(communicationObject.VariableValue);
+ _dbHandler.SetDiscardedWaitTime(value);
+ }
+
+ if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
+ {
+ int value = int.Parse(communicationObject.VariableValue);
+ _ipScanner.SetTimeout(value);
+ }
+
+ if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
+ {
+ int value = int.Parse(communicationObject.VariableValue);
+ _contentFilter.SetTimeout(value);
+ }
+
+ break;
+ }
}
}
private static void SendStringResponse(ResponseSocket rep, string response)
{
- MessagePackSerializerOptions withCompression = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray);
- byte[] serializedResult = MessagePackSerializer.Serialize(response, withCompression);
+ byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
rep.SendFrame(serializedResult);
}
diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs
index a1287a6..f5be79d 100644
--- a/Backend/Handler/ContentFilter.cs
+++ b/Backend/Handler/ContentFilter.cs
@@ -11,14 +11,25 @@ public class ContentFilter
{
private readonly ConcurrentQueue _queue;
private readonly DbHandler _dbHandler;
- private const string GetDomainPort80 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort80.sh";
- private const string GetDomainPort443 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort443.sh";
+ private readonly string _getDomainPort80;
+ private readonly string _getDomainPort443;
private bool _stop;
+ private int _timeOut;
+ private string _basePath;
- public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler)
+ public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler, string basePath)
{
_queue = queue;
_dbHandler = dbHandler;
+ _basePath = basePath;
+
+ _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
+ _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
+ }
+
+ public void SetTimeout(int timeOut)
+ {
+ _timeOut = timeOut;
}
public WaitHandle[] Start()
@@ -34,49 +45,54 @@ public class ContentFilter
private void Filter(object obj)
{
- long indexes = DbHandler.GetUnfilteredIndexes();
-
- for (long i = 0; i < indexes; i++)
+ while (!_stop)
{
- if (_stop) break;
+ long indexes = _dbHandler.GetUnfilteredIndexes();
- Unfiltered? unfiltered = DbHandler.ReadUnfilteredWithId(i);
-
- if (unfiltered is null || unfiltered.Filtered == 1) continue;
-
- unfiltered.Filtered = 1;
-
- QueueItem superUnfilteredObject = new()
+ for (long i = 0; i < indexes; i++)
{
- Unfiltered = unfiltered,
- Operations = Operations.Update
- };
-
- _queue.Enqueue(superUnfilteredObject);
-
- if (_dbHandler.GetFilteredIp(unfiltered.Ip))
- {
- continue;
+ if (_stop) break;
+
+ Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i);
+
+ if (unfiltered is null || unfiltered.Filtered == 1) continue;
+
+ unfiltered.Filtered = 1;
+
+ QueueItem superUnfilteredObject = new()
+ {
+ Unfiltered = unfiltered,
+ Operations = Operations.Update
+ };
+
+ _queue.Enqueue(superUnfilteredObject);
+
+ if (_dbHandler.GetFilteredIp(unfiltered.Ip))
+ {
+ continue;
+ }
+
+ Filtered filtered = GetSiteData(unfiltered.Ip);
+
+ filtered.Port1 = unfiltered.Port1;
+ filtered.Port2 = unfiltered.Port2;
+
+ QueueItem superFilteredObject = new()
+ {
+ Filtered = filtered,
+ Operations = Operations.Insert
+ };
+
+ _queue.Enqueue(superFilteredObject);
}
- Filtered filtered = GetSiteData(unfiltered.Ip);
-
- filtered.Port1 = unfiltered.Port1;
- filtered.Port2 = unfiltered.Port2;
-
- QueueItem superFilteredObject = new()
- {
- Filtered = filtered,
- Operations = Operations.Insert
- };
-
- _queue.Enqueue(superFilteredObject);
+ Thread.Sleep(_timeOut);
}
((EventWaitHandle) obj).Set();
}
- private static Filtered GetSiteData(string ip)
+ private Filtered GetSiteData(string ip)
{
StartProcess(ip, 80);
StartProcess(ip, 443);
@@ -112,7 +128,7 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
- using StreamReader streamReader = new($"{ports[i]}Header.txt");
+ using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
while (streamReader.Peek() != -1)
{
@@ -204,9 +220,9 @@ public class ContentFilter
return siteData;
}
- private static void StartProcess(string ip, int port)
+ private void StartProcess(string ip, int port)
{
- string fileName = port == 80 ? GetDomainPort80 : GetDomainPort443;
+ string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
Process proc = new();
proc.StartInfo = new()
diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs
index 750260b..32fb10a 100644
--- a/Backend/Handler/IpScanner.cs
+++ b/Backend/Handler/IpScanner.cs
@@ -22,12 +22,21 @@ public class IpScanner
private readonly ConcurrentQueue _discardedQueue;
private readonly DbHandler _dbHandler;
private bool _stop;
+ private int _timeout;
public IpScanner(ConcurrentQueue queue, DbHandler dbHandler, ConcurrentQueue discardedQueue)
{
_queue = queue;
_dbHandler = dbHandler;
_discardedQueue = discardedQueue;
+
+ SetTimeout(128);
+ }
+
+ public void SetTimeout(int milliseconds)
+ {
+ Console.WriteLine($"Setting timeout to {milliseconds}ms");
+ _timeout = milliseconds;
}
public WaitHandle[] Start(int threads)
@@ -133,7 +142,7 @@ public class IpScanner
_ = IPAddress.TryParse(ip, out IPAddress? address);
if (address is not null)
{
- responseCode = ping.Send(address, 512, buf, null).Status;
+ responseCode = ping.Send(address, _timeout, buf, null).Status;
}
}
catch (Exception e)
diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs
index 90a171b..f742acb 100644
--- a/Backend/Handler/ThreadHandler.cs
+++ b/Backend/Handler/ThreadHandler.cs
@@ -14,23 +14,22 @@ public class ThreadHandler
private bool _communicationStopped;
private bool _ipScannerStopped;
private bool _contentFilterStopped;
- private bool _stopSignal;
- public ThreadHandler()
+ public ThreadHandler(string path)
{
ConcurrentQueue contentQueue = new();
ConcurrentQueue discardedQueue = new();
- _dbHandler = new(contentQueue, discardedQueue);
- _communication = new(_dbHandler, this);
+ _dbHandler = new(contentQueue, discardedQueue, path);
_ipScanner = new(contentQueue, _dbHandler, discardedQueue);
- _contentFilter = new(contentQueue, _dbHandler);
+ _contentFilter = new(contentQueue, _dbHandler, path);
+ _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
}
public void Start()
{
Thread scanner = new(StartScanner);
- Thread indexer = new(StartIndexer);
+ Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
Thread communication = new(StartCommunicationHandler);
@@ -48,18 +47,11 @@ public class ThreadHandler
communication.Join();
}
- public static void ManualGc()
- {
- GC.Collect();
- GC.WaitForPendingFinalizers();
- GC.Collect();
- }
-
private void StartScanner()
{
Thread.Sleep(10000); // Let the database handler instantiate and warm up first.
- WaitHandle[] wait = _ipScanner.Start(4);
+ WaitHandle[] wait = _ipScanner.Start(2);
WaitHandle.WaitAll(wait);
@@ -68,18 +60,13 @@ public class ThreadHandler
_ipScannerStopped = true;
}
- private void StartIndexer()
+ private void StartContentFilter()
{
- while (!_stopSignal)
- {
- WaitHandle[] wait = _contentFilter.Start();
-
- WaitHandle.WaitAll(wait);
-
- Thread.Sleep(300000); // 5 minutes
- }
+ WaitHandle[] wait = _contentFilter.Start();
- Console.WriteLine("Indexer finished");
+ WaitHandle.WaitAll(wait);
+
+ Console.WriteLine("Content filter finished");
_contentFilterStopped = true;
}
@@ -113,7 +100,6 @@ public class ThreadHandler
private void Stop()
{
- _stopSignal = true;
_ipScanner.Stop();
_contentFilter.Stop();
diff --git a/Backend/Helper/FilesystemHelper.cs b/Backend/Helper/FilesystemHelper.cs
index efc42f6..b74cd9b 100644
--- a/Backend/Helper/FilesystemHelper.cs
+++ b/Backend/Helper/FilesystemHelper.cs
@@ -4,24 +4,24 @@ namespace Backend.Helper;
public static class FilesystemHelper
{
- public static DatabaseSizes GetDatabaseSizes()
+ public static DatabaseSizes GetDatabaseSizes(string basePath)
{
DatabaseSizes databaseSizes = new();
- FileInfo fileInfo = new("../../../../Models/mydb.db");
+ FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
- databaseSizes.DiscardedDbSize = GetDiscardedDbSizes().ToSize(SizeUnits.KB);
+ databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
- fileInfo = new("../../../../Models/Filtered.db");
+ fileInfo = new($"{basePath}/Models/Filtered.db");
databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
return databaseSizes;
}
- private static long GetDiscardedDbSizes()
+ private static long GetDiscardedDbSizes(string basePath)
{
- const string folder = "../../../../Models";
+ string folder = $"{basePath}/Models";
string[] files = Directory.GetFiles(folder, "*.db");
long size = 0;
diff --git a/Backend/Helper/ProjectPathHelper.cs b/Backend/Helper/ProjectPathHelper.cs
new file mode 100644
index 0000000..8b535ef
--- /dev/null
+++ b/Backend/Helper/ProjectPathHelper.cs
@@ -0,0 +1,9 @@
+namespace Backend.Helper;
+
+public static class ProjectPathHelper
+{
+ public static string GetProjectPath()
+ {
+ return new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory).Parent!.Parent!.Parent!.Parent!.FullName;
+ }
+}
\ No newline at end of file
diff --git a/Backend/Program.cs b/Backend/Program.cs
index dd999b7..fb54b6e 100644
--- a/Backend/Program.cs
+++ b/Backend/Program.cs
@@ -1,8 +1,9 @@
using Backend.Handler;
+using Backend.Helper;
Console.WriteLine("Program started");
-ThreadHandler threadHandler = new();
+ThreadHandler threadHandler = new(ProjectPathHelper.GetProjectPath());
threadHandler.Start();
Console.WriteLine("Done");
\ No newline at end of file
diff --git a/Manager/Commands.cs b/Manager/Commands.cs
index fe3296b..9837684 100644
--- a/Manager/Commands.cs
+++ b/Manager/Commands.cs
@@ -1,4 +1,6 @@
-using MessagePack;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Models.Model.Backend;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
@@ -32,7 +34,7 @@ public static class Commands
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
-
+
public static void Vacuum()
{
CommunicationObject communicationObject = new();
@@ -48,6 +50,18 @@ public static class Commands
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
+
+ public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
+ {
+ CommunicationObject communicationObject = new()
+ {
+ Command = CommunicationCommand.ChangeRuntimeVariable,
+ Variable = runtimeVariable.ToString(),
+ VariableValue = value
+ };
+
+ SendAndRecieveStringMessage(communicationObject);
+ }
public static void GetHelp()
{
@@ -59,32 +73,53 @@ public static class Commands
Console.WriteLine(" g - manual garbage collect on the server");
Console.WriteLine(" r - manual reindex the databases");
Console.WriteLine(" v - manual vacuum the databases");
+ Console.WriteLine(" R - change runtime variable");
+ Console.WriteLine(" lR - list runtime variables");
Console.WriteLine(" help - shows this help");
Console.WriteLine();
}
- private static string SendAndRecieveStringMessage(CommunicationObject communicationObject)
+ public static void PrintRuntimeVariables()
{
- byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
-
- using RequestSocket client = new();
- client.Connect("tcp://127.0.0.1:5556");
- client.SendFrame(bytes);
- byte[] msg = client.ReceiveFrameBytes();
- client.Close();
- return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
+ Console.WriteLine("Runtime variables:");
+ Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()} - Sets the timeout in milliseconds for the scanner");
+ Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()} - Sets the timeout in milliseconds for the content filter");
+ Console.WriteLine($"{RuntimeVariable.DbContent.ToString()} - Sets the wait time in milliseconds for the content database if the queue is empty");
+ Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()} - Sets the wait time in milliseconds for the discarded database if the queue is empty");
}
- private static ScanningStatus GetProgress(CommunicationObject communicationObject)
+ [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")]
+ [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")]
+ private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
{
- byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
+ //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
+
+ byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
- client.SendFrame(bytes);
+ client.SendFrame(lol);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
- return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
+ return JsonSerializer.Deserialize(msg);
+ }
+
+ [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")]
+ [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")]
+ private static ScanningStatus GetProgress(CommunicationObject communicationObject)
+ {
+ //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
+
+ byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
+
+
+ using RequestSocket client = new();
+ client.Connect("tcp://127.0.0.1:5556");
+ client.SendFrame(lol);
+ byte[] msg = client.ReceiveFrameBytes();
+ client.Close();
+
+ return JsonSerializer.Deserialize(msg);
}
}
\ No newline at end of file
diff --git a/Manager/Manager.csproj b/Manager/Manager.csproj
index a0f2c15..6bed32a 100644
--- a/Manager/Manager.csproj
+++ b/Manager/Manager.csproj
@@ -5,14 +5,13 @@
net8.0
enable
enable
-
- true
- x64
+
+
+ true
true
link
- true
+ true-->
@@ -20,12 +19,6 @@
-
-
-
- all
- runtime; build; native; contentfiles; analyzers; buildtransitive
-
diff --git a/Manager/Program.cs b/Manager/Program.cs
index d1b8b2b..abd960a 100644
--- a/Manager/Program.cs
+++ b/Manager/Program.cs
@@ -1,4 +1,5 @@
using Manager;
+using Models.Model.Backend;
bool stop = false;
@@ -36,6 +37,43 @@ do
Commands.ReIndex();
}
+ else if (string.Equals(input, "R"))
+ {
+ string? variable = Console.ReadLine();
+ string? value = Console.ReadLine();
+
+ if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
+ {
+ Console.WriteLine("Please enter a value.");
+ return;
+ }
+
+ if (variable == RuntimeVariable.ScannerTimeout.ToString())
+ {
+ Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
+ }
+
+ else if (variable == RuntimeVariable.ContentFilter.ToString())
+ {
+ Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
+ }
+
+ else if (variable == RuntimeVariable.DbDiscarded.ToString())
+ {
+ Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
+ }
+
+ else if (variable == RuntimeVariable.DbContent.ToString())
+ {
+ Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
+ }
+ }
+
+ else if (string.Equals(input, "lR"))
+ {
+ Commands.PrintRuntimeVariables();
+ }
+
else if (string.Equals(input, "help"))
{
Commands.GetHelp();
diff --git a/Models/Discarded0.db b/Models/Discarded0.db
index ac1f416..da3fc68 100644
Binary files a/Models/Discarded0.db and b/Models/Discarded0.db differ
diff --git a/Models/Discarded1.db b/Models/Discarded1.db
index ad218d2..51d8865 100644
Binary files a/Models/Discarded1.db and b/Models/Discarded1.db differ
diff --git a/Models/Filtered.db b/Models/Filtered.db
index 51c8926..48380b1 100644
Binary files a/Models/Filtered.db and b/Models/Filtered.db differ
diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs
index a478acf..8ca07ad 100644
--- a/Models/Handler/DbHandler.cs
+++ b/Models/Handler/DbHandler.cs
@@ -9,11 +9,11 @@ public class DbHandler
{
private readonly ConcurrentQueue _contentQueue;
private readonly ConcurrentQueue _discardedQueue;
-
- private const string UnfilteredConnectionString = "Data Source=../../../../Models/mydb.db";
- private const string DiscardedConnectionString = "Data Source=../../../../Models/Discarded.db";
- private const string FilteredConnectionString = "Data Source=../../../../Models/Filtered.db";
- private const string ResumeConnectionString = "Data Source=../../../../Models/ScannerResume.db";
+
+ private readonly string _unfilteredConnectionString;
+ private readonly string _discardedConnectionString;
+ private readonly string _filteredConnectionString;
+ private readonly string _resumeConnectionString;
private readonly List _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)";
@@ -33,18 +33,43 @@ public class DbHandler
private const string ReIndexDatabasesStatement = "REINDEX;";
private const string VacuumDatabasesStatement = "VACUUM;";
-
+
private readonly object _readFilteredLock = new();
private readonly object _readAndDeleteResumeLock = new();
private bool _stop;
private bool _pause;
private bool _paused;
+
+ private int _contentWaitTime;
+ private int _discardedWaitTime;
+
+ private readonly string _basePath;
- public DbHandler(ConcurrentQueue contentQueue, ConcurrentQueue discardedQueue)
+ public DbHandler(ConcurrentQueue contentQueue, ConcurrentQueue discardedQueue, string basePath)
{
_contentQueue = contentQueue;
_discardedQueue = discardedQueue;
+
+ SetContentWaitTime(10);
+ SetDiscardedWaitTime(10);
+
+ _basePath = basePath;
+
+ _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
+ _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
+ _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
+ _resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
+ }
+
+ public void SetContentWaitTime(int waitTime)
+ {
+ _contentWaitTime = waitTime;
+ }
+
+ public void SetDiscardedWaitTime(int waitTime)
+ {
+ _discardedWaitTime = waitTime;
}
public void StartContent()
@@ -55,7 +80,7 @@ public class DbHandler
{
if (_contentQueue.IsEmpty || _pause)
{
- Thread.Sleep(10);
+ Thread.Sleep(_contentWaitTime);
_paused = true;
continue;
}
@@ -120,7 +145,7 @@ public class DbHandler
{
if (_discardedQueue.IsEmpty || _pause)
{
- Thread.Sleep(10);
+ Thread.Sleep(_discardedWaitTime);
_paused = true;
continue;
}
@@ -137,9 +162,9 @@ public class DbHandler
Console.WriteLine("Content DbHandler stopped.");
}
- private static void InsertUnfiltered(Unfiltered unfiltered)
+ private void InsertUnfiltered(Unfiltered unfiltered)
{
- using SqliteConnection connection = new(UnfilteredConnectionString);
+ using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertStatement, connection);
@@ -168,9 +193,9 @@ public class DbHandler
connection.Close();
}
- private static void InsertFiltered(Filtered filtered)
+ private void InsertFiltered(Filtered filtered)
{
- using SqliteConnection connection = new(FilteredConnectionString);
+ using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoFiltered, connection);
@@ -209,9 +234,9 @@ public class DbHandler
connection.Close();
}
- private static void InsertResumeObject(ScannerResumeObject resumeObject)
+ private void InsertResumeObject(ScannerResumeObject resumeObject)
{
- using SqliteConnection connection = new(ResumeConnectionString);
+ using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoResume, connection);
@@ -228,9 +253,9 @@ public class DbHandler
connection.Close();
}
- private static void UpdateUnfiltered(Unfiltered unfiltered)
+ private void UpdateUnfiltered(Unfiltered unfiltered)
{
- using SqliteConnection connection = new(UnfilteredConnectionString);
+ using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(UpdateUnfilteredStatement, connection);
@@ -241,9 +266,9 @@ public class DbHandler
connection.Close();
}
- public static Unfiltered? ReadUnfilteredWithId(long id)
+ public Unfiltered? ReadUnfilteredWithId(long id)
{
- using SqliteConnection connection = new(UnfilteredConnectionString);
+ using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredStatement, connection);
@@ -266,11 +291,11 @@ public class DbHandler
return unfiltered;
}
- public static long GetUnfilteredIndexes()
+ public long GetUnfilteredIndexes()
{
long rowId = 0;
- using SqliteConnection connection = new(UnfilteredConnectionString);
+ using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredIdsStatement, connection);
@@ -289,11 +314,11 @@ public class DbHandler
return rowId;
}
- public static long GetFilteredIndexes()
+ public long GetFilteredIndexes()
{
long rowId = 0;
- using SqliteConnection connection = new(FilteredConnectionString);
+ using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
@@ -347,7 +372,7 @@ public class DbHandler
{
lock (_readFilteredLock)
{
- using SqliteConnection connection = new(FilteredConnectionString);
+ using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredStatement, connection);
@@ -377,7 +402,7 @@ public class DbHandler
{
lock (_readAndDeleteResumeLock)
{
- using SqliteConnection connection = new(ResumeConnectionString);
+ using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
@@ -417,21 +442,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
- SqliteConnection connection = new(DiscardedConnectionString);
+ SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
- connection = new(FilteredConnectionString);
+ connection = new(_filteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
- connection = new(UnfilteredConnectionString);
+ connection = new(_unfilteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
@@ -456,21 +481,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
- SqliteConnection connection = new(DiscardedConnectionString);
+ SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
- connection = new(FilteredConnectionString);
+ connection = new(_filteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
- connection = new(UnfilteredConnectionString);
+ connection = new(_unfilteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
@@ -486,7 +511,7 @@ public class DbHandler
private string CreateDiscardedDb(int threadNumber)
{
- string databaseName = $"Data Source=../../../../Models/Discarded{threadNumber}.db";
+ string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
diff --git a/Models/Model/Backend/DatabaseSizes.cs b/Models/Model/Backend/DatabaseSizes.cs
index 9720a0e..ba4c06b 100644
--- a/Models/Model/Backend/DatabaseSizes.cs
+++ b/Models/Model/Backend/DatabaseSizes.cs
@@ -1,16 +1,10 @@
-using MessagePack;
-
namespace Models.Model.Backend;
-[MessagePackObject]
public struct DatabaseSizes
{
- [Key(0)]
public double DiscardedDbSize { get; set; }
-
- [Key(1)]
+
public double FilteredDbSize { get; set; }
-
- [Key(2)]
+
public double MyDbSize { get; set; }
}
\ No newline at end of file
diff --git a/Models/Model/Backend/RuntimeVariable.cs b/Models/Model/Backend/RuntimeVariable.cs
new file mode 100644
index 0000000..536a86f
--- /dev/null
+++ b/Models/Model/Backend/RuntimeVariable.cs
@@ -0,0 +1,9 @@
+namespace Models.Model.Backend;
+
+public enum RuntimeVariable
+{
+ DbContent,
+ DbDiscarded,
+ ContentFilter,
+ ScannerTimeout
+}
\ No newline at end of file
diff --git a/Models/Model/External/CommunicationCommand.cs b/Models/Model/External/CommunicationCommand.cs
index e37fa86..cc314a1 100644
--- a/Models/Model/External/CommunicationCommand.cs
+++ b/Models/Model/External/CommunicationCommand.cs
@@ -7,4 +7,5 @@ public enum CommunicationCommand
StopScanning,
DbReindex,
DbVacuum,
+ ChangeRuntimeVariable,
}
\ No newline at end of file
diff --git a/Models/Model/External/CommunicationObject.cs b/Models/Model/External/CommunicationObject.cs
index a6f6231..9fa6609 100644
--- a/Models/Model/External/CommunicationObject.cs
+++ b/Models/Model/External/CommunicationObject.cs
@@ -1,13 +1,17 @@
-using MessagePack;
-
namespace Models.Model.External;
-[MessagePackObject]
+//[MessagePackObject]
public class CommunicationObject
{
- [Key(0)]
+ //[Key(0)]
public CommunicationCommand Command { get; set; }
- [Key(1)]
- public string? SearchTerm { get; set; }
+ //[Key(1)]
+ public string? SearchTerm { get; set; } = "";
+
+ //[Key(2)]
+ public string? Variable { get; set; } = "";
+
+ //[Key(3)]
+ public string? VariableValue { get; set; } = "";
}
\ No newline at end of file
diff --git a/Models/Model/External/CommunicationResult.cs b/Models/Model/External/CommunicationResult.cs
index 574a36b..95808d9 100644
--- a/Models/Model/External/CommunicationResult.cs
+++ b/Models/Model/External/CommunicationResult.cs
@@ -1,13 +1,8 @@
-using MessagePack;
-
namespace Models.Model.External;
-[MessagePackObject]
public class CommunicationResult
{
- [Key(0)]
public List? Result { get; set; }
- [Key(1)]
public ScanningStatus? Status { get; set; }
}
\ No newline at end of file
diff --git a/Models/Model/External/ScanningStatus.cs b/Models/Model/External/ScanningStatus.cs
index 3b94356..29a09e4 100644
--- a/Models/Model/External/ScanningStatus.cs
+++ b/Models/Model/External/ScanningStatus.cs
@@ -1,29 +1,18 @@
-using MessagePack;
-using Models.Model.Backend;
-
namespace Models.Model.External;
-[MessagePackObject]
public struct ScanningStatus
{
- [Key(0)]
public float PercentageOfIpv4Scanned { get; set; }
- [Key(1)]
public long TotalFiltered { get; set; }
- [Key(2)]
public long AmountOfIpv4Left { get; set; }
- [Key(3)]
public long TotalDiscarded { get; set; }
- [Key(4)]
public double DiscardedDbSize { get; set; }
-
- [Key(5)]
+
public double FilteredDbSize { get; set; }
-
- [Key(6)]
+
public double MyDbSize { get; set; }
}
\ No newline at end of file
diff --git a/Models/Models.csproj b/Models/Models.csproj
index 26af8be..f009e73 100644
--- a/Models/Models.csproj
+++ b/Models/Models.csproj
@@ -7,12 +7,6 @@
-
-
-
- all
- runtime; build; native; contentfiles; analyzers; buildtransitive
-
diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db
index 60a12f6..a44725c 100644
Binary files a/Models/ScannerResume.db and b/Models/ScannerResume.db differ
diff --git a/Models/mydb.db b/Models/mydb.db
index 65da4af..4944bea 100644
Binary files a/Models/mydb.db and b/Models/mydb.db differ
diff --git a/Proxy/Program.cs b/Proxy/Program.cs
index 81d0b30..ae00150 100644
--- a/Proxy/Program.cs
+++ b/Proxy/Program.cs
@@ -31,8 +31,8 @@ progressApi.MapGet("/", () =>
{
Command = CommunicationCommand.GetScanningProgress
};
-
- byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
+
+ byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
@@ -40,7 +40,7 @@ progressApi.MapGet("/", () =>
byte[] msg = client.ReceiveFrameBytes();
client.Close();
- return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
+ return JsonSerializer.Deserialize(msg);
});
RouteGroupBuilder searchApi = app.MapGroup("/search");
@@ -50,8 +50,8 @@ searchApi.MapGet("/{term}", (string term) =>
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.GetSearches;
communicationObject.SearchTerm = term;
-
- byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
+
+ byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
@@ -66,6 +66,7 @@ app.Run();
[JsonSerializable(typeof(ScanningStatus))]
[JsonSerializable(typeof(SearchResults))]
+[JsonSerializable(typeof(CommunicationObject))]
internal partial class AppJsonSerializerContext : JsonSerializerContext
{
}
\ No newline at end of file
diff --git a/Proxy/Proxy.csproj b/Proxy/Proxy.csproj
index 4f7dbe2..fd6bb2b 100644
--- a/Proxy/Proxy.csproj
+++ b/Proxy/Proxy.csproj
@@ -5,16 +5,10 @@
enable
enable
true
- true
+ false
-
-
-
- all
- runtime; build; native; contentfiles; analyzers; buildtransitive
-
diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user
index 66ca8e5..8248f74 100644
--- a/RSE.sln.DotSettings.user
+++ b/RSE.sln.DotSettings.user
@@ -8,6 +8,7 @@
ForceIncluded
ForceIncluded
ForceIncluded
+ ForceIncluded
ForceIncluded
ForceIncluded
ForceIncluded
@@ -15,9 +16,12 @@
ForceIncluded
ForceIncluded
ForceIncluded
+ ForceIncluded
ForceIncluded
ForceIncluded
ForceIncluded
ForceIncluded
+ ForceIncluded
+ ForceIncluded
ForceIncluded
ForceIncluded
\ No newline at end of file
diff --git a/identifier.sqlite b/identifier.sqlite
new file mode 100644
index 0000000..e69de29