Create runtime variables and remove MessagePack.

This commit is contained in:
Rasmus Rasmussen 2024-11-26 19:40:27 +01:00
parent 28c5918a83
commit be07cdb13e
28 changed files with 336 additions and 200 deletions

View File

@ -16,12 +16,6 @@
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="MessagePack" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePack.Annotations" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePackAnalyzer" Version="3.0.238-rc.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>

View File

@ -1,7 +1,6 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Backend.Helper;
using MessagePack;
using Models.Handler;
using Models.Model.Backend;
using Models.Model.External;
@ -14,12 +13,18 @@ public class Communication
{
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private bool _isRunning = true;
private string _basePath;
public Communication(DbHandler dbHandler, ThreadHandler threadHandler)
public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
{
_dbHandler = dbHandler;
_threadHandler = threadHandler;
_ipScanner = ipScanner;
_contentFilter = contentFilter;
_basePath = basePath;
}
public WaitHandle[] Start()
@ -43,15 +48,20 @@ public class Communication
while (_isRunning)
{
byte[] message = rep.ReceiveFrameBytes();
CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
CommunicationObject communicationObject = MessagePackSerializer.Deserialize<CommunicationObject>(message);
if (communicationObject is null)
{
continue;
}
OnServerOnReceiveReady(communicationObject, rep);
}
((EventWaitHandle) obj).Set();
}
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
@ -60,8 +70,8 @@ public class Communication
{
case CommunicationCommand.GetScanningProgress:
{
DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes();
DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
long discardedIndexes = _dbHandler.GetDiscardedIndexes();
ScanningStatus status = new();
@ -77,13 +87,13 @@ public class Communication
}
status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
status.TotalFiltered = DbHandler.GetFilteredIndexes();
status.TotalFiltered = _dbHandler.GetFilteredIndexes();
status.TotalDiscarded = discardedIndexes;
status.MyDbSize = databaseSizes.MyDbSize;
status.FilteredDbSize = databaseSizes.FilteredDbSize;
status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
byte[] serializedResult = MessagePackSerializer.Serialize(status, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
rep.SendFrame(serializedResult);
@ -93,7 +103,7 @@ public class Communication
case CommunicationCommand.DbReindex:
{
_dbHandler.ReIndex();
SendStringResponse(rep, "All Dbs have been reindexed.");
break;
}
@ -101,7 +111,7 @@ public class Communication
case CommunicationCommand.DbVacuum:
{
_dbHandler.Vacuum();
SendStringResponse(rep, "All Dbs have been vacuumed.");
break;
}
@ -121,13 +131,48 @@ public class Communication
_isRunning = false;
break;
}
case CommunicationCommand.ChangeRuntimeVariable:
{
Console.WriteLine("lmao");
if (string.IsNullOrWhiteSpace(communicationObject.VariableValue))
{
break;
}
if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetContentWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetDiscardedWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_ipScanner.SetTimeout(value);
}
if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_contentFilter.SetTimeout(value);
}
break;
}
}
}
private static void SendStringResponse(ResponseSocket rep, string response)
{
MessagePackSerializerOptions withCompression = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray);
byte[] serializedResult = MessagePackSerializer.Serialize(response, withCompression);
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
rep.SendFrame(serializedResult);
}

View File

@ -11,14 +11,25 @@ public class ContentFilter
{
private readonly ConcurrentQueue<QueueItem> _queue;
private readonly DbHandler _dbHandler;
private const string GetDomainPort80 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort80.sh";
private const string GetDomainPort443 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort443.sh";
private readonly string _getDomainPort80;
private readonly string _getDomainPort443;
private bool _stop;
private int _timeOut;
private string _basePath;
public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler)
public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, string basePath)
{
_queue = queue;
_dbHandler = dbHandler;
_basePath = basePath;
_getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
_getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
}
public void SetTimeout(int timeOut)
{
_timeOut = timeOut;
}
public WaitHandle[] Start()
@ -34,49 +45,54 @@ public class ContentFilter
private void Filter(object obj)
{
long indexes = DbHandler.GetUnfilteredIndexes();
for (long i = 0; i < indexes; i++)
while (!_stop)
{
if (_stop) break;
long indexes = _dbHandler.GetUnfilteredIndexes();
Unfiltered? unfiltered = DbHandler.ReadUnfilteredWithId(i);
if (unfiltered is null || unfiltered.Filtered == 1) continue;
unfiltered.Filtered = 1;
QueueItem superUnfilteredObject = new()
for (long i = 0; i < indexes; i++)
{
Unfiltered = unfiltered,
Operations = Operations.Update
};
_queue.Enqueue(superUnfilteredObject);
if (_dbHandler.GetFilteredIp(unfiltered.Ip))
{
continue;
if (_stop) break;
Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i);
if (unfiltered is null || unfiltered.Filtered == 1) continue;
unfiltered.Filtered = 1;
QueueItem superUnfilteredObject = new()
{
Unfiltered = unfiltered,
Operations = Operations.Update
};
_queue.Enqueue(superUnfilteredObject);
if (_dbHandler.GetFilteredIp(unfiltered.Ip))
{
continue;
}
Filtered filtered = GetSiteData(unfiltered.Ip);
filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2;
QueueItem superFilteredObject = new()
{
Filtered = filtered,
Operations = Operations.Insert
};
_queue.Enqueue(superFilteredObject);
}
Filtered filtered = GetSiteData(unfiltered.Ip);
filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2;
QueueItem superFilteredObject = new()
{
Filtered = filtered,
Operations = Operations.Insert
};
_queue.Enqueue(superFilteredObject);
Thread.Sleep(_timeOut);
}
((EventWaitHandle) obj).Set();
}
private static Filtered GetSiteData(string ip)
private Filtered GetSiteData(string ip)
{
StartProcess(ip, 80);
StartProcess(ip, 443);
@ -112,7 +128,7 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
using StreamReader streamReader = new($"{ports[i]}Header.txt");
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
while (streamReader.Peek() != -1)
{
@ -204,9 +220,9 @@ public class ContentFilter
return siteData;
}
private static void StartProcess(string ip, int port)
private void StartProcess(string ip, int port)
{
string fileName = port == 80 ? GetDomainPort80 : GetDomainPort443;
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
Process proc = new();
proc.StartInfo = new()

View File

@ -22,12 +22,21 @@ public class IpScanner
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly DbHandler _dbHandler;
private bool _stop;
private int _timeout;
public IpScanner(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, ConcurrentQueue<Discarded> discardedQueue)
{
_queue = queue;
_dbHandler = dbHandler;
_discardedQueue = discardedQueue;
SetTimeout(128);
}
public void SetTimeout(int milliseconds)
{
Console.WriteLine($"Setting timeout to {milliseconds}ms");
_timeout = milliseconds;
}
public WaitHandle[] Start(int threads)
@ -133,7 +142,7 @@ public class IpScanner
_ = IPAddress.TryParse(ip, out IPAddress? address);
if (address is not null)
{
responseCode = ping.Send(address, 512, buf, null).Status;
responseCode = ping.Send(address, _timeout, buf, null).Status;
}
}
catch (Exception e)

View File

@ -14,23 +14,22 @@ public class ThreadHandler
private bool _communicationStopped;
private bool _ipScannerStopped;
private bool _contentFilterStopped;
private bool _stopSignal;
public ThreadHandler()
public ThreadHandler(string path)
{
ConcurrentQueue<QueueItem> contentQueue = new();
ConcurrentQueue<Discarded> discardedQueue = new();
_dbHandler = new(contentQueue, discardedQueue);
_communication = new(_dbHandler, this);
_dbHandler = new(contentQueue, discardedQueue, path);
_ipScanner = new(contentQueue, _dbHandler, discardedQueue);
_contentFilter = new(contentQueue, _dbHandler);
_contentFilter = new(contentQueue, _dbHandler, path);
_communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
}
public void Start()
{
Thread scanner = new(StartScanner);
Thread indexer = new(StartIndexer);
Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
Thread communication = new(StartCommunicationHandler);
@ -48,18 +47,11 @@ public class ThreadHandler
communication.Join();
}
public static void ManualGc()
{
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
private void StartScanner()
{
Thread.Sleep(10000); // Let the database handler instantiate and warm up first.
WaitHandle[] wait = _ipScanner.Start(4);
WaitHandle[] wait = _ipScanner.Start(2);
WaitHandle.WaitAll(wait);
@ -68,18 +60,13 @@ public class ThreadHandler
_ipScannerStopped = true;
}
private void StartIndexer()
private void StartContentFilter()
{
while (!_stopSignal)
{
WaitHandle[] wait = _contentFilter.Start();
WaitHandle.WaitAll(wait);
Thread.Sleep(300000); // 5 minutes
}
WaitHandle[] wait = _contentFilter.Start();
Console.WriteLine("Indexer finished");
WaitHandle.WaitAll(wait);
Console.WriteLine("Content filter finished");
_contentFilterStopped = true;
}
@ -113,7 +100,6 @@ public class ThreadHandler
private void Stop()
{
_stopSignal = true;
_ipScanner.Stop();
_contentFilter.Stop();

View File

@ -4,24 +4,24 @@ namespace Backend.Helper;
public static class FilesystemHelper
{
public static DatabaseSizes GetDatabaseSizes()
public static DatabaseSizes GetDatabaseSizes(string basePath)
{
DatabaseSizes databaseSizes = new();
FileInfo fileInfo = new("../../../../Models/mydb.db");
FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
databaseSizes.DiscardedDbSize = GetDiscardedDbSizes().ToSize(SizeUnits.KB);
databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
fileInfo = new("../../../../Models/Filtered.db");
fileInfo = new($"{basePath}/Models/Filtered.db");
databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
return databaseSizes;
}
private static long GetDiscardedDbSizes()
private static long GetDiscardedDbSizes(string basePath)
{
const string folder = "../../../../Models";
string folder = $"{basePath}/Models";
string[] files = Directory.GetFiles(folder, "*.db");
long size = 0;

View File

@ -0,0 +1,9 @@
namespace Backend.Helper;
public static class ProjectPathHelper
{
public static string GetProjectPath()
{
return new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory).Parent!.Parent!.Parent!.Parent!.FullName;
}
}

View File

@ -1,8 +1,9 @@
using Backend.Handler;
using Backend.Helper;
Console.WriteLine("Program started");
ThreadHandler threadHandler = new();
ThreadHandler threadHandler = new(ProjectPathHelper.GetProjectPath());
threadHandler.Start();
Console.WriteLine("Done");

View File

@ -1,4 +1,6 @@
using MessagePack;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Models.Model.Backend;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
@ -32,7 +34,7 @@ public static class Commands
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void Vacuum()
{
CommunicationObject communicationObject = new();
@ -48,6 +50,18 @@ public static class Commands
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
{
CommunicationObject communicationObject = new()
{
Command = CommunicationCommand.ChangeRuntimeVariable,
Variable = runtimeVariable.ToString(),
VariableValue = value
};
SendAndRecieveStringMessage(communicationObject);
}
public static void GetHelp()
{
@ -59,32 +73,53 @@ public static class Commands
Console.WriteLine(" g - manual garbage collect on the server");
Console.WriteLine(" r - manual reindex the databases");
Console.WriteLine(" v - manual vacuum the databases");
Console.WriteLine(" R - change runtime variable");
Console.WriteLine(" lR - list runtime variables");
Console.WriteLine(" help - shows this help");
Console.WriteLine();
}
private static string SendAndRecieveStringMessage(CommunicationObject communicationObject)
public static void PrintRuntimeVariables()
{
byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(bytes);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return MessagePackSerializer.Deserialize<string>(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
Console.WriteLine("Runtime variables:");
Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()} - Sets the timeout in milliseconds for the scanner");
Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()} - Sets the timeout in milliseconds for the content filter");
Console.WriteLine($"{RuntimeVariable.DbContent.ToString()} - Sets the wait time in milliseconds for the content database if the queue is empty");
Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()} - Sets the wait time in milliseconds for the discarded database if the queue is empty");
}
private static ScanningStatus GetProgress(CommunicationObject communicationObject)
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
{
byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
//byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(bytes);
client.SendFrame(lol);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return MessagePackSerializer.Deserialize<ScanningStatus>(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
return JsonSerializer.Deserialize<string>(msg);
}
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
private static ScanningStatus GetProgress(CommunicationObject communicationObject)
{
//byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(lol);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return JsonSerializer.Deserialize<ScanningStatus>(msg);
}
}

View File

@ -5,14 +5,13 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<PublishAot>true</PublishAot>
<Platform>x64</Platform>
<!--<Platform>x64</Platform>
<Optimize>true</Optimize>
<!--<PublishAot>true</PublishAot>-->
<PublishAot>true</PublishAot>
<PublishTrimmed>true</PublishTrimmed>
<TrimMode>link</TrimMode>
<TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>
<TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
</PropertyGroup>
<ItemGroup>
@ -20,12 +19,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="MessagePack" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePack.Annotations" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePackAnalyzer" Version="3.0.238-rc.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>

View File

@ -1,4 +1,5 @@
using Manager;
using Models.Model.Backend;
bool stop = false;
@ -36,6 +37,43 @@ do
Commands.ReIndex();
}
else if (string.Equals(input, "R"))
{
string? variable = Console.ReadLine();
string? value = Console.ReadLine();
if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
{
Console.WriteLine("Please enter a value.");
return;
}
if (variable == RuntimeVariable.ScannerTimeout.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
}
else if (variable == RuntimeVariable.ContentFilter.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
}
else if (variable == RuntimeVariable.DbDiscarded.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
}
else if (variable == RuntimeVariable.DbContent.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
}
}
else if (string.Equals(input, "lR"))
{
Commands.PrintRuntimeVariables();
}
else if (string.Equals(input, "help"))
{
Commands.GetHelp();

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -9,11 +9,11 @@ public class DbHandler
{
private readonly ConcurrentQueue<QueueItem> _contentQueue;
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private const string UnfilteredConnectionString = "Data Source=../../../../Models/mydb.db";
private const string DiscardedConnectionString = "Data Source=../../../../Models/Discarded.db";
private const string FilteredConnectionString = "Data Source=../../../../Models/Filtered.db";
private const string ResumeConnectionString = "Data Source=../../../../Models/ScannerResume.db";
private readonly string _unfilteredConnectionString;
private readonly string _discardedConnectionString;
private readonly string _filteredConnectionString;
private readonly string _resumeConnectionString;
private readonly List<string> _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)";
@ -33,18 +33,43 @@ public class DbHandler
private const string ReIndexDatabasesStatement = "REINDEX;";
private const string VacuumDatabasesStatement = "VACUUM;";
private readonly object _readFilteredLock = new();
private readonly object _readAndDeleteResumeLock = new();
private bool _stop;
private bool _pause;
private bool _paused;
private int _contentWaitTime;
private int _discardedWaitTime;
private readonly string _basePath;
public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue)
public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
{
_contentQueue = contentQueue;
_discardedQueue = discardedQueue;
SetContentWaitTime(10);
SetDiscardedWaitTime(10);
_basePath = basePath;
_unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
_discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
_filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
_resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
}
public void SetContentWaitTime(int waitTime)
{
_contentWaitTime = waitTime;
}
public void SetDiscardedWaitTime(int waitTime)
{
_discardedWaitTime = waitTime;
}
public void StartContent()
@ -55,7 +80,7 @@ public class DbHandler
{
if (_contentQueue.IsEmpty || _pause)
{
Thread.Sleep(10);
Thread.Sleep(_contentWaitTime);
_paused = true;
continue;
}
@ -120,7 +145,7 @@ public class DbHandler
{
if (_discardedQueue.IsEmpty || _pause)
{
Thread.Sleep(10);
Thread.Sleep(_discardedWaitTime);
_paused = true;
continue;
}
@ -137,9 +162,9 @@ public class DbHandler
Console.WriteLine("Content DbHandler stopped.");
}
private static void InsertUnfiltered(Unfiltered unfiltered)
private void InsertUnfiltered(Unfiltered unfiltered)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertStatement, connection);
@ -168,9 +193,9 @@ public class DbHandler
connection.Close();
}
private static void InsertFiltered(Filtered filtered)
private void InsertFiltered(Filtered filtered)
{
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoFiltered, connection);
@ -209,9 +234,9 @@ public class DbHandler
connection.Close();
}
private static void InsertResumeObject(ScannerResumeObject resumeObject)
private void InsertResumeObject(ScannerResumeObject resumeObject)
{
using SqliteConnection connection = new(ResumeConnectionString);
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoResume, connection);
@ -228,9 +253,9 @@ public class DbHandler
connection.Close();
}
private static void UpdateUnfiltered(Unfiltered unfiltered)
private void UpdateUnfiltered(Unfiltered unfiltered)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(UpdateUnfilteredStatement, connection);
@ -241,9 +266,9 @@ public class DbHandler
connection.Close();
}
public static Unfiltered? ReadUnfilteredWithId(long id)
public Unfiltered? ReadUnfilteredWithId(long id)
{
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredStatement, connection);
@ -266,11 +291,11 @@ public class DbHandler
return unfiltered;
}
public static long GetUnfilteredIndexes()
public long GetUnfilteredIndexes()
{
long rowId = 0;
using SqliteConnection connection = new(UnfilteredConnectionString);
using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadUnfilteredIdsStatement, connection);
@ -289,11 +314,11 @@ public class DbHandler
return rowId;
}
public static long GetFilteredIndexes()
public long GetFilteredIndexes()
{
long rowId = 0;
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
@ -347,7 +372,7 @@ public class DbHandler
{
lock (_readFilteredLock)
{
using SqliteConnection connection = new(FilteredConnectionString);
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredStatement, connection);
@ -377,7 +402,7 @@ public class DbHandler
{
lock (_readAndDeleteResumeLock)
{
using SqliteConnection connection = new(ResumeConnectionString);
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
@ -417,21 +442,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(DiscardedConnectionString);
SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(FilteredConnectionString);
connection = new(_filteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(UnfilteredConnectionString);
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
@ -456,21 +481,21 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(DiscardedConnectionString);
SqliteConnection connection = new(_discardedConnectionString);
connection.Open();
SqliteCommand command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(FilteredConnectionString);
connection = new(_filteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(UnfilteredConnectionString);
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
@ -486,7 +511,7 @@ public class DbHandler
private string CreateDiscardedDb(int threadNumber)
{
string databaseName = $"Data Source=../../../../Models/Discarded{threadNumber}.db";
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";

View File

@ -1,16 +1,10 @@
using MessagePack;
namespace Models.Model.Backend;
[MessagePackObject]
public struct DatabaseSizes
{
[Key(0)]
public double DiscardedDbSize { get; set; }
[Key(1)]
public double FilteredDbSize { get; set; }
[Key(2)]
public double MyDbSize { get; set; }
}

View File

@ -0,0 +1,9 @@
namespace Models.Model.Backend;
public enum RuntimeVariable
{
DbContent,
DbDiscarded,
ContentFilter,
ScannerTimeout
}

View File

@ -7,4 +7,5 @@ public enum CommunicationCommand
StopScanning,
DbReindex,
DbVacuum,
ChangeRuntimeVariable,
}

View File

@ -1,13 +1,17 @@
using MessagePack;
namespace Models.Model.External;
[MessagePackObject]
//[MessagePackObject]
public class CommunicationObject
{
[Key(0)]
//[Key(0)]
public CommunicationCommand Command { get; set; }
[Key(1)]
public string? SearchTerm { get; set; }
//[Key(1)]
public string? SearchTerm { get; set; } = "";
//[Key(2)]
public string? Variable { get; set; } = "";
//[Key(3)]
public string? VariableValue { get; set; } = "";
}

View File

@ -1,13 +1,8 @@
using MessagePack;
namespace Models.Model.External;
[MessagePackObject]
public class CommunicationResult
{
[Key(0)]
public List<SearchResult?>? Result { get; set; }
[Key(1)]
public ScanningStatus? Status { get; set; }
}

View File

@ -1,29 +1,18 @@
using MessagePack;
using Models.Model.Backend;
namespace Models.Model.External;
[MessagePackObject]
public struct ScanningStatus
{
[Key(0)]
public float PercentageOfIpv4Scanned { get; set; }
[Key(1)]
public long TotalFiltered { get; set; }
[Key(2)]
public long AmountOfIpv4Left { get; set; }
[Key(3)]
public long TotalDiscarded { get; set; }
[Key(4)]
public double DiscardedDbSize { get; set; }
[Key(5)]
public double FilteredDbSize { get; set; }
[Key(6)]
public double MyDbSize { get; set; }
}

View File

@ -7,12 +7,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MessagePack" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePack.Annotations" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePackAnalyzer" Version="3.0.238-rc.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
<PackageReference Include="SQLite" Version="3.13.0" />
</ItemGroup>

Binary file not shown.

Binary file not shown.

View File

@ -31,8 +31,8 @@ progressApi.MapGet("/", () =>
{
Command = CommunicationCommand.GetScanningProgress
};
byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
@ -40,7 +40,7 @@ progressApi.MapGet("/", () =>
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return MessagePackSerializer.Deserialize<ScanningStatus>(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray));
return JsonSerializer.Deserialize<ScanningStatus>(msg);
});
RouteGroupBuilder searchApi = app.MapGroup("/search");
@ -50,8 +50,8 @@ searchApi.MapGet("/{term}", (string term) =>
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.GetSearches;
communicationObject.SearchTerm = term;
byte[] bytes = MessagePackSerializer.Serialize(communicationObject);
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
@ -66,6 +66,7 @@ app.Run();
[JsonSerializable(typeof(ScanningStatus))]
[JsonSerializable(typeof(SearchResults))]
[JsonSerializable(typeof(CommunicationObject))]
internal partial class AppJsonSerializerContext : JsonSerializerContext
{
}

View File

@ -5,16 +5,10 @@
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<PublishAot>true</PublishAot>
<PublishAot>false</PublishAot>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MessagePack" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePack.Annotations" Version="3.0.238-rc.1" />
<PackageReference Include="MessagePackAnalyzer" Version="3.0.238-rc.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>

View File

@ -8,6 +8,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttribute_002ECoreCLR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbb9be2cb8efb72a7d2286fbdd10a293b55b3e2a912f49fac6a7719673268e4b_003FAttribute_002ECoreCLR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConsole_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2ccd2056ec55b7b67558d52e32a887ed5ac7e346fc429b218c188d0a99cb5be_003FConsole_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5029ca69753a60cb407b1053f5834320dadee066_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F99b5d033f8bfb58a6a753ca88a3d956f2228bd48_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F13cbfe6856867dd1ed4e39575e46d816dae2a146a8ceec8f76b5897f5e0fe_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpContent_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a55e5df9328b51021ba85b8f12ff49eca8772dba9772c0d61d5e28edf255c_003FHttpContent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@ -15,9 +16,12 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIFormatterResolver_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc7d856f45ddf907f9face6c2bdba7836cc70fed4bb2b8ebd83ee6917584af3_003FIFormatterResolver_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIOThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F125fae6d49819da7be4fb21fbb4a936a7ec325971cf9264a24af55bf7111ad8_003FIOThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F8fe420eaf60c4dfca87ce1d5f1cdfa4816200_003Fbd_003F71bcce16_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5cffb8c94881849b6aa8abe7ba5e7cafe05d991f859903020362d6aac371_003FMessagePackReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AMessagePackSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe5f8b0187e62d74a4b9fefd657c192dab367bb342ef5ffbd90a4ed2ea428ec7a_003FMessagePackSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingOptions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F77bf2fbffc5491eadd62f1dbebc233798cd7fcb9b39ab7ee3bb35519f4d94ecc_003FPingOptions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8d57f5f5fd3290d6a89a5b767ad89988dd893c988eba430cd461b8b88d7ad9d_003FPing_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARep_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Feea8e921c916164ccc376e162da148b8215450dfae96e08bdd29119165c67f_003FRep_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AReq_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F33fafcc2bb6e91ae8abb5a52936d39cd93951ad9208861e758ca93efa619eaf3_003FReq_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATask_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F35f3a54f5acb408a3e219b2de039f1a39557b7e4515f11238cba07b60c0ce_003FTask_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATextWriter_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdda89b2ed0975050b6847325357a756a5866a5435e3ddb4feff535ba36facb7_003FTextWriter_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

0
identifier.sqlite Normal file
View File