From be07cdb13ebfaabdfb41a96959c4202547e797cd Mon Sep 17 00:00:00 2001 From: Rasmus Date: Tue, 26 Nov 2024 19:40:27 +0100 Subject: [PATCH] Create runtime variables and remove MessagePack. --- Backend/Backend.csproj | 6 -- Backend/Handler/Communication.cs | 73 +++++++++++--- Backend/Handler/ContentFilter.cs | 94 ++++++++++-------- Backend/Handler/IpScanner.cs | 11 +- Backend/Handler/ThreadHandler.cs | 36 ++----- Backend/Helper/FilesystemHelper.cs | 12 +-- Backend/Helper/ProjectPathHelper.cs | 9 ++ Backend/Program.cs | 3 +- Manager/Commands.cs | 65 +++++++++--- Manager/Manager.csproj | 15 +-- Manager/Program.cs | 38 +++++++ Models/Discarded0.db | Bin 20480 -> 12288 bytes Models/Discarded1.db | Bin 20480 -> 12288 bytes Models/Filtered.db | Bin 36864 -> 12288 bytes Models/Handler/DbHandler.cs | 89 +++++++++++------ Models/Model/Backend/DatabaseSizes.cs | 10 +- Models/Model/Backend/RuntimeVariable.cs | 9 ++ Models/Model/External/CommunicationCommand.cs | 1 + Models/Model/External/CommunicationObject.cs | 16 +-- Models/Model/External/CommunicationResult.cs | 5 - Models/Model/External/ScanningStatus.cs | 15 +-- Models/Models.csproj | 6 -- Models/ScannerResume.db | Bin 8192 -> 8192 bytes Models/mydb.db | Bin 24576 -> 24576 bytes Proxy/Program.cs | 11 +- Proxy/Proxy.csproj | 8 +- RSE.sln.DotSettings.user | 4 + identifier.sqlite | 0 28 files changed, 336 insertions(+), 200 deletions(-) create mode 100644 Backend/Helper/ProjectPathHelper.cs create mode 100644 Models/Model/Backend/RuntimeVariable.cs create mode 100644 identifier.sqlite diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj index f445ed4..87db91a 100644 --- a/Backend/Backend.csproj +++ b/Backend/Backend.csproj @@ -16,12 +16,6 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/Backend/Handler/Communication.cs b/Backend/Handler/Communication.cs index a22971f..6086039 100644 --- a/Backend/Handler/Communication.cs +++ b/Backend/Handler/Communication.cs @@ -1,7 +1,6 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json; using Backend.Helper; -using MessagePack; using Models.Handler; using Models.Model.Backend; using Models.Model.External; @@ -14,12 +13,18 @@ public class Communication { private readonly DbHandler _dbHandler; private readonly ThreadHandler _threadHandler; + private readonly IpScanner _ipScanner; + private readonly ContentFilter _contentFilter; private bool _isRunning = true; + private string _basePath; - public Communication(DbHandler dbHandler, ThreadHandler threadHandler) + public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath) { _dbHandler = dbHandler; _threadHandler = threadHandler; + _ipScanner = ipScanner; + _contentFilter = contentFilter; + _basePath = basePath; } public WaitHandle[] Start() @@ -43,15 +48,20 @@ public class Communication while (_isRunning) { byte[] message = rep.ReceiveFrameBytes(); + + CommunicationObject? communicationObject = JsonSerializer.Deserialize(message); - CommunicationObject communicationObject = MessagePackSerializer.Deserialize(message); - + if (communicationObject is null) + { + continue; + } + OnServerOnReceiveReady(communicationObject, rep); } ((EventWaitHandle) obj).Set(); } - + [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")] [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")] private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep) @@ -60,8 +70,8 @@ public class Communication { case CommunicationCommand.GetScanningProgress: { - DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(); - + DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath); + long discardedIndexes = _dbHandler.GetDiscardedIndexes(); ScanningStatus status = new(); @@ -77,13 +87,13 @@ public class Communication } status.AmountOfIpv4Left = 4294967296 - discardedIndexes; - status.TotalFiltered = DbHandler.GetFilteredIndexes(); + status.TotalFiltered = _dbHandler.GetFilteredIndexes(); status.TotalDiscarded = discardedIndexes; status.MyDbSize = databaseSizes.MyDbSize; status.FilteredDbSize = databaseSizes.FilteredDbSize; status.DiscardedDbSize = databaseSizes.DiscardedDbSize; - - byte[] serializedResult = MessagePackSerializer.Serialize(status, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + + byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status); rep.SendFrame(serializedResult); @@ -93,7 +103,7 @@ public class Communication case CommunicationCommand.DbReindex: { _dbHandler.ReIndex(); - + SendStringResponse(rep, "All Dbs have been reindexed."); break; } @@ -101,7 +111,7 @@ public class Communication case CommunicationCommand.DbVacuum: { _dbHandler.Vacuum(); - + SendStringResponse(rep, "All Dbs have been vacuumed."); break; } @@ -121,13 +131,48 @@ public class Communication _isRunning = false; break; } + + case CommunicationCommand.ChangeRuntimeVariable: + { + Console.WriteLine("lmao"); + + if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) + { + break; + } + + if (communicationObject.Variable == RuntimeVariable.DbContent.ToString()) + { + int value = int.Parse(communicationObject.VariableValue); + _dbHandler.SetContentWaitTime(value); + } + + if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString()) + { + int value = int.Parse(communicationObject.VariableValue); + _dbHandler.SetDiscardedWaitTime(value); + } + + if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString()) + { + int value = int.Parse(communicationObject.VariableValue); + _ipScanner.SetTimeout(value); + } + + if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString()) + { + int value = int.Parse(communicationObject.VariableValue); + _contentFilter.SetTimeout(value); + } + + break; + } } } private static void SendStringResponse(ResponseSocket rep, string response) { - MessagePackSerializerOptions withCompression = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray); - byte[] serializedResult = MessagePackSerializer.Serialize(response, withCompression); + byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response); rep.SendFrame(serializedResult); } diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index a1287a6..f5be79d 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -11,14 +11,25 @@ public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly DbHandler _dbHandler; - private const string GetDomainPort80 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort80.sh"; - private const string GetDomainPort443 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort443.sh"; + private readonly string _getDomainPort80; + private readonly string _getDomainPort443; private bool _stop; + private int _timeOut; + private string _basePath; - public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler) + public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler, string basePath) { _queue = queue; _dbHandler = dbHandler; + _basePath = basePath; + + _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; + _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; + } + + public void SetTimeout(int timeOut) + { + _timeOut = timeOut; } public WaitHandle[] Start() @@ -34,49 +45,54 @@ public class ContentFilter private void Filter(object obj) { - long indexes = DbHandler.GetUnfilteredIndexes(); - - for (long i = 0; i < indexes; i++) + while (!_stop) { - if (_stop) break; + long indexes = _dbHandler.GetUnfilteredIndexes(); - Unfiltered? unfiltered = DbHandler.ReadUnfilteredWithId(i); - - if (unfiltered is null || unfiltered.Filtered == 1) continue; - - unfiltered.Filtered = 1; - - QueueItem superUnfilteredObject = new() + for (long i = 0; i < indexes; i++) { - Unfiltered = unfiltered, - Operations = Operations.Update - }; - - _queue.Enqueue(superUnfilteredObject); - - if (_dbHandler.GetFilteredIp(unfiltered.Ip)) - { - continue; + if (_stop) break; + + Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i); + + if (unfiltered is null || unfiltered.Filtered == 1) continue; + + unfiltered.Filtered = 1; + + QueueItem superUnfilteredObject = new() + { + Unfiltered = unfiltered, + Operations = Operations.Update + }; + + _queue.Enqueue(superUnfilteredObject); + + if (_dbHandler.GetFilteredIp(unfiltered.Ip)) + { + continue; + } + + Filtered filtered = GetSiteData(unfiltered.Ip); + + filtered.Port1 = unfiltered.Port1; + filtered.Port2 = unfiltered.Port2; + + QueueItem superFilteredObject = new() + { + Filtered = filtered, + Operations = Operations.Insert + }; + + _queue.Enqueue(superFilteredObject); } - Filtered filtered = GetSiteData(unfiltered.Ip); - - filtered.Port1 = unfiltered.Port1; - filtered.Port2 = unfiltered.Port2; - - QueueItem superFilteredObject = new() - { - Filtered = filtered, - Operations = Operations.Insert - }; - - _queue.Enqueue(superFilteredObject); + Thread.Sleep(_timeOut); } ((EventWaitHandle) obj).Set(); } - private static Filtered GetSiteData(string ip) + private Filtered GetSiteData(string ip) { StartProcess(ip, 80); StartProcess(ip, 443); @@ -112,7 +128,7 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - using StreamReader streamReader = new($"{ports[i]}Header.txt"); + using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt"); while (streamReader.Peek() != -1) { @@ -204,9 +220,9 @@ public class ContentFilter return siteData; } - private static void StartProcess(string ip, int port) + private void StartProcess(string ip, int port) { - string fileName = port == 80 ? GetDomainPort80 : GetDomainPort443; + string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; Process proc = new(); proc.StartInfo = new() diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index 750260b..32fb10a 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -22,12 +22,21 @@ public class IpScanner private readonly ConcurrentQueue _discardedQueue; private readonly DbHandler _dbHandler; private bool _stop; + private int _timeout; public IpScanner(ConcurrentQueue queue, DbHandler dbHandler, ConcurrentQueue discardedQueue) { _queue = queue; _dbHandler = dbHandler; _discardedQueue = discardedQueue; + + SetTimeout(128); + } + + public void SetTimeout(int milliseconds) + { + Console.WriteLine($"Setting timeout to {milliseconds}ms"); + _timeout = milliseconds; } public WaitHandle[] Start(int threads) @@ -133,7 +142,7 @@ public class IpScanner _ = IPAddress.TryParse(ip, out IPAddress? address); if (address is not null) { - responseCode = ping.Send(address, 512, buf, null).Status; + responseCode = ping.Send(address, _timeout, buf, null).Status; } } catch (Exception e) diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 90a171b..f742acb 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -14,23 +14,22 @@ public class ThreadHandler private bool _communicationStopped; private bool _ipScannerStopped; private bool _contentFilterStopped; - private bool _stopSignal; - public ThreadHandler() + public ThreadHandler(string path) { ConcurrentQueue contentQueue = new(); ConcurrentQueue discardedQueue = new(); - _dbHandler = new(contentQueue, discardedQueue); - _communication = new(_dbHandler, this); + _dbHandler = new(contentQueue, discardedQueue, path); _ipScanner = new(contentQueue, _dbHandler, discardedQueue); - _contentFilter = new(contentQueue, _dbHandler); + _contentFilter = new(contentQueue, _dbHandler, path); + _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path); } public void Start() { Thread scanner = new(StartScanner); - Thread indexer = new(StartIndexer); + Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); Thread discarded = new(StartDiscardedDbHandler); Thread communication = new(StartCommunicationHandler); @@ -48,18 +47,11 @@ public class ThreadHandler communication.Join(); } - public static void ManualGc() - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - GC.Collect(); - } - private void StartScanner() { Thread.Sleep(10000); // Let the database handler instantiate and warm up first. - WaitHandle[] wait = _ipScanner.Start(4); + WaitHandle[] wait = _ipScanner.Start(2); WaitHandle.WaitAll(wait); @@ -68,18 +60,13 @@ public class ThreadHandler _ipScannerStopped = true; } - private void StartIndexer() + private void StartContentFilter() { - while (!_stopSignal) - { - WaitHandle[] wait = _contentFilter.Start(); - - WaitHandle.WaitAll(wait); - - Thread.Sleep(300000); // 5 minutes - } + WaitHandle[] wait = _contentFilter.Start(); - Console.WriteLine("Indexer finished"); + WaitHandle.WaitAll(wait); + + Console.WriteLine("Content filter finished"); _contentFilterStopped = true; } @@ -113,7 +100,6 @@ public class ThreadHandler private void Stop() { - _stopSignal = true; _ipScanner.Stop(); _contentFilter.Stop(); diff --git a/Backend/Helper/FilesystemHelper.cs b/Backend/Helper/FilesystemHelper.cs index efc42f6..b74cd9b 100644 --- a/Backend/Helper/FilesystemHelper.cs +++ b/Backend/Helper/FilesystemHelper.cs @@ -4,24 +4,24 @@ namespace Backend.Helper; public static class FilesystemHelper { - public static DatabaseSizes GetDatabaseSizes() + public static DatabaseSizes GetDatabaseSizes(string basePath) { DatabaseSizes databaseSizes = new(); - FileInfo fileInfo = new("../../../../Models/mydb.db"); + FileInfo fileInfo = new($"{basePath}/Models/mydb.db"); databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB); - databaseSizes.DiscardedDbSize = GetDiscardedDbSizes().ToSize(SizeUnits.KB); + databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB); - fileInfo = new("../../../../Models/Filtered.db"); + fileInfo = new($"{basePath}/Models/Filtered.db"); databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB); return databaseSizes; } - private static long GetDiscardedDbSizes() + private static long GetDiscardedDbSizes(string basePath) { - const string folder = "../../../../Models"; + string folder = $"{basePath}/Models"; string[] files = Directory.GetFiles(folder, "*.db"); long size = 0; diff --git a/Backend/Helper/ProjectPathHelper.cs b/Backend/Helper/ProjectPathHelper.cs new file mode 100644 index 0000000..8b535ef --- /dev/null +++ b/Backend/Helper/ProjectPathHelper.cs @@ -0,0 +1,9 @@ +namespace Backend.Helper; + +public static class ProjectPathHelper +{ + public static string GetProjectPath() + { + return new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory).Parent!.Parent!.Parent!.Parent!.FullName; + } +} \ No newline at end of file diff --git a/Backend/Program.cs b/Backend/Program.cs index dd999b7..fb54b6e 100644 --- a/Backend/Program.cs +++ b/Backend/Program.cs @@ -1,8 +1,9 @@ using Backend.Handler; +using Backend.Helper; Console.WriteLine("Program started"); -ThreadHandler threadHandler = new(); +ThreadHandler threadHandler = new(ProjectPathHelper.GetProjectPath()); threadHandler.Start(); Console.WriteLine("Done"); \ No newline at end of file diff --git a/Manager/Commands.cs b/Manager/Commands.cs index fe3296b..9837684 100644 --- a/Manager/Commands.cs +++ b/Manager/Commands.cs @@ -1,4 +1,6 @@ -using MessagePack; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Models.Model.Backend; using Models.Model.External; using NetMQ; using NetMQ.Sockets; @@ -32,7 +34,7 @@ public static class Commands Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); } - + public static void Vacuum() { CommunicationObject communicationObject = new(); @@ -48,6 +50,18 @@ public static class Commands Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); } + + public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value) + { + CommunicationObject communicationObject = new() + { + Command = CommunicationCommand.ChangeRuntimeVariable, + Variable = runtimeVariable.ToString(), + VariableValue = value + }; + + SendAndRecieveStringMessage(communicationObject); + } public static void GetHelp() { @@ -59,32 +73,53 @@ public static class Commands Console.WriteLine(" g - manual garbage collect on the server"); Console.WriteLine(" r - manual reindex the databases"); Console.WriteLine(" v - manual vacuum the databases"); + Console.WriteLine(" R - change runtime variable"); + Console.WriteLine(" lR - list runtime variables"); Console.WriteLine(" help - shows this help"); Console.WriteLine(); } - private static string SendAndRecieveStringMessage(CommunicationObject communicationObject) + public static void PrintRuntimeVariables() { - byte[] bytes = MessagePackSerializer.Serialize(communicationObject); - - using RequestSocket client = new(); - client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(bytes); - byte[] msg = client.ReceiveFrameBytes(); - client.Close(); - return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + Console.WriteLine("Runtime variables:"); + Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()} - Sets the timeout in milliseconds for the scanner"); + Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()} - Sets the timeout in milliseconds for the content filter"); + Console.WriteLine($"{RuntimeVariable.DbContent.ToString()} - Sets the wait time in milliseconds for the content database if the queue is empty"); + Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()} - Sets the wait time in milliseconds for the discarded database if the queue is empty"); } - private static ScanningStatus GetProgress(CommunicationObject communicationObject) + [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] + private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject) { - byte[] bytes = MessagePackSerializer.Serialize(communicationObject); + //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options); + + byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject); using RequestSocket client = new(); client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(bytes); + client.SendFrame(lol); byte[] msg = client.ReceiveFrameBytes(); client.Close(); - return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + return JsonSerializer.Deserialize(msg); + } + + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] + [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] + private static ScanningStatus GetProgress(CommunicationObject communicationObject) + { + //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options); + + byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject); + + + using RequestSocket client = new(); + client.Connect("tcp://127.0.0.1:5556"); + client.SendFrame(lol); + byte[] msg = client.ReceiveFrameBytes(); + client.Close(); + + return JsonSerializer.Deserialize(msg); } } \ No newline at end of file diff --git a/Manager/Manager.csproj b/Manager/Manager.csproj index a0f2c15..6bed32a 100644 --- a/Manager/Manager.csproj +++ b/Manager/Manager.csproj @@ -5,14 +5,13 @@ net8.0 enable enable - - true - x64 + + + true true link - true + true--> @@ -20,12 +19,6 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/Manager/Program.cs b/Manager/Program.cs index d1b8b2b..abd960a 100644 --- a/Manager/Program.cs +++ b/Manager/Program.cs @@ -1,4 +1,5 @@ using Manager; +using Models.Model.Backend; bool stop = false; @@ -36,6 +37,43 @@ do Commands.ReIndex(); } + else if (string.Equals(input, "R")) + { + string? variable = Console.ReadLine(); + string? value = Console.ReadLine(); + + if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value)) + { + Console.WriteLine("Please enter a value."); + return; + } + + if (variable == RuntimeVariable.ScannerTimeout.ToString()) + { + Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value); + } + + else if (variable == RuntimeVariable.ContentFilter.ToString()) + { + Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value); + } + + else if (variable == RuntimeVariable.DbDiscarded.ToString()) + { + Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value); + } + + else if (variable == RuntimeVariable.DbContent.ToString()) + { + Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value); + } + } + + else if (string.Equals(input, "lR")) + { + Commands.PrintRuntimeVariables(); + } + else if (string.Equals(input, "help")) { Commands.GetHelp(); diff --git a/Models/Discarded0.db b/Models/Discarded0.db index ac1f416942a4d8db9bc6e1e26e44da6e3bbda8c2..da3fc68f31cf1f516df04251daed26ca4be7d204 100644 GIT binary patch literal 12288 zcmeI2OKcle6ozLUxA8pgy%Sz339Y9s&?HWcA93P@(#B0q>(+4+*MX)jZJpSGG^x|Z z=>|3hRYHPYg^++ZqJmvFNRe2V^*8TbAl`GX%KW%u1X&R%xZy3fBe7C{(_GQA0@V0>8sQ>9&GUD?; z3rK~OXQ)~Dp)XB96VL=S0Zl*?&;&FAO+XXS1T+Cn;9o#sGPJWZ7Bgp7tFue>=CvjG zaD1^@zjnRen0r*Vjg|AcO5U&Jju!L&BiWut^-8@TCVSuRHrqR+QS+T0TWcO$XwFqv zYV}&{MTo7_vi+Vy%`cQH`Q!PrUz(`+rRid^-!EMCEBP~^T28s%yt>?I*2k7>^}iXI zEEmRe}O;;ugCD?L2U#j%>;%D>4$Pb44P2E@Dsk`b6^^tm4T~~8zN*z%L zRkyO_1NoD@C%=_n%g^M8@}^vom*k{8Ec<1bjEGIKA=bs3xFc?hTjGXjhzp`54v9X| zB_i&oyWy_8YwjKQwtLIH;Wpg!ZqXfa``le_#MyK z&K^hFf7-v=_wDcPyY}by$M$>noAzb9w5l_aGiDWFKE>j~B3w11n zjs2lyDhnA%r6AMs2DdQvIyhJxes~EvrQ1 z#+Ue#SnndYFm-`h39R=Tx1gRpAFQ3tfIdg`0O+$sW7ip?p=-w(z9dFoB@rkhUkUUO zw2;$8V}%OQi03rXgP_YqW5p?=p`te#HuknoYZxZH3FhETiOeA4aiS4%k!b9ED$s-2 z`6SWUxj-~@_D;|pvykwPGY4C}%p7c$C#ztKV}Z`#E@MO^nWIGGe2!>5tWl!jB)lVR zN4)RDB!N4=6zBoCPwx<&*gCBjNdoU{ge9=mFiGHjW?2Gz4v_?&*dR+_(+qQ9DKfGWCok|u>>^rdYOY= zdxDu1K2HaTMyk)#i41mqjyc%%Su%rE_Y;jx_pu3Vx|bxdsm~I~G)fZC)7wK6TTjGp z=3v)nn1fxrnS;+j7jv-f)4@(jY`cpk@EPc24)%SDIoS6}vI_JSPlS!`?LVb3@Hibv zLlN;<5Ww^8U;)@wctHTSR5XA`+EC>tfO{py?;G?yMlr$ObzI#+Nc5bREQd|sxqkor1A5=seU)$OJACRCZGvu0-As( fpb2OKnt&#t31|YEfF|%?At21QsM-1_gT=o9k7ki% literal 20480 zcmeHOZH!b`8J@Xw=kwls&Yhy7eC=SVf=kz(nPoqu*3xBLmo5w4E+1Pc?ZVCi3d_er z3zV-jF~+D(5EG+`sg>45V@w*<)~Nl_v}j0(_D3nEiPgktX-$dXeFyhW9;(z+VJe+Y-BGEeQt47EmCE9G z^hx6%ZuEe^>G{8W5Tk4Q%>XDzhAFunf7p*@z%pPNunbrRECZGS%YbFTGGH073|I!< z1q`fra&tR7(-Vg#x9@4p9@vAI;|FIO2M#x8cD(a3GdMQ1YJ8|NzUt1Ap~^e3OJ^qc zHkQM4nVO%@&+X_)H?unC>fN(DCJ#GP-WOGP+@8 zWO-$He`S1VGoE70SYvkozM0v^;J&HGe{fhoHoSJ#*hJ-?p$Uq(YQy-t;ZZcXc4%~b z*)n>;Ps+SQ;-Gj) zjEdXEtzx0@g6qNS!R6pW@crPM!Iy$l!QNm?FdXy+w*+&7jQKda+>%^5m&(4CeKmVA`)u~9?Ah$8?4ImI z_O5I_Tge9Q4fl2TW%qgaY4>sW3+_>O+8uWX-Kx9D^)uHqS2M3sW^W6@98(vm($OupN^J>Q0v{) zLZ{l*-Bs-x=ZadF(2=-ObzpDXY88*xdvgVjy^hYPFkM9QS2lfI<(l(EYH4 z>P`)6cdi9m#XH<+Zlor6H8P#;e+>epVk85yl0%u0rVQ>Z>>V8EnM z0}KPFQ1u%a5Ghq3ZylCU^%^N^(+a~+k4O)Ttub}OO%F(o$JPT7fcaNKYPVB!HZR609|i2@H(xh@W73b=IDNL6Sn17)DR9qJhzD7qqE8o%(dz zdBda=$Z-?3CsdXN*K@$-!FzRk8N&@M3FR<1oj}@f(2#w;SNz>Zpm=P6>$)ntb(mldls7H?&F34X#%w1^h9cC`{e26{M$j!26 z1P}+=Gxha=(F6_1exqx!h<%Aq52ZC@_#leh%hRIa*~1~!efen)q51MD78RL?yA2m= zXBW?81uSwW4?^9UWI0 zmUw@FA<(Ma%F^heytZ%y_5Xe&293mo(Jn>V%$=y2O-VC#in@^lDEAFKsvf;@9+fQO zJ{E*X95WC!G4Evv;tYZZnxOsPeGjsiP(1M>>Y<4Rxs%9-9$G|P?9^!yLF+IG#f8{m zajAn$>ePu7C>@5Nc?6vkiLW{}43)#8k`H7Kvk(R%C$Z>U43$%-*$0u6v_uvOkrVf^ zPDDZCFbMI_B~G1s)KWO@FFbM*5I9l3HDV6(CW%UO8uBIvP%F?jtQ87sh?_b{kkB?s z5cE($+r&X=9zfWzAT(!N!Y0ZLJPGKU*arv+Q40cvyTT&)!n??~zCiWqOA!BMZ0EUXG(L^;wjJ6mW zrbYt`2@}VKkA!}SsjJdjg?fnrG$4>JNe4BVE$I@yi4b*AE=iZGNH>6Tncse;A^uys zrQN?OW#5u5Q4nefs)YqXZUJOV6oguVWMKff1duEd0HY(dT#3F`XgNWxB%OnUgjVT@ zXBQF+pjDy>NLxau;OFhEXd*$UER5c;Xr@7=Bmt0A0Fe>}AUYvXlAj|$0C^JqoDzUE zNgAMW*21Ly3P*i_E=d3s5vn9HPzY#}qzNDdG)XH*yivMpJ?+h><+T)k*^gzwGGH07 z3|Iy%1C{~HfMvikU>UFsSOzQu|0fKHbf&|J&lseO^#1=6wwL8K`5XBQyZ>+Z|Ko~R zwVO&#cK@H%5eKNsXZQb6QfwC??f!pPKNZ}SQ9JLanzcglJgJ4ZOsJnHMd^N&V?%`s zIzv$nSD}cW5R4fYs#X!bT@ml6x~@VMJpo~#Kqyqvqgs5Q_G;D9+x2O?|4&4sz@7|1 zABYSUitLHW?*F%rk@aE*+x`D!g{`3f3a66=MtjV5|DSyY_0YGV-T$}y|1=Nm{(p_W zAME}=NU;0=ZQ1s2+yAF?;5c%C_x}O!rz!bo`I@{We=NT(zapD*KTiLzk-c(>yh;31 z{83yHFN*Wxaq)R^L`;b>oc!+;bA=mR3w|A344w(j1!sce!LDFqFc@?P3xb0GSDgC4 zFQG~Mu?$!SECZGS%YbFTGGH073|I#K2L{Sb zoF`^iKlL?%CJq%Zj>_}ZD$c~0o2Ytn7^=nPCd!_$MMtXj^?@di7c*Bn6Wwf`Fs6&8 zT{ve?jyPh>gHfrwiBrbR5pO!OiF3v{c-(eGsB#lmpSYv0e(T#$?Y)F*6)(+Yb<#`> zu(CQ~+VI{|R>ut+9sVz?M@<{$tBz?~e3~4|mDN!br)@_}o6hAC(?-Yf)x$<#b@BhO zVWQHh!qupx`v8>{E=RHM(}DZ4!UZWd2xOBAm!uL;jlv}z`1Gl9(47O0=z7&JeXer#7;xO1kwcCJc)^=%F#4e4XR6AJ@sv3pLDT2i) z@F-P-qcq!7CQ7rm!%PDH=?3O1=WbChOhlZk>&s*N0_Sw(R*o)_KPg3DEjMKgoT ZRfz>%77Ev^qD>*XDJ>McT{VDw{tcS&J)i&p diff --git a/Models/Discarded1.db b/Models/Discarded1.db index ad218d2d48e62947b6010fc9f5c020dac813157b..51d8865e92d4b6afacc49ba6639cfbcc6b5d6572 100644 GIT binary patch literal 12288 zcmeI2O=w(I7>4gmrs@4V=iKq9iH&h?t=6QIjG6pSqNbUqV>|6+l1wJr)Yv*nCYqW| zer95-h-fK@E{cn86cJQ#p`ak@stb{#8{J6hLd1n=7ovzF>PGSVikn(erJCPkct$8>jQGA`7^mRX0I#c82>ys%74RGNzg?${ z#Nu}XQeov8Y7TzrOB2upGyzRO6VL=S0Zl*?&;&FAO+XX)4-l9LH#WuNp_y7~Zn<1t zUxp9IC#vQ3&2nY_TG=pK$fS!IznI>i%lOx1+bX4%atEwzf2Jk0y(tz8o!)lko(Gny z^QDc2@=xel}ms9LyB_{CLsNPv>$Ses;|-W{!fYJB4y}ZM9M@kFGA1 z|6##IAv=~X%=kw#GkC}Jba6bJhb_l4`C@xJem37SO+)>tzEYp5^Xe`2qIy^#%kHdu*d21)+-A4Y`NjFhx#(PQ-gRDcUT~gv zD$cBvbB3I~PP5Z!|7u^hFWDd4AJ}i$=j=0f&7QL->|wjZ-f5fG@787OqV04BF64;vOAUNg0EzEPdprbj9cCDu-uFot>Gjl5`%-FYec6&SBV|~y+L$8=yjs| zK(7(q3tO(TEt9Y$THzL~imq@A7cO%P>p#jZtba1tE`{}%h{pPl5RLU8CK|gevRigR zBwFScrWUw`jpw-qjpZD8c9)Y^ihq#5Q6Wr>GhrQzwV^{s;@Kn49nFE7)v&_NK z9tgG{09xKLmVkoZ40AAZlsVAQyPr8w(7TU0*l(ITSFjg@z5DU0m?9czc?Grt+fI@M z!kZu(n#pmZ@u2fWLkBq)=srC79MRCgJHmFv=?^mp_sEhpIQt=%fZ63ilEBIDWeIdL z)EU4>?m)1n7bf*aSpwwi(b%(_uE3M@lFUJ(UBQ}e?0F~A z*fYUaAkj|dVABq=2AjrN0-Nq731r&N63|t)1&JgQy@P1%xrb=%c{|Z~?za(*EqBu$ zda>oLB!TC?i)ifV6OA2X><-BD7V7jsOVJuJTK;~A$78>l1;7z4K>(lEoh$&3xG4xA z?q(K%K}1s!fGNBkEC7zUF$f_18(07a6W0d;oa;Ik07pcF0FLRg06K~~_?UQ-IS5=Z z2b;Le!L1$Y;4ynPbC99M9AsiL2O(@{4(`>+9N0^3ix{!sjUJ2$HDCtSKn)l{g{c8| zRUv8s2>kpXQa>5+tuIYL6VL=S0Zl*?&;&FAO+XXS1T+CnKoj`C5D=k;Sg8I_23vmt DWp=eu literal 20480 zcmeHOYm8l06~6b}bKmFeeUD-vl-gShb=s+Q?%a8`w7|5CblT~(Os7g8T4y>_tZk={ znJOZW?gmQy;TuCEiBXIsf)8Q{B%)%_n3^a^!~~icQDUOiG#F}(SdHIW_nvdsCMNze zQL}F!nX~3wYp?a~eQv&U_Fg+~9yvHST|aR2#F2@)dNWhYWV4wY>h(+}>t`~VJpRR> zEPfzc{GscA^FfT&2QGv_dAgR-llael90QI4$ADwNG2j?*3^)cH1C9a5fMdWh@L#~d zcCRqMvopJEZesHA^z8A&csSlSJAM4*^vwP@UgkEA4GoSD)yD@nj11M^fbE!>I5NEo zo-53vY;pdQCE3$?j(Ow3+5HnIrlzNoA3Q5gVy<@#Pt}J<$A@kj8mo_P8?TS<92r?v zA3jzeAG!rsiDhhh_Sn&x+3Af(r>6gf!}hV^t%GB`>RX0(QN+QWMM0c{Z(C1FR63tN%gS$oVrJyR8wk9 z4XSQcS1P<5UI(l{2hRlG3ceJ4Iye=~ z1bc(c!P=lBSP&Gef2sbq`a<=Y>SNU}RzF$2vwEnyySk}5P+eY~SM@8GDz8+YuRLA( zdgZ~&$11l~4peqlhAPdNA4eDPfIiQ>b>`-}G!j~Dk9M~c@L zR}|+J{lcFLFBhIIJXLt4aJF!7VXiP)*j89ySXo$5$md_nznp(A|DF7!`Lp>m`Puwr zetUj>er0}PzTp4a|Be5=|2_Xv|EzzXKj%;Q+x+YNPJf|a$X&|)I`?euySYbl59H3| zW^UB z;x!1Z%%E@sBr-SHIX205y&VcRZLX7{kW*yV+nx=wt+PF0)8<;~iQFTz)<$tXYeZr& z)XadNS?V>~(?umJ)1P7nFr_uq$C!4_gsGWc%MH_@OpkD5Ow+<(5^AQ~ip2BM5P1)P ztD5PuK+NzvB?t`H%xVk7q`yNXz!SVmdV$QE>9kS;Z#A>hGBHUjL?Fg=NKA{9x6AQc zrsWojsk|nQ)X!92ZDE+VWdg&zz0C^En56>4MAj_~&*Tyrnnu&+t%87ltS@C2i$Kid zRWeXN#@A+%^kUksv_PEELJ^2D3oHzhRA#=wFpu-B6nSRfA_nVYimtFgJdtmfAm->z z7Km@1NP>`K&D1OqveZUP5GSe9B>UJaWUuyW@8^3fkX}eznX2&O38`3Km=JBs(u-GU zNqF&6FG?>eUz>vTLSZ$Nmu2+v6!{W_!j#F0(Cp<|p;4JOcm$;X*TgTq1gD?H#i$ggMHV_Sa)C2+}ihAgcd}1;Nnzuz*l< zwL?~1TGAhs5L(d(ttiCV2P`Mz;8sxpjeNlJp}TO}@*$q5QlBRAv|kQPL{AC;B{E?J zrD5K01trTqiQx+0CoyDE?-g;0x4m*;%5{$wgcj&-38BTe%YxAIyhTJMlHOyDOZB{4 zATW&HDS|esz;P=mg6JI*LbE<*K`7aqt#QZ#?SosV{WZ};*+TgU3-JWqlUkU5=$_;j zY*IlGJ+Z*(rlkVOlira{B%>gB5(MQBxg!u117asZQ2LNMVpt-drA}L+G>JDT9RZ>H z1xiPRr8NqvljchI9CS`H4&CQaIU)#!h0GB_h<=EiBnTG>g(I9OEEG->7S|f;CUqeO zAZ-!=(F$RchM=HI$eISR*itp^$>}E6AZpUgx``!A)sRI))T9HWKqVwig8)!8DFF2< z=$SaP2JtM#OgG9`&@u^tz8a*E>84_#W73c)PD8~c2C79t#Uvq!QD~SHK&7=5Ok6Ax z1Oby4%Y?KPOdCTM4FQveLGCq6zBDK>)Qbp1oVDaj9EL1VF9JhxTJj}+0HCD>^#V-n zMV6L)i7~X6TIwankOksJc+u*Dco7zG5kkEzisd`Zxg}n5#vq%c=l`E!ds)A#f2E&u`~PnL zA1}jkz94>0*hTND-2T7Y|Hq46xBo9*g%8j>f75z{-o9Df{=eJ*7b}ELHn{zN+HPo_ zLhAt^ZvVdrTV!tk-|hc5X{*%j|GWKvv6Jui|8ZK`?foAP_I0hU8jseGj zW56-s7;p?Y1{?$b0}Mo`ak5zSol+l#KHWN9-0CrH;)Qs08pn$zie7S@=7y*CndAl? za@Xc=K9<~aq-KK1pyx?!J5j$xWYF~_gRu>rPtrzboDF)PaLBnG4T(hty-zY4rDo9m zBy6ow(E%lG$SpGHf|548wTuipph( zz>a8K}VH9 PQKz^z=&4Hihu=Q|tFI*OzWjtm-QIwC#-{R54ygRcT^fRqf|MN`TAxeT6m+ zs$;-*eDD33{QNn#kH6;}5A5nG)=h1+VvXnPT1uItsH(C_(-cLGD2fuKe={c^UGdEP zK$q&Q>-V@yjKt{GC~b9xKU5;i=}ewv1!M(e1!M(e1!M(e1!M(e1!M*Odll$&`4+5N zrS7fg50=c@krKVTAFi24Cd_i-&L7>GexoaAXt}PR-^* zq`v5%W^O5#>ZWCm%v`!|qcqRl+^n^Pg3a9{&B5;8oUzsD*Lrv6wBFr4J#CHKtIffj zF+?}ay0Ncf)sxQqH#X_cn{vf^$)v;X!cA|WgYMdpHERW{SgjW;WvAn2ZFV|(w^edF zif(q=d%(1gnpSSI>iA#0wbkj^{>s5hy_OrwIo;nE8=a2aR<*(h*BWZ|(ld`Ye!21d6O8{m{?GWe_|^F3(C49ZAuH4t z{3v)fxGy+A@Il~IV6*=-|6Bg5f2HrT?~HHAH_!XN_XY1J&!?U@J!Mae`;z;#d(b_{ z^`7f_*Awb>^$oS8u23#2uioF2TQ;#&X;NL$>C^IPJl=HU>do^7(PW~X{-oFSsqbio zQe|Rfw3N5Z_V#VLT;G~xB1yN^%hHID2%&HF$&}E5{$yQ z+k{aVcfSr%bjIDq7=>~7YZ!%bcM(Qm++B!K7g5EQ5bh47=>{+j8PbOLl}i|H;7Th?q&i@ zmBsvfOtHEdKLin*oAE&qvA7v81QC0i@jwu zt-DMgV3f_eHkz-|wM2KhR?nA8w4dOh>R6?0UOMqg?bhMr6^jlaMw?c(ibtut+@|Hr zBbsHFij7ZS5~KO?VrepQt8=n?q+ClBD&w>JE6JJu%Jikadmz^{a5R}n(Zz6=F)*xm ztR3E(*)cq@tt-`;KBjkcB-b06HFS^TCT%;K)LQ6bO;YdFHW%x)H8r!)xNL3HYQ;lk zb3~h{QHNBkjpb>#J=8(&ALycPsx>+N0SNs-$G4|D*Cdnr{Q$=F!=_nXoi7!SnkU=U zSr1r7^nf*VBgMIiHk=g}21|97s7%>SM1BJ+Rey2$+Bxh^vQcdm=f|DEe1^MB{M z$o${2E;9eWd&I2F{~ZSn$o$`V0D;W^AL=0+jsAb9^12fLUA!Jo#ID4C8QULwBzhtG zqi8nrdE{KginN743ZD({3(pUI5IPmw9Q-W!Rj^v?0T=Xu`qg!{Vt4R^`C!gbO0sw?M;spnN&-JpCzcX%HsV|1yqlpn2? zY!?q>GfWU89>->kfEe*WHlqMy#3R|V2chwB>%~LavU!Xm9?O5zCw1i%}SNhagHP9l$7zyZso2ad#I+VchM*D2%&1F$&{uFGN|#xVr~7 z+Zcs$w;Q7{?ry^#iV1O71u6TQ2o6+sV0@b)7{}G!=@RwNUxu1 z>V{1j8^jS8%&=*eEyNKwcf+Q6wh%|$;0>E*+Cm(0lQ(S2+8~a&(Hl0+wuLz2W^dRu z-xlJC8@^%Fj9Z8!Zu*8znH$6rH-5vWS+@v>Y5s;y^KKyy=QGW`g*cqgl)XV5&S#o^ z3voD~Y5pz5;e4hUxDbc)nKC$t!}(0Ja3K!oGtI+=IGoQk6Bpqy3vU=U&BcW{oX<2J z7vgX}(~Mk*!}&~;av=`qGtJ9|IGoQkH5cM=KD$5;=d%iOIG+`e!}xq0!!T=c7~cmm zjLqKzF^tU5ff&Z+?}8Xc42AcnEH z24WbCEfB+4d<4WW7FR(GV{rwV*PIP|4%9LbMb1tC3Z3Pa;z^Fh`t*=7HyAQiJXlLMdpOhhfjn%L)SvD zg${%k1}_9p1vdq*2YwS63uyko_+RpG^DDk{zAB~v7rifg`@8|qyPjj7cJ~$cS@#gF z2zcIg!qur>Q(sdLs0)<~%Bj1%|AtLr7x1=3Pl|sZ!=|_k;xO&Wroap0FfR)>MP3kx zX_z*JUJ!?AnKs2<2#0H$HU(c0hiRKOMPCqyX`D8NUl50Boi@c^5Qk}=HU(f1hx3^t zFo?tXOd%M=;e4hT4B~J;QxFDmIG-sB12{=$ayXw?fgH}~l^}=nxfSGaKDU4z&gaK54l`=TuvdT_ z&gaKK4(IcuAcymLImqFBZU#A=&&xm#=d%WKIG>k-9M0z@AV=(TqyOKgT>W44|8>_H z0zWXPlkHQAT%GbETTpb4ef-8_W-pFTM3bTwMH}s9_KKoA%cfQtonF*+X2I9#1AJUF8HbyZ>}ZFEb{urauN^!BCi>nqAg(2WbyNkVPnY_Eq5#YrUk>sqAk6Hd!4|t zExm(#oxs8^y@Pw5z|t+fgL|F8;w`;{d%UoGOJ|3zU;&q&=HB;U375`%Ka62x5tp9k z-uGY`m!9T)#zHPV&H0R_TzZ=G8H>5}H0LvxbLnZ$XDsN_)11#((xtP*R=}L6F1wyc^_*eQxys{gGcQv0h63{}6pHx;Ao+ev~I!0a*cA0a*cA0a*cA z0a*cA0a*cAf&XI#GE;sicMuOL&rH#@E+D-S_XaalG^q=qi2H(>DVov+P{cjK%oI)N h0x05sU}lP@a{(0bry3^{Q?tJTP=00O1OJtc{{dup37!A| diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index a478acf..8ca07ad 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -9,11 +9,11 @@ public class DbHandler { private readonly ConcurrentQueue _contentQueue; private readonly ConcurrentQueue _discardedQueue; - - private const string UnfilteredConnectionString = "Data Source=../../../../Models/mydb.db"; - private const string DiscardedConnectionString = "Data Source=../../../../Models/Discarded.db"; - private const string FilteredConnectionString = "Data Source=../../../../Models/Filtered.db"; - private const string ResumeConnectionString = "Data Source=../../../../Models/ScannerResume.db"; + + private readonly string _unfilteredConnectionString; + private readonly string _discardedConnectionString; + private readonly string _filteredConnectionString; + private readonly string _resumeConnectionString; private readonly List _discardedConnectionStrings = []; private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)"; @@ -33,18 +33,43 @@ public class DbHandler private const string ReIndexDatabasesStatement = "REINDEX;"; private const string VacuumDatabasesStatement = "VACUUM;"; - + private readonly object _readFilteredLock = new(); private readonly object _readAndDeleteResumeLock = new(); private bool _stop; private bool _pause; private bool _paused; + + private int _contentWaitTime; + private int _discardedWaitTime; + + private readonly string _basePath; - public DbHandler(ConcurrentQueue contentQueue, ConcurrentQueue discardedQueue) + public DbHandler(ConcurrentQueue contentQueue, ConcurrentQueue discardedQueue, string basePath) { _contentQueue = contentQueue; _discardedQueue = discardedQueue; + + SetContentWaitTime(10); + SetDiscardedWaitTime(10); + + _basePath = basePath; + + _unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db"; + _discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db"; + _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db"; + _resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db"; + } + + public void SetContentWaitTime(int waitTime) + { + _contentWaitTime = waitTime; + } + + public void SetDiscardedWaitTime(int waitTime) + { + _discardedWaitTime = waitTime; } public void StartContent() @@ -55,7 +80,7 @@ public class DbHandler { if (_contentQueue.IsEmpty || _pause) { - Thread.Sleep(10); + Thread.Sleep(_contentWaitTime); _paused = true; continue; } @@ -120,7 +145,7 @@ public class DbHandler { if (_discardedQueue.IsEmpty || _pause) { - Thread.Sleep(10); + Thread.Sleep(_discardedWaitTime); _paused = true; continue; } @@ -137,9 +162,9 @@ public class DbHandler Console.WriteLine("Content DbHandler stopped."); } - private static void InsertUnfiltered(Unfiltered unfiltered) + private void InsertUnfiltered(Unfiltered unfiltered) { - using SqliteConnection connection = new(UnfilteredConnectionString); + using SqliteConnection connection = new(_unfilteredConnectionString); connection.Open(); using SqliteCommand command = new(InsertStatement, connection); @@ -168,9 +193,9 @@ public class DbHandler connection.Close(); } - private static void InsertFiltered(Filtered filtered) + private void InsertFiltered(Filtered filtered) { - using SqliteConnection connection = new(FilteredConnectionString); + using SqliteConnection connection = new(_filteredConnectionString); connection.Open(); using SqliteCommand command = new(InsertIntoFiltered, connection); @@ -209,9 +234,9 @@ public class DbHandler connection.Close(); } - private static void InsertResumeObject(ScannerResumeObject resumeObject) + private void InsertResumeObject(ScannerResumeObject resumeObject) { - using SqliteConnection connection = new(ResumeConnectionString); + using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); using SqliteCommand command = new(InsertIntoResume, connection); @@ -228,9 +253,9 @@ public class DbHandler connection.Close(); } - private static void UpdateUnfiltered(Unfiltered unfiltered) + private void UpdateUnfiltered(Unfiltered unfiltered) { - using SqliteConnection connection = new(UnfilteredConnectionString); + using SqliteConnection connection = new(_unfilteredConnectionString); connection.Open(); using SqliteCommand command = new(UpdateUnfilteredStatement, connection); @@ -241,9 +266,9 @@ public class DbHandler connection.Close(); } - public static Unfiltered? ReadUnfilteredWithId(long id) + public Unfiltered? ReadUnfilteredWithId(long id) { - using SqliteConnection connection = new(UnfilteredConnectionString); + using SqliteConnection connection = new(_unfilteredConnectionString); connection.Open(); using SqliteCommand command = new(ReadUnfilteredStatement, connection); @@ -266,11 +291,11 @@ public class DbHandler return unfiltered; } - public static long GetUnfilteredIndexes() + public long GetUnfilteredIndexes() { long rowId = 0; - using SqliteConnection connection = new(UnfilteredConnectionString); + using SqliteConnection connection = new(_unfilteredConnectionString); connection.Open(); using SqliteCommand command = new(ReadUnfilteredIdsStatement, connection); @@ -289,11 +314,11 @@ public class DbHandler return rowId; } - public static long GetFilteredIndexes() + public long GetFilteredIndexes() { long rowId = 0; - using SqliteConnection connection = new(FilteredConnectionString); + using SqliteConnection connection = new(_filteredConnectionString); connection.Open(); using SqliteCommand command = new(ReadFilteredIdsStatement, connection); @@ -347,7 +372,7 @@ public class DbHandler { lock (_readFilteredLock) { - using SqliteConnection connection = new(FilteredConnectionString); + using SqliteConnection connection = new(_filteredConnectionString); connection.Open(); using SqliteCommand command = new(ReadFilteredStatement, connection); @@ -377,7 +402,7 @@ public class DbHandler { lock (_readAndDeleteResumeLock) { - using SqliteConnection connection = new(ResumeConnectionString); + using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection); @@ -417,21 +442,21 @@ public class DbHandler Thread.Sleep(5000); // Just for safety. } - SqliteConnection connection = new(DiscardedConnectionString); + SqliteConnection connection = new(_discardedConnectionString); connection.Open(); SqliteCommand command = new(ReIndexDatabasesStatement, connection); _ = command.ExecuteNonQuery(); connection.Close(); - connection = new(FilteredConnectionString); + connection = new(_filteredConnectionString); connection.Open(); command = new(ReIndexDatabasesStatement, connection); _ = command.ExecuteNonQuery(); connection.Close(); - connection = new(UnfilteredConnectionString); + connection = new(_unfilteredConnectionString); connection.Open(); command = new(ReIndexDatabasesStatement, connection); @@ -456,21 +481,21 @@ public class DbHandler Thread.Sleep(5000); // Just for safety. } - SqliteConnection connection = new(DiscardedConnectionString); + SqliteConnection connection = new(_discardedConnectionString); connection.Open(); SqliteCommand command = new(VacuumDatabasesStatement, connection); _ = command.ExecuteNonQuery(); connection.Close(); - connection = new(FilteredConnectionString); + connection = new(_filteredConnectionString); connection.Open(); command = new(VacuumDatabasesStatement, connection); _ = command.ExecuteNonQuery(); connection.Close(); - connection = new(UnfilteredConnectionString); + connection = new(_unfilteredConnectionString); connection.Open(); command = new(VacuumDatabasesStatement, connection); @@ -486,7 +511,7 @@ public class DbHandler private string CreateDiscardedDb(int threadNumber) { - string databaseName = $"Data Source=../../../../Models/Discarded{threadNumber}.db"; + string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db"; const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))"; diff --git a/Models/Model/Backend/DatabaseSizes.cs b/Models/Model/Backend/DatabaseSizes.cs index 9720a0e..ba4c06b 100644 --- a/Models/Model/Backend/DatabaseSizes.cs +++ b/Models/Model/Backend/DatabaseSizes.cs @@ -1,16 +1,10 @@ -using MessagePack; - namespace Models.Model.Backend; -[MessagePackObject] public struct DatabaseSizes { - [Key(0)] public double DiscardedDbSize { get; set; } - - [Key(1)] + public double FilteredDbSize { get; set; } - - [Key(2)] + public double MyDbSize { get; set; } } \ No newline at end of file diff --git a/Models/Model/Backend/RuntimeVariable.cs b/Models/Model/Backend/RuntimeVariable.cs new file mode 100644 index 0000000..536a86f --- /dev/null +++ b/Models/Model/Backend/RuntimeVariable.cs @@ -0,0 +1,9 @@ +namespace Models.Model.Backend; + +public enum RuntimeVariable +{ + DbContent, + DbDiscarded, + ContentFilter, + ScannerTimeout +} \ No newline at end of file diff --git a/Models/Model/External/CommunicationCommand.cs b/Models/Model/External/CommunicationCommand.cs index e37fa86..cc314a1 100644 --- a/Models/Model/External/CommunicationCommand.cs +++ b/Models/Model/External/CommunicationCommand.cs @@ -7,4 +7,5 @@ public enum CommunicationCommand StopScanning, DbReindex, DbVacuum, + ChangeRuntimeVariable, } \ No newline at end of file diff --git a/Models/Model/External/CommunicationObject.cs b/Models/Model/External/CommunicationObject.cs index a6f6231..9fa6609 100644 --- a/Models/Model/External/CommunicationObject.cs +++ b/Models/Model/External/CommunicationObject.cs @@ -1,13 +1,17 @@ -using MessagePack; - namespace Models.Model.External; -[MessagePackObject] +//[MessagePackObject] public class CommunicationObject { - [Key(0)] + //[Key(0)] public CommunicationCommand Command { get; set; } - [Key(1)] - public string? SearchTerm { get; set; } + //[Key(1)] + public string? SearchTerm { get; set; } = ""; + + //[Key(2)] + public string? Variable { get; set; } = ""; + + //[Key(3)] + public string? VariableValue { get; set; } = ""; } \ No newline at end of file diff --git a/Models/Model/External/CommunicationResult.cs b/Models/Model/External/CommunicationResult.cs index 574a36b..95808d9 100644 --- a/Models/Model/External/CommunicationResult.cs +++ b/Models/Model/External/CommunicationResult.cs @@ -1,13 +1,8 @@ -using MessagePack; - namespace Models.Model.External; -[MessagePackObject] public class CommunicationResult { - [Key(0)] public List? Result { get; set; } - [Key(1)] public ScanningStatus? Status { get; set; } } \ No newline at end of file diff --git a/Models/Model/External/ScanningStatus.cs b/Models/Model/External/ScanningStatus.cs index 3b94356..29a09e4 100644 --- a/Models/Model/External/ScanningStatus.cs +++ b/Models/Model/External/ScanningStatus.cs @@ -1,29 +1,18 @@ -using MessagePack; -using Models.Model.Backend; - namespace Models.Model.External; -[MessagePackObject] public struct ScanningStatus { - [Key(0)] public float PercentageOfIpv4Scanned { get; set; } - [Key(1)] public long TotalFiltered { get; set; } - [Key(2)] public long AmountOfIpv4Left { get; set; } - [Key(3)] public long TotalDiscarded { get; set; } - [Key(4)] public double DiscardedDbSize { get; set; } - - [Key(5)] + public double FilteredDbSize { get; set; } - - [Key(6)] + public double MyDbSize { get; set; } } \ No newline at end of file diff --git a/Models/Models.csproj b/Models/Models.csproj index 26af8be..f009e73 100644 --- a/Models/Models.csproj +++ b/Models/Models.csproj @@ -7,12 +7,6 @@ - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db index 60a12f6ee4789a4e26af1a4ddf65be7f45dfc765..a44725c94c534967899d2392373bfd3dd9f212b2 100644 GIT binary patch delta 116 zcmWm2u?@m76h+bZz6*Y&#NV`#86u^pu*4v&;x-+u5>rum0(M}9G;Si4q`1XBDW>?m za(?x9n=w2t9RT_YU-K9ypR4cKA^^77&%XA_A-wJ8PzV%<|Jf-;L>mz#lq1`Z!rdcM NR4gr3mCXSstQ(>4843UZ delta 116 zcmZp0XmFSy&3J91j5Gf=2EDXbybKHsjC|i2_`Yv86zJsJ{GE@Nk%xu<2m}8Y{%8DG z`HyTC5ZJU~w)@AEzRR2wMz&N!K}Y^E<>lKpDFv0hTb z&ZgqGc5!d%tF#l3@9AJ^e?>ZF7gYzXKv2{;zaKXmPFNom=}V%nq8CJkqUV~*&#BPW z|DK4izB@C25%Cl~B^p!oglL_j$3&xw9ubWwdT3V}F%iTZ=wbOhT_6bgeOlzGMSCU< zBW9N><{9 literal 24576 zcmeI2dyG`|702g2bLVy4844CfhOHW z)6kSOHZ@I6Y}M92Y)os2ZLP7bv6`CNhpDYGscCF#8d`0miEXuMdw$<@fABje!@o^q z?kd^kGr!%rpEI*(zUK@ZH>}+=(Ws5=AKy1LQR|GI7mLSZOKY`QEYTl}#WM8AeoUq> zQuZ(OIqrOYjxXtF%o{mfqFa-tug3f-<)6@}=p(8?RDq}hQ3aw3L=}iC5LF+KtyouA#G$V|EHviMe1=g`4XnrL?(Y8*V=7`wTl{+bkZQa8>T8`{^HOW!%W zITFuKnKmu{@#*-P1Bb^Z_UvmMvY!*;r?gvaYF(obT%;~{O&Om3CdW^z&E=oAabjqE zVvz2?y`~;3+*$9~@VRds+&$hH8tyy1Z&zcyy{4}zKikaaH1tOwQ3aw3L=}iC5LFT>6^JSjRUoQBRDq}h zQ3aw3L=}iC@WH5nA5YFmoc*^I@tJAziu>;wyNv&yf69N*Kk47$@AfzO%l&!&R6kRG zyZl=Dh4K^Shsr0*$I2t+f%1~_obu#yqV!hj)zWjN$4d{CPLz(6c9r@{i%XZ3D#d>n zPZxhve5UwV@r%WKisQu{#p{Ya#r9&k@Lu7K!b^px3*Ri8kEg?izlLO%a) z{`LIN^H1g<$={p5D}QT#bAEY#UVdsm zujZc1J)V0YcOrKrw=36|Tb!GjJ1_fw_KobXvQK5dmc2K7Ji9x)F}ox?J3A@!pUmmZ z%b6c#9?jgBX=X+<*JqYzF3+5wNv7XS|2F;O^rPv|r|(YhNpDPFmA*9HmU=(+$J8%U z-%mZ9`b_G!)KF?osyj73+RkxFRt#;m7|O|3m!6_;=$E#y=fDjHgkx*_Dk=&&3nDj(PLxzm5fq`r|*S zHaitv2wGQk0ceM!J)rXy?FRj*qFtc#6zv4PT+uq{Ttz!TFH>|r=p0FV*lo6=>^4hL zcDqzjcDqDTcAKdvyIrg(yM07ac57FZ-DW7tZZ%1}*=@R_>~@i&>^4nNcDqnfcDq1P zcAKgwyM0(ub~|5DcAKIoyG@p~i`_n?D7#Hkl-;U|vRg$_c573V-Of{#-F!vat*j`! zl@w*SqNJVdR#24P@`|#XrzpGS6lJ%pqU@GYl-<&bvRg_~c1tSCZV5^2>=sv)-C~Nu z?QR93c9()MJFXza?o<$7cPI$0V+z9Rb_F4IRKgB8-KHRvjwlGD!wSM^LO~cEQV>St z3c~21f-pLuAdL1a2%|9x=fh~9f-o9Y5Jr0ygwd@E!f20zFxsskjBZg7Mk5NssG*=S zstjji)6TwzEiyutn-~lr&ui56YP?g9oKdm%)QFrpw?#3DafppnT~vcu=}@89XRkx(pta zEL{c<%9Reo%YB7XrOV(!nbKwOphW31cu<~n89XRWx(ptaC0zy&N|G*v2jxhY!Glty z%iuv7(qVXsuTX+?89XRIx(pta9$f|x%8o9B2PH?B!Gm(6%iuw&(Pi+U%;++BP-1kM z@o1yGc!BdeXb^=(2N{btN{S9L5^WR{9b_EZC?h(^D6~;PbdWJ2SWxpMsy3+)3u>N6)h2afLCsUC+N54AsChP3o79a3 zHNOtkCiP>N%K6_~^spQCWO0_=s4EM~Zq%0rWjE@~g0dU+WqfRa;yHPK9 zsht0_%N(PLkpLD+sYW6@=Fv3PS6cg0Q+>K}a2yP|W}KIy<0rM4g0D zIRBqH38M*h5=Ms+6O`IRD#o@K`9%!6Tgi?Kyb#$aC-r=YM++9$oSrJUS(V2jx#Q^Mv0k=6}oJ zLD|zi2Mzjm29I$5w+tTP{2wwg|67K~Bb@&& zgGV_3TLurxlg{WP=rVW|Br_g%{tvI&tdH&dA0qQ; z*!e$1M#9ejAuF>??*808xr4dwxmCG(ZdxvveJA@`_NUoX*)L^JWDjR=%C5=w zWM^cHnZIXV&%BuVUgqJs$T zXce*o+S9qohK*Xqte{oS3R(rNpjFZeT1Bm(Rn`hxg{`1f+6rhrXWqjI9}dK=fR=L( zL*5E#HRmt{u7DPE4nyJ!Xf5Y3M6Q6Aat=f03TP$gFodpv7IF?l>I!Hb=P<;sATF56 zUDZIY;_BVzo~uo{i>r5=`>rU$@$n`?z{HuAzBM z>~-ql%2#gW>fy@ws5)yZT5aB@sHthSc|=imJFFOCJl$*JF z7rPyhXTz1R+|AX)mG783%WnG=Ww%j9*=?_)>?Rj<^-gw^8@hTYyU7(@y_4PKj;`Lx zZgNRi4_Cf&OIHt9zQgj$>g;y2qU^RyQFa?rl-+Jpl-+hJ%5FOpWw#p@Ww-5;^6IyF zgQDhlyxQERD7$S{l-;%{%5K*y%5Iw#Ww%X=vfH4d>_&?`XDMIrV7CEDc?sO4WuAMM zx4_MQMcHk=qU_eED7&pwl-<@U%5G~EWj9*wi7R6_TJ8zTZnWNWXb1Me)g~?Y1m(w# zR(yi8Td$(*c8#L!wn9;MTdpX(EmM@;mMY3_S1ZbHOC-gPxZ0$ppKGy@-L6!W-L6oS d-4-j#Zi^IUw}pzb+X6+|tw&LI>sHjb{TD^f0)zkn diff --git a/Proxy/Program.cs b/Proxy/Program.cs index 81d0b30..ae00150 100644 --- a/Proxy/Program.cs +++ b/Proxy/Program.cs @@ -31,8 +31,8 @@ progressApi.MapGet("/", () => { Command = CommunicationCommand.GetScanningProgress }; - - byte[] bytes = MessagePackSerializer.Serialize(communicationObject); + + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject); using RequestSocket client = new(); client.Connect("tcp://127.0.0.1:5556"); @@ -40,7 +40,7 @@ progressApi.MapGet("/", () => byte[] msg = client.ReceiveFrameBytes(); client.Close(); - return MessagePackSerializer.Deserialize(msg, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray)); + return JsonSerializer.Deserialize(msg); }); RouteGroupBuilder searchApi = app.MapGroup("/search"); @@ -50,8 +50,8 @@ searchApi.MapGet("/{term}", (string term) => CommunicationObject communicationObject = new(); communicationObject.Command = CommunicationCommand.GetSearches; communicationObject.SearchTerm = term; - - byte[] bytes = MessagePackSerializer.Serialize(communicationObject); + + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject); using RequestSocket client = new(); client.Connect("tcp://127.0.0.1:5556"); @@ -66,6 +66,7 @@ app.Run(); [JsonSerializable(typeof(ScanningStatus))] [JsonSerializable(typeof(SearchResults))] +[JsonSerializable(typeof(CommunicationObject))] internal partial class AppJsonSerializerContext : JsonSerializerContext { } \ No newline at end of file diff --git a/Proxy/Proxy.csproj b/Proxy/Proxy.csproj index 4f7dbe2..fd6bb2b 100644 --- a/Proxy/Proxy.csproj +++ b/Proxy/Proxy.csproj @@ -5,16 +5,10 @@ enable enable true - true + false - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index 66ca8e5..8248f74 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -8,6 +8,7 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -15,9 +16,12 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded \ No newline at end of file diff --git a/identifier.sqlite b/identifier.sqlite new file mode 100644 index 0000000..e69de29