owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Made a lot of changes. Enhanced memory usage.

Browse Source
This commit is contained in:
owner
2025-02-16 09:38:00 +01:00
parent 8d659d4261
commit c70b42c9d7
169 changed files with 3628 additions and 1304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Backend/Handler/Communication.cs
-201
View File
@@ -1,201 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Backend.Helper;
using Models.Handler;
using Models.Model.Backend;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
namespace Backend.Handler;
public class Communication
{
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private bool _isRunning = true;
private string _basePath;
public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
{
_dbHandler = dbHandler;
_threadHandler = threadHandler;
_ipScanner = ipScanner;
_contentFilter = contentFilter;
_basePath = basePath;
}
public WaitHandle[] Start()
{
WaitHandle[] waitHandles = new WaitHandle[1];
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
waitHandles[0] = handle;
Thread thread = new(Server!);
thread.Start(handle);
return waitHandles;
}
private void Server(object obj)
{
using ResponseSocket rep = new();
//rep.Options.IPv4Only = true;
rep.Bind("tcp://127.0.0.1:5556");
while (_isRunning)
{
byte[] message = rep.ReceiveFrameBytes();
CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
//rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
if (communicationObject is null)
{
continue;
}
OnServerOnReceiveReady(communicationObject, rep);
}
((EventWaitHandle) obj).Set();
}
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
{
switch (communicationObject.Command)
{
case CommunicationCommand.GetScanningProgress:
{
DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
long discardedIndexes = _dbHandler.GetDiscardedIndexes();
ScanningStatus status = new();
// 4294967296 is all Ipv4 addresses.
if (discardedIndexes != 0)
{
status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
}
else
{
// This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
status.PercentageOfIpv4Scanned = 0.0000000001f;
}
status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
status.TotalFiltered = _dbHandler.GetFilteredIndexes();
status.TotalDiscarded = discardedIndexes;
status.MyDbSize = databaseSizes.MyDbSize;
status.FilteredDbSize = databaseSizes.FilteredDbSize;
status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
rep.SendFrame(serializedResult);
break;
}
case CommunicationCommand.DbReindex:
{
_dbHandler.ReIndex();
SendStringResponse(rep, "All Dbs have been reindexed.");
break;
}
case CommunicationCommand.DbVacuum:
{
_dbHandler.Vacuum();
SendStringResponse(rep, "All Dbs have been vacuumed.");
break;
}
case CommunicationCommand.GetSearches:
{
if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
{
SendSearchResponse(rep, communicationObject.SearchTerm);
}
break;
}
case CommunicationCommand.StopScanning:
{
_isRunning = false;
break;
}
case CommunicationCommand.ChangeRuntimeVariable:
{
if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetContentWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetDiscardedWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_ipScanner.SetTimeout(value);
}
if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_contentFilter.SetTimeout(value);
}
rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
break;
}
}
}
private static void SendStringResponse(ResponseSocket rep, string response)
{
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
rep.SendFrame(serializedResult);
}
private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
{
//SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
SearchResults result = new()
{
Results = []
};
SearchResult lol = new()
{
Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
};
result.Results.Add(lol);
string serializedResult = JsonSerializer.Serialize(result);
rep.SendFrame(serializedResult);
}
}
Backend/Handler/ContentFilter.cs
+16 -4
View File
@@ -16,7 +16,7 @@ public class Content
public class ContentThread
{
public int ThreadId { get; set; }
public EventWaitHandle EventWaitHandle { get; set; }
public EventWaitHandle? EventWaitHandle { get; set; }
}
public class ContentFilter
@@ -25,17 +25,19 @@ public class ContentFilter
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Content?> _contentQueue = new();
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
private readonly string _getDomainPort80;
private readonly string _getDomainPort443;
private bool _stop;
private int _timeOut;
private readonly string _basePath;
public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
{
_queue = queue;
_dbHandler = dbHandler;
_basePath = basePath;
_threadHandler = threadHandler;
_unfilteredQueue = unfilteredQueue;
_getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
@@ -67,6 +69,13 @@ public class ContentFilter
while (!_stop)
{
List<long> indexes = _dbHandler.GetUnfilteredIndexes();
if (indexes.Count == 0)
{
_stop = true;
_threadHandler.Stop();
break;
}
for (int i = 0; i < indexes.Count; i++)
{
@@ -126,6 +135,8 @@ public class ContentFilter
Thread thread = new(FilterThread!);
thread.Start(contentThread);
Thread.Sleep(8);
}
return waitHandle;
@@ -133,6 +144,7 @@ public class ContentFilter
private void FilterThread(object obj)
{
Console.WriteLine("Filter Thread started.");
ContentThread thread = (ContentThread) obj;
while (!_stop)
@@ -157,7 +169,7 @@ public class ContentFilter
_queue.Enqueue(filtered);
}
thread.EventWaitHandle.Set();
thread.EventWaitHandle!.Set();
}
private Filtered GetSiteData(Ip ip, int threadId)
Backend/Handler/IpFilterHandler.cs
+40 -8
View File
@@ -1,6 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Backend.Helper;
using Models.Handler;
using Models.Model.Backend;
namespace Backend.Handler;
@@ -10,17 +11,23 @@ public class IpFilterHandler
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private DbHandler _dbHandler;
private ThreadHandler _threadHandler;
private bool _stop;
private bool _fillerStop;
private bool _stopAutoscaledThreads;
private int _timeout;
public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
ConcurrentQueue<FilterQueueItem> filteredQueue)
ConcurrentQueue<FilterQueueItem> preFilteredQueue,
DbHandler dbHandler, ThreadHandler threadHandler)
{
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_preFilteredQueue = filteredQueue;
_preFilteredQueue = preFilteredQueue;
_dbHandler = dbHandler;
_threadHandler = threadHandler;
_timeout = 16;
}
@@ -46,7 +53,7 @@ public class IpFilterHandler
f.Start(handle);
Console.WriteLine($"Filter thread ({i}) started");
Thread.Sleep(128);
Thread.Sleep(16);
continue;
}
@@ -112,29 +119,54 @@ public class IpFilterHandler
private void Filter(object obj)
{
int counter = 0;
while (!_stop)
{
if (_preFilteredQueue.IsEmpty)
if (_preFilteredQueue.IsEmpty && _fillerStop)
{
Thread.Sleep(_timeout);
continue;
if (counter == 100)
{
_threadHandler.Stop();
_stop = true;
}
counter++;
Thread.Sleep(128);
}
_preFilteredQueue.TryDequeue(out FilterQueueItem item);
(int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
}
((EventWaitHandle) obj).Set();
}
public void FillFilterQueue()
{
Console.WriteLine("Fill FilterQueue started.");
while (!_stop)
{
if (_preFilteredQueue.Count > 500) continue;
if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
{
_preFilteredQueue.Enqueue(item);
}
else
{
_fillerStop = true;
}
}
}
private void Filter_AutoScaler(object obj)
{
Backend/Handler/IpScanner.cs
+117 -15
View File
@@ -1,7 +1,11 @@
using System.Buffers.Binary;
using System.Collections.Concurrent;
using System.Net;
using System.Net.NetworkInformation;
using System.Numerics;
using System.Runtime.InteropServices;
using Backend.Helper;
using Models.Experimental;
using Models.Handler;
using Models.Model.Backend;
@@ -22,7 +26,7 @@ public class IpScanner
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly DbHandler _dbHandler;
private bool _stop;
private int _timeout;
private readonly int _timeout;
public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler,
@@ -32,13 +36,8 @@ public class IpScanner
_preFilteredQueue = preFilteredQueue;
_discardedQueue = discardedQueue;
_resumeQueue = resumeQueue;
SetTimeout(16);
}
public void SetTimeout(int milliseconds)
{
_timeout = milliseconds;
_timeout = 32;
}
public List<WaitHandle[]> Start(int threads)
@@ -186,12 +185,20 @@ public class IpScanner
{
// Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
// https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
if (address is not null)
//_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
if (i % 2 == 0)
{
responseCode = ping.Send(address, _timeout, buf, null).Status;
//Thread.Sleep(4);
responseCode = IPStatus.Success;
}
else
{
responseCode = IPStatus.TimedOut;
}
//CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
Thread.Sleep(16);
}
catch
{
@@ -229,7 +236,7 @@ public class IpScanner
Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
}
if (_stop)
{
resumeObject.Paused = true;
@@ -238,9 +245,9 @@ public class IpScanner
{
resumeObject.Completed = true;
}
resumeObject.Operation = Operations.Update;
_resumeQueue.Enqueue(resumeObject);
scanSettings.Handle!.Set();
@@ -287,4 +294,99 @@ public class IpScanner
{
_stop = true;
}
private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
{
int length = ipSpan.Length;
long nonCanonical;
fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
nonCanonical = ParseNonCanonical(name, 0, ref length, true);
return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
}
private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
{
long* numPtr = stackalloc long[4];
long num1 = 0;
bool flag = false;
int index1 = 0;
int index2;
for (index2 = start; index2 < end; ++index2)
{
char ch = name[index2];
num1 = 0L;
int num2 = 10;
if (ch == '0')
{
num2 = 8;
++index2;
flag = true;
if (index2 < end)
{
switch (name[index2])
{
case 'X':
case 'x':
num2 = 16;
++index2;
flag = false;
break;
}
}
}
for (; index2 < end; ++index2)
{
char c = name[index2];
int num3;
if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
num3 = (int) c - 48;
else if (num2 == 8 && '0' <= c && c <= '7')
num3 = (int) c - 48;
else if (num2 == 16 && 'a' <= c && c <= 'f')
num3 = (int) c + 10 - 97;
else if (num2 == 16 && 'A' <= c && c <= 'F')
num3 = (int) c + 10 - 65;
else
break;
num1 = num1 * (long) num2 + (long) num3;
if (num1 > (long) uint.MaxValue)
return -1;
flag = true;
}
if (index2 < end && name[index2] == '.')
{
if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
return -1;
numPtr[index1] = num1;
++index1;
flag = false;
}
else
break;
}
if (!flag)
return -1;
if (index2 < end)
{
char ch;
if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
return -1;
end = index2;
}
numPtr[index1] = num1;
switch (index1)
{
case 0:
return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
case 1:
return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
case 2:
return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
case 3:
return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
default:
return -1;
}
}
}
Backend/Handler/ThreadHandler.cs
+90 -63
View File
@@ -7,29 +7,30 @@ namespace Backend.Handler;
public class ThreadHandler
{
private readonly DbHandler _dbHandler;
private readonly Communication _communication;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private readonly IpFilterHandler _ipFilterHandler;
private bool _communicationStopped;
private bool _ipScannerStopped;
private bool _contentFilterStopped;
private bool _ipFilterStopped;
private bool _stage1;
private bool _stage2 = true;
private bool _stage3;
ConcurrentQueue<Filtered> filteredQueue = new();
ConcurrentQueue<Discarded> discardedQueue = new();
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
public ThreadHandler(string path)
{
ConcurrentQueue<Filtered> filteredQueue = new();
ConcurrentQueue<Discarded> discardedQueue = new();
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
_dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
_dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
_ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
_contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
_communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
_ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue);
_contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
_ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
}
public void Start()
@@ -41,31 +42,66 @@ public class ThreadHandler
Thread discarded = new(StartDiscardedDbHandler);
Thread filtered = new(StartFilteredDbHandler);
Thread resume = new(StartResumeDbHandler);
Thread communication = new(StartCommunicationHandler);
Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread contentFilterThread = new(StartContentFilterThread);
ipFilter.Start();
scanner.Start();
ipFilterAutoScaler.Start();
indexer.Start();
database.Start();
discarded.Start();
filtered.Start();
resume.Start();
communication.Start();
contentFilterThread.Start();
scanner.Join();
ipFilter.Join();
indexer.Join();
database.Join();
discarded.Join();
filtered.Join();
resume.Join();
communication.Join();
ipFilterAutoScaler.Join();
contentFilterThread.Join();
Thread prefilterDb = new(StartPreFilterDbHandler);
Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
//Thread check = new(CheckQueue);
if (_stage1)
{
discarded.Start(); // de-queues from discardedQueue
prefilterDb.Start(); // de-queues from preFilteredQueue
scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
resume.Start(); // de-queues from resumeQueue
discarded.Join();
prefilterDb.Join();
scanner.Join();
resume.Join();
}
if (_stage2)
{
database.Start(); // de-queues from unfilteredQueue
discarded.Start(); // de-queues from discardedQueue
ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
database.Join();
discarded.Join();
ipFilter.Join();
ipFilterAutoScaler.Join();
fillIpFilterQueue.Join();
}
if (_stage3)
{
filtered.Start(); // de-queues from filteredQueue
database.Start(); // de-queues from unfilteredQueue
indexer.Start(); // en-queues to unfilteredQueue and contentQueue
contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
contentFilterThread.Join();
filtered.Join();
database.Join();
indexer.Join();
}
}
private void CheckQueue()
{
while (true)
{
Console.Clear();
Console.WriteLine($"filteredQueue.Count: {filteredQueue.Count}");
Console.WriteLine($"discardedQueue.Count: {discardedQueue.Count}");
Console.WriteLine($"unfilteredQueue.Count: {unfilteredQueue.Count}");
Console.WriteLine($"scannerResumeQueue.Count: {scannerResumeQueue.Count}");
Console.WriteLine($"preFilteredQueue.Count: {preFilteredQueue.Count}");
Thread.Sleep(5);
}
}
private void StartScanner()
@@ -80,8 +116,6 @@ public class ThreadHandler
}
Console.WriteLine("Scanner finished");
_ipScannerStopped = true;
}
private void StartContentFilter()
@@ -99,7 +133,7 @@ public class ThreadHandler
private void StartContentFilterThread()
{
WaitHandle[] wait = _contentFilter.StartFilterThread(4);
WaitHandle[] wait = _contentFilter.StartFilterThread(64);
WaitHandle.WaitAll(wait);
}
@@ -109,6 +143,11 @@ public class ThreadHandler
_ipFilterHandler.AutoScaler();
}
private void StartFillIpFilterQueue()
{
_ipFilterHandler.FillFilterQueue();
}
private void StartIpFilter()
{
Thread.Sleep(1000);
@@ -134,6 +173,11 @@ public class ThreadHandler
{
_dbHandler.FilteredDbHandler();
}
private void StartPreFilterDbHandler()
{
_dbHandler.PrefilteredDbHandler();
}
private void StartResumeDbHandler()
{
@@ -149,35 +193,18 @@ public class ThreadHandler
Console.WriteLine("Discarded DbHandler finished");
}
private void StartCommunicationHandler()
{
WaitHandle[] wait = _communication.Start();
WaitHandle.WaitAll(wait);
Console.WriteLine("Communicator finished");
_communicationStopped = true;
Stop();
}
private void Stop()
public void Stop()
{
Console.WriteLine("Stopping...");
_ipScanner.Stop();
_contentFilter.Stop();
_ipFilterHandler.Stop();
Console.WriteLine("Stopping Extra...");
bool stopping = true;
Thread.Sleep(30_000);
while (stopping)
{
if (_ipScannerStopped && _contentFilterStopped && _ipFilterStopped)
{
_dbHandler.Stop();
stopping = false;
}
Thread.Sleep(3000);
}
Console.WriteLine("Stopping Super Extra...");
_dbHandler.Stop();
Console.WriteLine("Stopped.");
}
}