owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Implement autosave

Browse Source
This commit is contained in:
owner
2025-02-08 13:02:18 +01:00
parent 45b6d8ecab
commit 8d659d4261
20 changed files with 511 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Backend/Handler/ContentFilter.cs
+80 -13
View File
@@ -6,10 +6,24 @@ using Models.Model.Backend;
namespace Backend.Handler;
public class Content
{
public int Port1 { get; set; }
public int Port2 { get; set; }
public Ip Ip { get; set; }
}
public class ContentThread
{
public int ThreadId { get; set; }
public EventWaitHandle EventWaitHandle { get; set; }
}
public class ContentFilter
{
private readonly ConcurrentQueue<Filtered> _queue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Content?> _contentQueue = new();
private readonly DbHandler _dbHandler;
private readonly string _getDomainPort80;
private readonly string _getDomainPort443;
@@ -58,6 +72,11 @@ public class ContentFilter
{
if (_stop) break;
if (_contentQueue.Count >= 500)
{
Thread.Sleep(1000);
}
Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
if (unfiltered.Filtered) continue;
@@ -78,13 +97,13 @@ public class ContentFilter
{
continue;
}
Content content = new();
content.Ip = ip;
content.Port1 = unfiltered.Port1;
content.Port2 = unfiltered.Port2;
Filtered filtered = GetSiteData(ip);
filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2;
_queue.Enqueue(filtered);
_contentQueue.Enqueue(content);
}
Thread.Sleep(_timeOut);
@@ -92,11 +111,59 @@ public class ContentFilter
((EventWaitHandle) obj).Set();
}
private Filtered GetSiteData(Ip ip)
public WaitHandle[] StartFilterThread(int threads)
{
StartProcess(ip, 80);
StartProcess(ip, 443);
WaitHandle[] waitHandle = new WaitHandle[threads];
for (int i = 0; i < threads; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
ContentThread contentThread = new();
contentThread.ThreadId = i;
contentThread.EventWaitHandle = handle;
waitHandle[i] = handle;
Thread thread = new(FilterThread!);
thread.Start(contentThread);
}
return waitHandle;
}
private void FilterThread(object obj)
{
ContentThread thread = (ContentThread) obj;
while (!_stop)
{
if (_contentQueue.IsEmpty)
{
Thread.Sleep(1000);
}
_contentQueue.TryDequeue(out Content? content);
if (content is null)
{
continue;
}
Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
filtered.Port1 = content.Port1;
filtered.Port2 = content.Port2;
_queue.Enqueue(filtered);
}
thread.EventWaitHandle.Set();
}
private Filtered GetSiteData(Ip ip, int threadId)
{
StartProcess(ip, 80, threadId);
StartProcess(ip, 443, threadId);
string url1 = "";
string url2 = "";
@@ -125,7 +192,7 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
while (streamReader.Peek() != -1)
{
@@ -192,7 +259,7 @@ public class ContentFilter
return siteData;
}
private void StartProcess(Ip ip, int port)
private void StartProcess(Ip ip, int port, int threadId)
{
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
@@ -200,7 +267,7 @@ public class ContentFilter
proc.StartInfo = new()
{
FileName = "/bin/bash",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
UseShellExecute = false,
RedirectStandardOutput = false,
RedirectStandardError = false,
Backend/Handler/IpFilterHandler.cs
+1
View File
@@ -73,6 +73,7 @@ public class IpFilterHandler
if (i == 10)
{
_stopAutoscaledThreads = false;
Console.WriteLine("Autoscaler started");
while (!_stopAutoscaledThreads)
{
Backend/Handler/IpScanner.cs
+50 -4
View File
@@ -108,16 +108,26 @@ public class IpScanner
if (resumeNow is not null)
{
if (resumeNow.Completed)
{
return;
}
scanSettings.Start = resumeNow.FirstByte;
scanSettings.End = resumeNow.EndRange;
secondByte = resumeNow.SecondByte;
thirdByte = resumeNow.ThirdByte;
fourthByte = resumeNow.FourthByte;
}
else
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
}
// Empty buffer so we use the lowest abstracted ping.Send() method.
byte[] buf = [];
using Ping ping = new();
int x = 0;
for (int i = scanSettings.Start; i < scanSettings.End; i++)
{
@@ -148,6 +158,14 @@ public class IpScanner
for (int l = fourthByte; l < 256; l++)
{
if (x == 75_000)
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
x = 0;
}
x++;
if (_stop)
{
resumeObject.FourthByte = l;
@@ -172,12 +190,12 @@ public class IpScanner
if (address is not null)
{
responseCode = ping.Send(address, _timeout, buf, null).Status;
//Thread.Sleep(256);
//Thread.Sleep(4);
}
}
catch (Exception e)
catch
{
Console.WriteLine(e);
//
}
if (responseCode != IPStatus.Success)
@@ -211,11 +229,39 @@ public class IpScanner
Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
}
if (_stop)
{
resumeObject.Paused = true;
}
else
{
resumeObject.Completed = true;
}
resumeObject.Operation = Operations.Update;
_resumeQueue.Enqueue(resumeObject);
scanSettings.Handle!.Set();
}
private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
{
ScannerResumeObject resumeObject = new();
resumeObject.ThreadNumber = threadNumber;
resumeObject.StartRange = startRange;
resumeObject.EndRange = endRange;
resumeObject.FirstByte = firstByte;
resumeObject.SecondByte = secondByte;
resumeObject.ThirdByte = thirdByte;
resumeObject.FourthByte = fourthByte;
resumeObject.Paused = paused;
resumeObject.Completed = completed;
resumeObject.Operation = operation;
_resumeQueue.Enqueue(resumeObject);
}
private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
{
Backend/Handler/ThreadHandler.cs
+24 -14
View File
@@ -34,35 +34,38 @@ public class ThreadHandler
public void Start()
{
//Thread scanner = new(StartScanner);
//Thread ipFilter = new(StartIpFilter);
Thread scanner = new(StartScanner);
Thread ipFilter = new(StartIpFilter);
Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
Thread filtered = new(StartFilteredDbHandler);
Thread resume = new(StartResumeDbHandler);
//Thread communication = new(StartCommunicationHandler);
//Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread communication = new(StartCommunicationHandler);
Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread contentFilterThread = new(StartContentFilterThread);
//ipFilter.Start();
//scanner.Start();
//ipFilterAutoScaler.Start();
ipFilter.Start();
scanner.Start();
ipFilterAutoScaler.Start();
indexer.Start();
database.Start();
discarded.Start();
filtered.Start();
resume.Start();
//communication.Start();
communication.Start();
contentFilterThread.Start();
//scanner.Join();
//ipFilter.Join();
scanner.Join();
ipFilter.Join();
indexer.Join();
database.Join();
discarded.Join();
filtered.Join();
resume.Join();
//communication.Join();
//ipFilterAutoScaler.Join();
communication.Join();
ipFilterAutoScaler.Join();
contentFilterThread.Join();
}
private void StartScanner()
@@ -94,6 +97,13 @@ public class ThreadHandler
_contentFilterStopped = true;
}
private void StartContentFilterThread()
{
WaitHandle[] wait = _contentFilter.StartFilterThread(4);
WaitHandle.WaitAll(wait);
}
private void StartIpFilterAutoScaler()
{
_ipFilterHandler.AutoScaler();
@@ -139,7 +149,7 @@ public class ThreadHandler
Console.WriteLine("Discarded DbHandler finished");
}
/*private void StartCommunicationHandler()
private void StartCommunicationHandler()
{
WaitHandle[] wait = _communication.Start();
@@ -150,7 +160,7 @@ public class ThreadHandler
_communicationStopped = true;
Stop();
}*/
}
private void Stop()
{