owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Way too much happened for me to create a commit message

Browse Source
This commit is contained in:
owner
2025-02-07 18:44:22 +01:00
parent 680b295f9e
commit 83897c72cf
17 changed files with 233 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Backend/Handler/ContentFilter.cs
-39
View File
@@ -99,10 +99,6 @@ public class ContentFilter
string url1 = "";
string url2 = "";
string title1 = "";
string title2 = "";
string description1 = "";
string description2 = "";
bool robotsTxt1 = false;
bool robotsTxt2 = false;
string serverType1 = "";
@@ -161,37 +157,6 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
if (ports[i] == 80)
{
if (string.IsNullOrWhiteSpace(url1)) continue;
try
{
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
title1 = temp.Item1;
description1 = temp.Item2;
}
catch
{
//
}
}
else
{
if (string.IsNullOrWhiteSpace(url2)) continue;
try
{
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
title2 = temp.Item1;
description2 = temp.Item2;
}
catch
{
//
}
}
if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
}
@@ -201,10 +166,6 @@ public class ContentFilter
Ip = ip,
Url1 = url1,
Url2 = url2,
Title1 = title1,
Title2 = title2,
Description1 = description1,
Description2 = description2,
ServerType1 = serverType1,
ServerType2 = serverType2,
RobotsTXT1 = robotsTxt1,
Backend/Handler/IpFilterHandler.cs
+118
View File
@@ -0,0 +1,118 @@
using System.Collections.Concurrent;
using Backend.Helper;
using Models.Model.Backend;
namespace Backend.Handler;
public class IpFilterHandler
{
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private bool _stop;
private int _timeout;
public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
ConcurrentQueue<FilterQueueItem> filteredQueue)
{
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_preFilteredQueue = filteredQueue;
_timeout = 16;
}
public List<WaitHandle[]> Start(int threadCount)
{
WaitHandle[] waitHandle = new WaitHandle[64];
int counter = 0;
List<WaitHandle[]> waitHandles = [];
for (int i = 0; i < threadCount; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
if (counter < 64)
{
waitHandle[counter] = handle;
counter++;
Thread f = new (Filter!);
f.Start(handle);
Console.WriteLine($"Filter thread ({i}) started");
Thread.Sleep(25);
continue;
}
counter = 0;
waitHandles.Add(waitHandle);
waitHandle = new WaitHandle[64];
}
return waitHandles;
}
private void Filter(object obj)
{
while (!_stop)
{
if (_preFilteredQueue.IsEmpty)
{
Thread.Sleep(_timeout);
continue;
}
_preFilteredQueue.TryDequeue(out FilterQueueItem item);
(int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
}
((EventWaitHandle) obj).Set();
}
private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
{
return new()
{
Ip = ip,
ResponseCode = responseCode
};
}
private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
{
Unfiltered unfiltered = new()
{
Ip = ip,
Port1 = ports.Item1,
Port2 = ports.Item2,
Filtered = false
};
return new()
{
Unfiltered = unfiltered,
Operations = Operations.Insert
};
}
public void Stop()
{
_stop = true;
}
}
Backend/Handler/IpScanner.cs
+21 -30
View File
@@ -18,22 +18,22 @@ public class ScanSettings
public class IpScanner
{
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly DbHandler _dbHandler;
private bool _stop;
private int _timeout;
public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
)
public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler,
ConcurrentQueue<FilterQueueItem> preFilteredQueue)
{
_dbHandler = dbHandler;
_preFilteredQueue = preFilteredQueue;
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_resumeQueue = resumeQueue;
SetTimeout(64);
SetTimeout(128);
}
public void SetTimeout(int milliseconds)
@@ -76,7 +76,7 @@ public class IpScanner
f.Start(scanSettings);
Console.WriteLine($"Scanner thread ({i}) started");
Thread.Sleep(100);
Thread.Sleep(50);
continue;
}
@@ -138,7 +138,12 @@ public class IpScanner
if (_discardedQueue.Count >= 2000)
{
Thread.Sleep(500);
Thread.Sleep(1000);
}
if (_preFilteredQueue.Count >= 2000)
{
Thread.Sleep(1000);
}
for (int l = fourthByte; l < 256; l++)
@@ -156,7 +161,7 @@ public class IpScanner
Ip3 = k,
Ip4 = l
};
IPStatus responseCode = IPStatus.Unknown;
try
@@ -166,8 +171,8 @@ public class IpScanner
_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
if (address is not null)
{
responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
Thread.Sleep(_timeout);
responseCode = /*IPStatus.TimedOut;*/ ping.Send(address, _timeout, buf, null).Status;
//Thread.Sleep(16);
}
}
catch
@@ -181,15 +186,7 @@ public class IpScanner
continue;
}
(int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
_preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
}
if (_stop)
@@ -229,21 +226,15 @@ public class IpScanner
};
}
private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
{
Unfiltered unfiltered = new()
FilterQueueItem filterQueueItem = new()
{
Ip = ip,
Port1 = ports.Item1,
Port2 = ports.Item2,
Filtered = false
ResponseCode = responseCode
};
return new()
{
Unfiltered = unfiltered,
Operations = Operations.Insert
};
return filterQueueItem;
}
public void Stop()
Backend/Handler/ThreadHandler.cs
+27 -4
View File
@@ -10,10 +10,12 @@ public class ThreadHandler
private readonly Communication _communication;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private readonly IpFilterHandler _ipFilterHandler;
private bool _communicationStopped;
private bool _ipScannerStopped;
private bool _contentFilterStopped;
private bool _ipFilterStopped;
public ThreadHandler(string path)
{
@@ -21,16 +23,19 @@ public class ThreadHandler
ConcurrentQueue<Discarded> discardedQueue = new();
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
_dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
_ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
_ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
_contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
_communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
_ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue);
}
public void Start()
{
Thread scanner = new(StartScanner);
Thread ipFilter = new(StartIpFilter);
Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
@@ -39,6 +44,7 @@ public class ThreadHandler
Thread communication = new(StartCommunicationHandler);
scanner.Start();
ipFilter.Start();
indexer.Start();
database.Start();
discarded.Start();
@@ -47,6 +53,7 @@ public class ThreadHandler
communication.Start();
scanner.Join();
ipFilter.Join();
indexer.Join();
database.Join();
discarded.Join();
@@ -57,7 +64,7 @@ public class ThreadHandler
private void StartScanner()
{
Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
List<WaitHandle[]> wait = _ipScanner.Start(256);
@@ -83,6 +90,22 @@ public class ThreadHandler
_contentFilterStopped = true;
}
private void StartIpFilter()
{
Thread.Sleep(1000);
List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
for (int i = 0; i < wait.Count; i++)
{
WaitHandle.WaitAll(wait[i]);
}
Console.WriteLine("Ip filter finished");
_ipFilterStopped = true;
}
private void StartDbHandler()
{
@@ -101,7 +124,7 @@ public class ThreadHandler
private void StartDiscardedDbHandler()
{
WaitHandle[] wait = _dbHandler.Start(5);
WaitHandle[] wait = _dbHandler.Start(4);
WaitHandle.WaitAll(wait);
@@ -130,7 +153,7 @@ public class ThreadHandler
while (stopping)
{
if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
if (_communicationStopped && _ipScannerStopped && _contentFilterStopped && _ipFilterStopped)
{
_dbHandler.Stop();
stopping = false;