diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 368e458..29c685b 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -27,6 +27,7 @@ public class ContentFilter _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; + SetTimeout(3000); } diff --git a/Backend/Handler/IpFilterHandler.cs b/Backend/Handler/IpFilterHandler.cs index 7462ff7..00f4a02 100644 --- a/Backend/Handler/IpFilterHandler.cs +++ b/Backend/Handler/IpFilterHandler.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Diagnostics; using Backend.Helper; using Models.Model.Backend; @@ -10,6 +11,7 @@ public class IpFilterHandler private readonly ConcurrentQueue _unfilteredQueue; private readonly ConcurrentQueue _preFilteredQueue; private bool _stop; + private bool _stopAutoscaledThreads; private int _timeout; public IpFilterHandler(ConcurrentQueue discardedQueue, @@ -44,7 +46,7 @@ public class IpFilterHandler f.Start(handle); Console.WriteLine($"Filter thread ({i}) started"); - Thread.Sleep(25); + Thread.Sleep(128); continue; } @@ -59,6 +61,54 @@ public class IpFilterHandler return waitHandles; } + public void AutoScaler() + { + int i = 0; + int j = 0; + + while (!_stop) + { + if (_preFilteredQueue.Count >= 2000) + { + if (i == 10) + { + _stopAutoscaledThreads = false; + + while (!_stopAutoscaledThreads) + { + if (_preFilteredQueue.Count <= 2000) + { + if (j == 1000) + { + _stopAutoscaledThreads = true; + } + + j++; + + Thread.Sleep(128); + } + else + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + Thread f = new (Filter_AutoScaler!); + f.Start(handle); + Thread.Sleep(16); + } + } + } + + i++; + } + else + { + i = 0; + j = 0; + } + + Thread.Sleep(128); + } + } + private void Filter(object obj) { while (!_stop) @@ -85,6 +135,32 @@ public class IpFilterHandler ((EventWaitHandle) obj).Set(); } + private void Filter_AutoScaler(object obj) + { + while (!_stopAutoscaledThreads) + { + if (_preFilteredQueue.IsEmpty) + { + Thread.Sleep(_timeout); + continue; + } + + _preFilteredQueue.TryDequeue(out FilterQueueItem item); + + (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443); + + if (ports is { Item1: 0, Item2: 0 }) + { + _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode)); + continue; + } + + _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports)); + } + + ((EventWaitHandle) obj).Set(); + } + private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) { return new() diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index 2971008..65f2aaf 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -33,7 +33,7 @@ public class IpScanner _discardedQueue = discardedQueue; _resumeQueue = resumeQueue; - SetTimeout(128); + SetTimeout(16); } public void SetTimeout(int milliseconds) @@ -76,7 +76,7 @@ public class IpScanner f.Start(scanSettings); Console.WriteLine($"Scanner thread ({i}) started"); - Thread.Sleep(50); + Thread.Sleep(128); continue; } @@ -171,13 +171,13 @@ public class IpScanner _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address); if (address is not null) { - responseCode = /*IPStatus.TimedOut;*/ ping.Send(address, _timeout, buf, null).Status; - //Thread.Sleep(16); + responseCode = ping.Send(address, _timeout, buf, null).Status; + //Thread.Sleep(256); } } - catch + catch (Exception e) { - // + Console.WriteLine(e); } if (responseCode != IPStatus.Success) diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index bafcdac..4fb3e54 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -34,32 +34,35 @@ public class ThreadHandler public void Start() { - Thread scanner = new(StartScanner); - Thread ipFilter = new(StartIpFilter); + //Thread scanner = new(StartScanner); + //Thread ipFilter = new(StartIpFilter); Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); Thread discarded = new(StartDiscardedDbHandler); Thread filtered = new(StartFilteredDbHandler); Thread resume = new(StartResumeDbHandler); - Thread communication = new(StartCommunicationHandler); + //Thread communication = new(StartCommunicationHandler); + //Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler); - scanner.Start(); - ipFilter.Start(); + //ipFilter.Start(); + //scanner.Start(); + //ipFilterAutoScaler.Start(); indexer.Start(); database.Start(); discarded.Start(); filtered.Start(); resume.Start(); - communication.Start(); + //communication.Start(); - scanner.Join(); - ipFilter.Join(); + //scanner.Join(); + //ipFilter.Join(); indexer.Join(); database.Join(); discarded.Join(); filtered.Join(); resume.Join(); - communication.Join(); + //communication.Join(); + //ipFilterAutoScaler.Join(); } private void StartScanner() @@ -91,6 +94,11 @@ public class ThreadHandler _contentFilterStopped = true; } + private void StartIpFilterAutoScaler() + { + _ipFilterHandler.AutoScaler(); + } + private void StartIpFilter() { Thread.Sleep(1000); @@ -131,7 +139,7 @@ public class ThreadHandler Console.WriteLine("Discarded DbHandler finished"); } - private void StartCommunicationHandler() + /*private void StartCommunicationHandler() { WaitHandle[] wait = _communication.Start(); @@ -142,7 +150,7 @@ public class ThreadHandler _communicationStopped = true; Stop(); - } + }*/ private void Stop() { @@ -153,7 +161,7 @@ public class ThreadHandler while (stopping) { - if (_communicationStopped && _ipScannerStopped && _contentFilterStopped && _ipFilterStopped) + if (_ipScannerStopped && _contentFilterStopped && _ipFilterStopped) { _dbHandler.Stop(); stopping = false; diff --git a/Backend/Scripts/443Header.txt b/Backend/Scripts/443Header.txt index a663ab6..1e15810 100644 --- a/Backend/Scripts/443Header.txt +++ b/Backend/Scripts/443Header.txt @@ -1,9 +1,14 @@ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 188.0.5.111:443... -* Connected to 188.0.5.111 (188.0.5.111) port 443 + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 1.0.0.95:443... +* Connected to 1.0.0.95 (1.0.0.95) port 443 * ALPN: curl offers h2,http/1.1 } [5 bytes data] * TLSv1.3 (OUT), TLS handshake, Client hello (1): } [512 bytes data] - 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 \ No newline at end of file +* TLSv1.3 (IN), TLS alert, handshake failure (552): +{ [2 bytes data] +* OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* closing connection #0 +curl: (35) OpenSSL/3.2.2: error:0A000410:SSL routines::ssl/tls alert handshake failure diff --git a/Backend/Scripts/80Header.txt b/Backend/Scripts/80Header.txt index 00cc4a2..178cfa0 100644 --- a/Backend/Scripts/80Header.txt +++ b/Backend/Scripts/80Header.txt @@ -1,7 +1,36 @@ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed - 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 188.0.5.111:80... - 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0* Connection timed out after 10002 milliseconds - 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 -* closing connection #0 -curl: (28) Connection timed out after 10002 milliseconds + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 1.0.0.95:80... +* Connected to 1.0.0.95 (1.0.0.95) port 80 +> HEAD / HTTP/1.1 +> Host: 1.0.0.95 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +< HTTP/1.1 403 Forbidden +< Date: Fri, 07 Feb 2025 21:37:37 GMT +< Content-Type: text/plain; charset=UTF-8 +< Content-Length: 16 +< Connection: close +< X-Frame-Options: SAMEORIGIN +< Referrer-Policy: same-origin +< Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0 +< Expires: Thu, 01 Jan 1970 00:00:01 GMT +< Server: cloudflare +< CF-RAY: 90e685af4a6b930d-CPH +< + 0 16 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* shutting down connection #0 +HTTP/1.1 403 Forbidden +Date: Fri, 07 Feb 2025 21:37:37 GMT +Content-Type: text/plain; charset=UTF-8 +Content-Length: 16 +Connection: close +X-Frame-Options: SAMEORIGIN +Referrer-Policy: same-origin +Cache-Control: private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0 +Expires: Thu, 01 Jan 1970 00:00:01 GMT +Server: cloudflare +CF-RAY: 90e685af4a6b930d-CPH + diff --git a/Models/CompressedDatabases.db b/Models/CompressedDatabases.db new file mode 100644 index 0000000..4a54790 Binary files /dev/null and b/Models/CompressedDatabases.db differ diff --git a/Models/Filtered.db b/Models/Filtered.db new file mode 100644 index 0000000..ccb8ab2 Binary files /dev/null and b/Models/Filtered.db differ diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db new file mode 100644 index 0000000..a44725c Binary files /dev/null and b/Models/ScannerResume.db differ diff --git a/Models/mydb.db b/Models/mydb.db new file mode 100644 index 0000000..f0567d0 Binary files /dev/null and b/Models/mydb.db differ