diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 29c685b..0b444a0 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -6,10 +6,24 @@ using Models.Model.Backend; namespace Backend.Handler; +public class Content +{ + public int Port1 { get; set; } + public int Port2 { get; set; } + public Ip Ip { get; set; } +} + +public class ContentThread +{ + public int ThreadId { get; set; } + public EventWaitHandle EventWaitHandle { get; set; } +} + public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _contentQueue = new(); private readonly DbHandler _dbHandler; private readonly string _getDomainPort80; private readonly string _getDomainPort443; @@ -58,6 +72,11 @@ public class ContentFilter { if (_stop) break; + if (_contentQueue.Count >= 500) + { + Thread.Sleep(1000); + } + Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]); if (unfiltered.Filtered) continue; @@ -78,13 +97,13 @@ public class ContentFilter { continue; } + + Content content = new(); + content.Ip = ip; + content.Port1 = unfiltered.Port1; + content.Port2 = unfiltered.Port2; - Filtered filtered = GetSiteData(ip); - - filtered.Port1 = unfiltered.Port1; - filtered.Port2 = unfiltered.Port2; - - _queue.Enqueue(filtered); + _contentQueue.Enqueue(content); } Thread.Sleep(_timeOut); @@ -92,11 +111,59 @@ public class ContentFilter ((EventWaitHandle) obj).Set(); } - - private Filtered GetSiteData(Ip ip) + + public WaitHandle[] StartFilterThread(int threads) { - StartProcess(ip, 80); - StartProcess(ip, 443); + WaitHandle[] waitHandle = new WaitHandle[threads]; + + for (int i = 0; i < threads; i++) + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + ContentThread contentThread = new(); + contentThread.ThreadId = i; + contentThread.EventWaitHandle = handle; + waitHandle[i] = handle; + + Thread thread = new(FilterThread!); + thread.Start(contentThread); + } + + return waitHandle; + } + + private void FilterThread(object obj) + { + ContentThread thread = (ContentThread) obj; + + while (!_stop) + { + if (_contentQueue.IsEmpty) + { + Thread.Sleep(1000); + } + + _contentQueue.TryDequeue(out Content? content); + + if (content is null) + { + continue; + } + + Filtered filtered = GetSiteData(content.Ip, thread.ThreadId); + + filtered.Port1 = content.Port1; + filtered.Port2 = content.Port2; + + _queue.Enqueue(filtered); + } + + thread.EventWaitHandle.Set(); + } + + private Filtered GetSiteData(Ip ip, int threadId) + { + StartProcess(ip, 80, threadId); + StartProcess(ip, 443, threadId); string url1 = ""; string url2 = ""; @@ -125,7 +192,7 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt"); + using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt"); while (streamReader.Peek() != -1) { @@ -192,7 +259,7 @@ public class ContentFilter return siteData; } - private void StartProcess(Ip ip, int port) + private void StartProcess(Ip ip, int port, int threadId) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; @@ -200,7 +267,7 @@ public class ContentFilter proc.StartInfo = new() { FileName = "/bin/bash", - Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt", + Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, diff --git a/Backend/Handler/IpFilterHandler.cs b/Backend/Handler/IpFilterHandler.cs index 00f4a02..cce748d 100644 --- a/Backend/Handler/IpFilterHandler.cs +++ b/Backend/Handler/IpFilterHandler.cs @@ -73,6 +73,7 @@ public class IpFilterHandler if (i == 10) { _stopAutoscaledThreads = false; + Console.WriteLine("Autoscaler started"); while (!_stopAutoscaledThreads) { diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index 65f2aaf..8dbb371 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -108,16 +108,26 @@ public class IpScanner if (resumeNow is not null) { + if (resumeNow.Completed) + { + return; + } + scanSettings.Start = resumeNow.FirstByte; scanSettings.End = resumeNow.EndRange; secondByte = resumeNow.SecondByte; thirdByte = resumeNow.ThirdByte; fourthByte = resumeNow.FourthByte; } + else + { + CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert); + } // Empty buffer so we use the lowest abstracted ping.Send() method. byte[] buf = []; using Ping ping = new(); + int x = 0; for (int i = scanSettings.Start; i < scanSettings.End; i++) { @@ -148,6 +158,14 @@ public class IpScanner for (int l = fourthByte; l < 256; l++) { + if (x == 75_000) + { + CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update); + x = 0; + } + + x++; + if (_stop) { resumeObject.FourthByte = l; @@ -172,12 +190,12 @@ public class IpScanner if (address is not null) { responseCode = ping.Send(address, _timeout, buf, null).Status; - //Thread.Sleep(256); + //Thread.Sleep(4); } } - catch (Exception e) + catch { - Console.WriteLine(e); + // } if (responseCode != IPStatus.Success) @@ -211,11 +229,39 @@ public class IpScanner Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})"); } - + + if (_stop) + { + resumeObject.Paused = true; + } + else + { + resumeObject.Completed = true; + } + + resumeObject.Operation = Operations.Update; + _resumeQueue.Enqueue(resumeObject); scanSettings.Handle!.Set(); } + + private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation) + { + ScannerResumeObject resumeObject = new(); + resumeObject.ThreadNumber = threadNumber; + resumeObject.StartRange = startRange; + resumeObject.EndRange = endRange; + resumeObject.FirstByte = firstByte; + resumeObject.SecondByte = secondByte; + resumeObject.ThirdByte = thirdByte; + resumeObject.FourthByte = fourthByte; + resumeObject.Paused = paused; + resumeObject.Completed = completed; + resumeObject.Operation = operation; + + _resumeQueue.Enqueue(resumeObject); + } private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) { diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 4fb3e54..d1f6d8e 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -34,35 +34,38 @@ public class ThreadHandler public void Start() { - //Thread scanner = new(StartScanner); - //Thread ipFilter = new(StartIpFilter); + Thread scanner = new(StartScanner); + Thread ipFilter = new(StartIpFilter); Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); Thread discarded = new(StartDiscardedDbHandler); Thread filtered = new(StartFilteredDbHandler); Thread resume = new(StartResumeDbHandler); - //Thread communication = new(StartCommunicationHandler); - //Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler); + Thread communication = new(StartCommunicationHandler); + Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler); + Thread contentFilterThread = new(StartContentFilterThread); - //ipFilter.Start(); - //scanner.Start(); - //ipFilterAutoScaler.Start(); + ipFilter.Start(); + scanner.Start(); + ipFilterAutoScaler.Start(); indexer.Start(); database.Start(); discarded.Start(); filtered.Start(); resume.Start(); - //communication.Start(); + communication.Start(); + contentFilterThread.Start(); - //scanner.Join(); - //ipFilter.Join(); + scanner.Join(); + ipFilter.Join(); indexer.Join(); database.Join(); discarded.Join(); filtered.Join(); resume.Join(); - //communication.Join(); - //ipFilterAutoScaler.Join(); + communication.Join(); + ipFilterAutoScaler.Join(); + contentFilterThread.Join(); } private void StartScanner() @@ -94,6 +97,13 @@ public class ThreadHandler _contentFilterStopped = true; } + private void StartContentFilterThread() + { + WaitHandle[] wait = _contentFilter.StartFilterThread(4); + + WaitHandle.WaitAll(wait); + } + private void StartIpFilterAutoScaler() { _ipFilterHandler.AutoScaler(); @@ -139,7 +149,7 @@ public class ThreadHandler Console.WriteLine("Discarded DbHandler finished"); } - /*private void StartCommunicationHandler() + private void StartCommunicationHandler() { WaitHandle[] wait = _communication.Start(); @@ -150,7 +160,7 @@ public class ThreadHandler _communicationStopped = true; Stop(); - }*/ + } private void Stop() { diff --git a/Backend/Scripts/443Header0.txt b/Backend/Scripts/443Header0.txt new file mode 100644 index 0000000..740521a --- /dev/null +++ b/Backend/Scripts/443Header0.txt @@ -0,0 +1,76 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.72.3:443... +* Connected to 192.0.72.3 (192.0.72.3) port 443 +* ALPN: curl offers h2,http/1.1 +} [5 bytes data] +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +} [512 bytes data] +* TLSv1.3 (IN), TLS handshake, Server hello (2): +{ [122 bytes data] +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +{ [15 bytes data] +* TLSv1.3 (IN), TLS handshake, Certificate (11): +{ [2063 bytes data] +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +{ [79 bytes data] +* TLSv1.3 (IN), TLS handshake, Finished (20): +{ [52 bytes data] +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +} [1 bytes data] +* TLSv1.3 (OUT), TLS handshake, Finished (20): +} [52 bytes data] +* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=files.wordpress.com +* start date: Dec 16 09:49:37 2024 GMT +* expire date: Mar 16 09:49:36 2025 GMT +* issuer: C=US; O=Let's Encrypt; CN=E6 +* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway. +* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384 +* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption +} [5 bytes data] +* using HTTP/2 +* [HTTP/2] [1] OPENED stream for https://192.0.72.3/ +* [HTTP/2] [1] [:method: HEAD] +* [HTTP/2] [1] [:scheme: https] +* [HTTP/2] [1] [:authority: 192.0.72.3] +* [HTTP/2] [1] [:path: /] +* [HTTP/2] [1] [user-agent: curl/8.9.1] +* [HTTP/2] [1] [accept: */*] +} [5 bytes data] +> HEAD / HTTP/2 +> Host: 192.0.72.3 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +{ [5 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +< HTTP/2 302 +< server: nginx +< date: Fri, 07 Feb 2025 22:58:34 GMT +< content-type: text/html; charset=utf-8 +< location: https://developer.wordpress.com +< vary: Cookie +< x-nc: MISS hhn 3 +< x-content-type-options: nosniff +< alt-svc: h3=":443"; ma=86400 +< +{ [0 bytes data] + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* Connection #0 to host 192.0.72.3 left intact +HTTP/2 302 +server: nginx +date: Fri, 07 Feb 2025 22:58:34 GMT +content-type: text/html; charset=utf-8 +location: https://developer.wordpress.com +vary: Cookie +x-nc: MISS hhn 3 +x-content-type-options: nosniff +alt-svc: h3=":443"; ma=86400 + diff --git a/Backend/Scripts/443Header1.txt b/Backend/Scripts/443Header1.txt new file mode 100644 index 0000000..450aab7 --- /dev/null +++ b/Backend/Scripts/443Header1.txt @@ -0,0 +1,70 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.251:443... +* Connected to 192.0.66.251 (192.0.66.251) port 443 +* ALPN: curl offers h2,http/1.1 +} [5 bytes data] +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +} [512 bytes data] +* TLSv1.3 (IN), TLS handshake, Server hello (2): +{ [122 bytes data] +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +{ [15 bytes data] +* TLSv1.3 (IN), TLS handshake, Certificate (11): +{ [2033 bytes data] +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +{ [79 bytes data] +* TLSv1.3 (IN), TLS handshake, Finished (20): +{ [52 bytes data] +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +} [1 bytes data] +* TLSv1.3 (OUT), TLS handshake, Finished (20): +} [52 bytes data] +* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=go-vip.co +* start date: Jan 18 19:43:58 2025 GMT +* expire date: Apr 18 19:43:57 2025 GMT +* issuer: C=US; O=Let's Encrypt; CN=E5 +* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway. +* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384 +* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption +} [5 bytes data] +* using HTTP/2 +* [HTTP/2] [1] OPENED stream for https://192.0.66.251/ +* [HTTP/2] [1] [:method: HEAD] +* [HTTP/2] [1] [:scheme: https] +* [HTTP/2] [1] [:authority: 192.0.66.251] +* [HTTP/2] [1] [:path: /] +* [HTTP/2] [1] [user-agent: curl/8.9.1] +* [HTTP/2] [1] [accept: */*] +} [5 bytes data] +> HEAD / HTTP/2 +> Host: 192.0.66.251 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +{ [5 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +< HTTP/2 404 +< server: nginx +< date: Fri, 07 Feb 2025 22:58:27 GMT +< content-type: text/html +< content-length: 146 +< x-rq: hhn2 +< +{ [0 bytes data] + 0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* Connection #0 to host 192.0.66.251 left intact +HTTP/2 404 +server: nginx +date: Fri, 07 Feb 2025 22:58:27 GMT +content-type: text/html +content-length: 146 +x-rq: hhn2 + diff --git a/Backend/Scripts/443Header2.txt b/Backend/Scripts/443Header2.txt new file mode 100644 index 0000000..2e5251c --- /dev/null +++ b/Backend/Scripts/443Header2.txt @@ -0,0 +1,70 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.254:443... +* Connected to 192.0.66.254 (192.0.66.254) port 443 +* ALPN: curl offers h2,http/1.1 +} [5 bytes data] +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +} [512 bytes data] +* TLSv1.3 (IN), TLS handshake, Server hello (2): +{ [122 bytes data] +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +{ [15 bytes data] +* TLSv1.3 (IN), TLS handshake, Certificate (11): +{ [2033 bytes data] +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +{ [78 bytes data] +* TLSv1.3 (IN), TLS handshake, Finished (20): +{ [52 bytes data] +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +} [1 bytes data] +* TLSv1.3 (OUT), TLS handshake, Finished (20): +} [52 bytes data] +* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=go-vip.co +* start date: Jan 18 19:43:58 2025 GMT +* expire date: Apr 18 19:43:57 2025 GMT +* issuer: C=US; O=Let's Encrypt; CN=E5 +* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway. +* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384 +* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption +} [5 bytes data] +* using HTTP/2 +* [HTTP/2] [1] OPENED stream for https://192.0.66.254/ +* [HTTP/2] [1] [:method: HEAD] +* [HTTP/2] [1] [:scheme: https] +* [HTTP/2] [1] [:authority: 192.0.66.254] +* [HTTP/2] [1] [:path: /] +* [HTTP/2] [1] [user-agent: curl/8.9.1] +* [HTTP/2] [1] [accept: */*] +} [5 bytes data] +> HEAD / HTTP/2 +> Host: 192.0.66.254 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +{ [5 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +< HTTP/2 404 +< server: nginx +< date: Fri, 07 Feb 2025 22:58:28 GMT +< content-type: text/html +< content-length: 146 +< x-rq: hhn2 +< +{ [0 bytes data] + 0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* Connection #0 to host 192.0.66.254 left intact +HTTP/2 404 +server: nginx +date: Fri, 07 Feb 2025 22:58:28 GMT +content-type: text/html +content-length: 146 +x-rq: hhn2 + diff --git a/Backend/Scripts/443Header3.txt b/Backend/Scripts/443Header3.txt new file mode 100644 index 0000000..215e22b --- /dev/null +++ b/Backend/Scripts/443Header3.txt @@ -0,0 +1,70 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.253:443... +* Connected to 192.0.66.253 (192.0.66.253) port 443 +* ALPN: curl offers h2,http/1.1 +} [5 bytes data] +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +} [512 bytes data] +* TLSv1.3 (IN), TLS handshake, Server hello (2): +{ [122 bytes data] +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +{ [15 bytes data] +* TLSv1.3 (IN), TLS handshake, Certificate (11): +{ [2033 bytes data] +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +{ [78 bytes data] +* TLSv1.3 (IN), TLS handshake, Finished (20): +{ [52 bytes data] +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +} [1 bytes data] +* TLSv1.3 (OUT), TLS handshake, Finished (20): +} [52 bytes data] +* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey +* ALPN: server accepted h2 +* Server certificate: +* subject: CN=go-vip.co +* start date: Jan 18 19:43:58 2025 GMT +* expire date: Apr 18 19:43:57 2025 GMT +* issuer: C=US; O=Let's Encrypt; CN=E5 +* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway. +* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384 +* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption +} [5 bytes data] +* using HTTP/2 +* [HTTP/2] [1] OPENED stream for https://192.0.66.253/ +* [HTTP/2] [1] [:method: HEAD] +* [HTTP/2] [1] [:scheme: https] +* [HTTP/2] [1] [:authority: 192.0.66.253] +* [HTTP/2] [1] [:path: /] +* [HTTP/2] [1] [user-agent: curl/8.9.1] +* [HTTP/2] [1] [accept: */*] +} [5 bytes data] +> HEAD / HTTP/2 +> Host: 192.0.66.253 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +{ [5 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +{ [249 bytes data] +< HTTP/2 404 +< server: nginx +< date: Fri, 07 Feb 2025 22:58:28 GMT +< content-type: text/html +< content-length: 146 +< x-rq: hhn2 +< +{ [0 bytes data] + 0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* Connection #0 to host 192.0.66.253 left intact +HTTP/2 404 +server: nginx +date: Fri, 07 Feb 2025 22:58:28 GMT +content-type: text/html +content-length: 146 +x-rq: hhn2 + diff --git a/Backend/Scripts/80Header0.txt b/Backend/Scripts/80Header0.txt new file mode 100644 index 0000000..44e3cbc --- /dev/null +++ b/Backend/Scripts/80Header0.txt @@ -0,0 +1,28 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.72.3:80... +* Connected to 192.0.72.3 (192.0.72.3) port 80 +> HEAD / HTTP/1.1 +> Host: 192.0.72.3 +> User-Agent: curl/8.9.1 +> Accept: */* +> +* Request completely sent off +< HTTP/1.1 301 Moved Permanently +< Server: nginx +< Date: Fri, 07 Feb 2025 22:58:33 GMT +< Content-Type: text/html +< Content-Length: 162 +< Connection: keep-alive +< Location: https://192.0.72.3/ +< + 0 162 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +* Connection #0 to host 192.0.72.3 left intact +HTTP/1.1 301 Moved Permanently +Server: nginx +Date: Fri, 07 Feb 2025 22:58:33 GMT +Content-Type: text/html +Content-Length: 162 +Connection: keep-alive +Location: https://192.0.72.3/ + diff --git a/Backend/Scripts/80Header1.txt b/Backend/Scripts/80Header1.txt new file mode 100644 index 0000000..7fa8cfd --- /dev/null +++ b/Backend/Scripts/80Header1.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.64.106:80... + 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 \ No newline at end of file diff --git a/Backend/Scripts/80Header2.txt b/Backend/Scripts/80Header2.txt new file mode 100644 index 0000000..35429d1 --- /dev/null +++ b/Backend/Scripts/80Header2.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.68.86:80... + 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 \ No newline at end of file diff --git a/Backend/Scripts/80Header3.txt b/Backend/Scripts/80Header3.txt new file mode 100644 index 0000000..33a551f --- /dev/null +++ b/Backend/Scripts/80Header3.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.68.78:80... + 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 \ No newline at end of file diff --git a/Models/BackupDB/ScannerResume.db b/Models/BackupDB/ScannerResume.db index a44725c..a57f8e7 100644 Binary files a/Models/BackupDB/ScannerResume.db and b/Models/BackupDB/ScannerResume.db differ diff --git a/Models/CompressedDatabases.db b/Models/CompressedDatabases.db deleted file mode 100644 index 4a54790..0000000 Binary files a/Models/CompressedDatabases.db and /dev/null differ diff --git a/Models/Filtered.db b/Models/Filtered.db deleted file mode 100644 index ccb8ab2..0000000 Binary files a/Models/Filtered.db and /dev/null differ diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 5b8136d..efe27a7 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -73,8 +73,8 @@ public class DbHandler private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + - " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" + - " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);"; + " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" + + " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);"; private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + @@ -86,11 +86,13 @@ public class DbHandler private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; - private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; + private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;"; private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;"; private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;"; + private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;"; + private const string ReIndexDatabasesStatement = "REINDEX;"; private const string VacuumDatabasesStatement = "VACUUM;"; @@ -256,7 +258,7 @@ public class DbHandler continue; } - if (i >= 500_000 && !_compressing) + if (i >= 5_000_000 && !_compressing) { _compressing = true; @@ -457,19 +459,31 @@ public class DbHandler { using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); - - using SqliteCommand command = new(InsertIntoResume, connection); + SqliteCommand command; + + if (resumeObject.Operation == Operations.Insert) + { + command = new(InsertIntoResume, connection); + command.Parameters.AddWithValue("@startRange", resumeObject.StartRange); + command.Parameters.AddWithValue("@endRange", resumeObject.EndRange); + } + else + { + command = new(UpdateResumeStatement, connection); + } command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber); - command.Parameters.AddWithValue("@startRange", resumeObject.StartRange); - command.Parameters.AddWithValue("@endRange", resumeObject.EndRange); command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte); command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte); command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte); command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte); + command.Parameters.AddWithValue("@paused", resumeObject.Paused); + command.Parameters.AddWithValue("@completed", resumeObject.Completed); _ = command.ExecuteNonQuery(); connection.Close(); + + command.Dispose(); } private void InsertCompressedDatabase(int threadNumber, long rows) @@ -739,7 +753,7 @@ public class DbHandler using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); - using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection); + using SqliteCommand command = new(ReadResumeStatement, connection); command.Parameters.AddWithValue("@threadNumber", threadNumber); using SqliteDataReader reader = command.ExecuteReader(); @@ -760,6 +774,8 @@ public class DbHandler resumeObject.SecondByte = reader.GetInt32(4); resumeObject.ThirdByte = reader.GetInt32(5); resumeObject.FourthByte = reader.GetInt32(6); + resumeObject.Paused = reader.GetBoolean(7); + resumeObject.Completed = reader.GetBoolean(8); } return resumeObject; diff --git a/Models/Model/Backend/ScannerResumeObject.cs b/Models/Model/Backend/ScannerResumeObject.cs index 0799b6e..e29c57d 100644 --- a/Models/Model/Backend/ScannerResumeObject.cs +++ b/Models/Model/Backend/ScannerResumeObject.cs @@ -9,4 +9,7 @@ public class ScannerResumeObject public int ThirdByte { get; set; } public int FourthByte { get; set; } public int ThreadNumber { get; set; } + public Operations Operation { get; set; } + public bool Paused { get; set; } + public bool Completed { get; set; } } \ No newline at end of file diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db deleted file mode 100644 index a44725c..0000000 Binary files a/Models/ScannerResume.db and /dev/null differ diff --git a/Models/mydb.db b/Models/mydb.db deleted file mode 100644 index f0567d0..0000000 Binary files a/Models/mydb.db and /dev/null differ diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index e7d85bb..a0f49b6 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -20,4 +20,5 @@ ForceIncluded ForceIncluded ForceIncluded - ForceIncluded \ No newline at end of file + ForceIncluded + ForceIncluded \ No newline at end of file