owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Implement autosave

Browse Source
This commit is contained in:
Rasmus Rasmussen 2025-02-08 13:02:18 +01:00
parent 45b6d8ecab
commit 8d659d4261
20 changed files with 511 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
Backend/Handler/ContentFilter.cs
View File

@ -6,10 +6,24 @@ using Models.Model.Backend;
namespace Backend.Handler;
public class Content
{
public int Port1 { get; set; }
public int Port2 { get; set; }
public Ip Ip { get; set; }
}
public class ContentThread
{
public int ThreadId { get; set; }
public EventWaitHandle EventWaitHandle { get; set; }
}
public class ContentFilter
{
private readonly ConcurrentQueue<Filtered> _queue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Content?> _contentQueue = new();
private readonly DbHandler _dbHandler;
private readonly string _getDomainPort80;
private readonly string _getDomainPort443;
@ -58,6 +72,11 @@ public class ContentFilter
{
if (_stop) break;
if (_contentQueue.Count >= 500)
{
Thread.Sleep(1000);
}
Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
if (unfiltered.Filtered) continue;
@ -79,12 +98,12 @@ public class ContentFilter
continue;
}
Filtered filtered = GetSiteData(ip);
Content content = new();
content.Ip = ip;
content.Port1 = unfiltered.Port1;
content.Port2 = unfiltered.Port2;
filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2;
_queue.Enqueue(filtered);
_contentQueue.Enqueue(content);
}
Thread.Sleep(_timeOut);
@ -93,10 +112,58 @@ public class ContentFilter
((EventWaitHandle) obj).Set();
}
private Filtered GetSiteData(Ip ip)
public WaitHandle[] StartFilterThread(int threads)
{
StartProcess(ip, 80);
StartProcess(ip, 443);
WaitHandle[] waitHandle = new WaitHandle[threads];
for (int i = 0; i < threads; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
ContentThread contentThread = new();
contentThread.ThreadId = i;
contentThread.EventWaitHandle = handle;
waitHandle[i] = handle;
Thread thread = new(FilterThread!);
thread.Start(contentThread);
}
return waitHandle;
}
private void FilterThread(object obj)
{
ContentThread thread = (ContentThread) obj;
while (!_stop)
{
if (_contentQueue.IsEmpty)
{
Thread.Sleep(1000);
}
_contentQueue.TryDequeue(out Content? content);
if (content is null)
{
continue;
}
Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
filtered.Port1 = content.Port1;
filtered.Port2 = content.Port2;
_queue.Enqueue(filtered);
}
thread.EventWaitHandle.Set();
}
private Filtered GetSiteData(Ip ip, int threadId)
{
StartProcess(ip, 80, threadId);
StartProcess(ip, 443, threadId);
string url1 = "";
string url2 = "";
@ -125,7 +192,7 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
while (streamReader.Peek() != -1)
{
@ -192,7 +259,7 @@ public class ContentFilter
return siteData;
}
private void StartProcess(Ip ip, int port)
private void StartProcess(Ip ip, int port, int threadId)
{
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
@ -200,7 +267,7 @@ public class ContentFilter
proc.StartInfo = new()
{
FileName = "/bin/bash",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
UseShellExecute = false,
RedirectStandardOutput = false,
RedirectStandardError = false,

1
Backend/Handler/IpFilterHandler.cs
View File

@ -73,6 +73,7 @@ public class IpFilterHandler
if (i == 10)
{
_stopAutoscaledThreads = false;
Console.WriteLine("Autoscaler started");
while (!_stopAutoscaledThreads)
{

52
Backend/Handler/IpScanner.cs
View File

@ -108,16 +108,26 @@ public class IpScanner
if (resumeNow is not null)
{
if (resumeNow.Completed)
{
return;
}
scanSettings.Start = resumeNow.FirstByte;
scanSettings.End = resumeNow.EndRange;
secondByte = resumeNow.SecondByte;
thirdByte = resumeNow.ThirdByte;
fourthByte = resumeNow.FourthByte;
}
else
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
}
// Empty buffer so we use the lowest abstracted ping.Send() method.
byte[] buf = [];
using Ping ping = new();
int x = 0;
for (int i = scanSettings.Start; i < scanSettings.End; i++)
{
@ -148,6 +158,14 @@ public class IpScanner
for (int l = fourthByte; l < 256; l++)
{
if (x == 75_000)
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
x = 0;
}
x++;
if (_stop)
{
resumeObject.FourthByte = l;
@ -172,12 +190,12 @@ public class IpScanner
if (address is not null)
{
responseCode = ping.Send(address, _timeout, buf, null).Status;
//Thread.Sleep(256);
//Thread.Sleep(4);
}
}
catch (Exception e)
catch
{
Console.WriteLine(e);
//
}
if (responseCode != IPStatus.Success)
@ -212,11 +230,39 @@ public class IpScanner
Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
}
if (_stop)
{
resumeObject.Paused = true;
}
else
{
resumeObject.Completed = true;
}
resumeObject.Operation = Operations.Update;
_resumeQueue.Enqueue(resumeObject);
scanSettings.Handle!.Set();
}
private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
{
ScannerResumeObject resumeObject = new();
resumeObject.ThreadNumber = threadNumber;
resumeObject.StartRange = startRange;
resumeObject.EndRange = endRange;
resumeObject.FirstByte = firstByte;
resumeObject.SecondByte = secondByte;
resumeObject.ThirdByte = thirdByte;
resumeObject.FourthByte = fourthByte;
resumeObject.Paused = paused;
resumeObject.Completed = completed;
resumeObject.Operation = operation;
_resumeQueue.Enqueue(resumeObject);
}
private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
{
return new()

38
Backend/Handler/ThreadHandler.cs
View File

@ -34,35 +34,38 @@ public class ThreadHandler
public void Start()
{
//Thread scanner = new(StartScanner);
//Thread ipFilter = new(StartIpFilter);
Thread scanner = new(StartScanner);
Thread ipFilter = new(StartIpFilter);
Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
Thread filtered = new(StartFilteredDbHandler);
Thread resume = new(StartResumeDbHandler);
//Thread communication = new(StartCommunicationHandler);
//Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread communication = new(StartCommunicationHandler);
Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread contentFilterThread = new(StartContentFilterThread);
//ipFilter.Start();
//scanner.Start();
//ipFilterAutoScaler.Start();
ipFilter.Start();
scanner.Start();
ipFilterAutoScaler.Start();
indexer.Start();
database.Start();
discarded.Start();
filtered.Start();
resume.Start();
//communication.Start();
communication.Start();
contentFilterThread.Start();
//scanner.Join();
//ipFilter.Join();
scanner.Join();
ipFilter.Join();
indexer.Join();
database.Join();
discarded.Join();
filtered.Join();
resume.Join();
//communication.Join();
//ipFilterAutoScaler.Join();
communication.Join();
ipFilterAutoScaler.Join();
contentFilterThread.Join();
}
private void StartScanner()
@ -94,6 +97,13 @@ public class ThreadHandler
_contentFilterStopped = true;
}
private void StartContentFilterThread()
{
WaitHandle[] wait = _contentFilter.StartFilterThread(4);
WaitHandle.WaitAll(wait);
}
private void StartIpFilterAutoScaler()
{
_ipFilterHandler.AutoScaler();
@ -139,7 +149,7 @@ public class ThreadHandler
Console.WriteLine("Discarded DbHandler finished");
}
/*private void StartCommunicationHandler()
private void StartCommunicationHandler()
{
WaitHandle[] wait = _communication.Start();
@ -150,7 +160,7 @@ public class ThreadHandler
_communicationStopped = true;
Stop();
}*/
}
private void Stop()
{

76
Backend/Scripts/443Header0.txt Normal file
View File

@ -0,0 +1,76 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.72.3:443...
* Connected to 192.0.72.3 (192.0.72.3) port 443
* ALPN: curl offers h2,http/1.1
} [5 bytes data]
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
} [512 bytes data]
* TLSv1.3 (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
{ [15 bytes data]
* TLSv1.3 (IN), TLS handshake, Certificate (11):
{ [2063 bytes data]
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
{ [79 bytes data]
* TLSv1.3 (IN), TLS handshake, Finished (20):
{ [52 bytes data]
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
} [1 bytes data]
* TLSv1.3 (OUT), TLS handshake, Finished (20):
} [52 bytes data]
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
* ALPN: server accepted h2
* Server certificate:
* subject: CN=files.wordpress.com
* start date: Dec 16 09:49:37 2024 GMT
* expire date: Mar 16 09:49:36 2025 GMT
* issuer: C=US; O=Let's Encrypt; CN=E6
* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
} [5 bytes data]
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://192.0.72.3/
* [HTTP/2] [1] [:method: HEAD]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: 192.0.72.3]
* [HTTP/2] [1] [:path: /]
* [HTTP/2] [1] [user-agent: curl/8.9.1]
* [HTTP/2] [1] [accept: */*]
} [5 bytes data]
> HEAD / HTTP/2
> Host: 192.0.72.3
> User-Agent: curl/8.9.1
> Accept: */*
>
* Request completely sent off
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
< HTTP/2 302
< server: nginx
< date: Fri, 07 Feb 2025 22:58:34 GMT
< content-type: text/html; charset=utf-8
< location: https://developer.wordpress.com
< vary: Cookie
< x-nc: MISS hhn 3
< x-content-type-options: nosniff
< alt-svc: h3=":443"; ma=86400
<
{ [0 bytes data]
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
* Connection #0 to host 192.0.72.3 left intact
HTTP/2 302
server: nginx
date: Fri, 07 Feb 2025 22:58:34 GMT
content-type: text/html; charset=utf-8
location: https://developer.wordpress.com
vary: Cookie
x-nc: MISS hhn 3
x-content-type-options: nosniff
alt-svc: h3=":443"; ma=86400

70
Backend/Scripts/443Header1.txt Normal file
View File

@ -0,0 +1,70 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.251:443...
* Connected to 192.0.66.251 (192.0.66.251) port 443
* ALPN: curl offers h2,http/1.1
} [5 bytes data]
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
} [512 bytes data]
* TLSv1.3 (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
{ [15 bytes data]
* TLSv1.3 (IN), TLS handshake, Certificate (11):
{ [2033 bytes data]
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
{ [79 bytes data]
* TLSv1.3 (IN), TLS handshake, Finished (20):
{ [52 bytes data]
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
} [1 bytes data]
* TLSv1.3 (OUT), TLS handshake, Finished (20):
} [52 bytes data]
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
* ALPN: server accepted h2
* Server certificate:
* subject: CN=go-vip.co
* start date: Jan 18 19:43:58 2025 GMT
* expire date: Apr 18 19:43:57 2025 GMT
* issuer: C=US; O=Let's Encrypt; CN=E5
* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
} [5 bytes data]
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://192.0.66.251/
* [HTTP/2] [1] [:method: HEAD]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: 192.0.66.251]
* [HTTP/2] [1] [:path: /]
* [HTTP/2] [1] [user-agent: curl/8.9.1]
* [HTTP/2] [1] [accept: */*]
} [5 bytes data]
> HEAD / HTTP/2
> Host: 192.0.66.251
> User-Agent: curl/8.9.1
> Accept: */*
>
* Request completely sent off
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
< HTTP/2 404
< server: nginx
< date: Fri, 07 Feb 2025 22:58:27 GMT
< content-type: text/html
< content-length: 146
< x-rq: hhn2
<
{ [0 bytes data]
0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
* Connection #0 to host 192.0.66.251 left intact
HTTP/2 404
server: nginx
date: Fri, 07 Feb 2025 22:58:27 GMT
content-type: text/html
content-length: 146
x-rq: hhn2

70
Backend/Scripts/443Header2.txt Normal file
View File

@ -0,0 +1,70 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.254:443...
* Connected to 192.0.66.254 (192.0.66.254) port 443
* ALPN: curl offers h2,http/1.1
} [5 bytes data]
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
} [512 bytes data]
* TLSv1.3 (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
{ [15 bytes data]
* TLSv1.3 (IN), TLS handshake, Certificate (11):
{ [2033 bytes data]
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
{ [78 bytes data]
* TLSv1.3 (IN), TLS handshake, Finished (20):
{ [52 bytes data]
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
} [1 bytes data]
* TLSv1.3 (OUT), TLS handshake, Finished (20):
} [52 bytes data]
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
* ALPN: server accepted h2
* Server certificate:
* subject: CN=go-vip.co
* start date: Jan 18 19:43:58 2025 GMT
* expire date: Apr 18 19:43:57 2025 GMT
* issuer: C=US; O=Let's Encrypt; CN=E5
* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
} [5 bytes data]
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://192.0.66.254/
* [HTTP/2] [1] [:method: HEAD]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: 192.0.66.254]
* [HTTP/2] [1] [:path: /]
* [HTTP/2] [1] [user-agent: curl/8.9.1]
* [HTTP/2] [1] [accept: */*]
} [5 bytes data]
> HEAD / HTTP/2
> Host: 192.0.66.254
> User-Agent: curl/8.9.1
> Accept: */*
>
* Request completely sent off
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
< HTTP/2 404
< server: nginx
< date: Fri, 07 Feb 2025 22:58:28 GMT
< content-type: text/html
< content-length: 146
< x-rq: hhn2
<
{ [0 bytes data]
0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
* Connection #0 to host 192.0.66.254 left intact
HTTP/2 404
server: nginx
date: Fri, 07 Feb 2025 22:58:28 GMT
content-type: text/html
content-length: 146
x-rq: hhn2

70
Backend/Scripts/443Header3.txt Normal file
View File

@ -0,0 +1,70 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.66.253:443...
* Connected to 192.0.66.253 (192.0.66.253) port 443
* ALPN: curl offers h2,http/1.1
} [5 bytes data]
* TLSv1.3 (OUT), TLS handshake, Client hello (1):
} [512 bytes data]
* TLSv1.3 (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8):
{ [15 bytes data]
* TLSv1.3 (IN), TLS handshake, Certificate (11):
{ [2033 bytes data]
* TLSv1.3 (IN), TLS handshake, CERT verify (15):
{ [78 bytes data]
* TLSv1.3 (IN), TLS handshake, Finished (20):
{ [52 bytes data]
* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1):
} [1 bytes data]
* TLSv1.3 (OUT), TLS handshake, Finished (20):
} [52 bytes data]
* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 / x25519 / id-ecPublicKey
* ALPN: server accepted h2
* Server certificate:
* subject: CN=go-vip.co
* start date: Jan 18 19:43:58 2025 GMT
* expire date: Apr 18 19:43:57 2025 GMT
* issuer: C=US; O=Let's Encrypt; CN=E5
* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway.
* Certificate level 0: Public key type EC/prime256v1 (256/128 Bits/secBits), signed using ecdsa-with-SHA384
* Certificate level 1: Public key type EC/secp384r1 (384/192 Bits/secBits), signed using sha256WithRSAEncryption
} [5 bytes data]
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://192.0.66.253/
* [HTTP/2] [1] [:method: HEAD]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: 192.0.66.253]
* [HTTP/2] [1] [:path: /]
* [HTTP/2] [1] [user-agent: curl/8.9.1]
* [HTTP/2] [1] [accept: */*]
} [5 bytes data]
> HEAD / HTTP/2
> Host: 192.0.66.253
> User-Agent: curl/8.9.1
> Accept: */*
>
* Request completely sent off
{ [5 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
{ [249 bytes data]
< HTTP/2 404
< server: nginx
< date: Fri, 07 Feb 2025 22:58:28 GMT
< content-type: text/html
< content-length: 146
< x-rq: hhn2
<
{ [0 bytes data]
0 146 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
* Connection #0 to host 192.0.66.253 left intact
HTTP/2 404
server: nginx
date: Fri, 07 Feb 2025 22:58:28 GMT
content-type: text/html
content-length: 146
x-rq: hhn2

28
Backend/Scripts/80Header0.txt Normal file
View File

@ -0,0 +1,28 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 192.0.72.3:80...
* Connected to 192.0.72.3 (192.0.72.3) port 80
> HEAD / HTTP/1.1
> Host: 192.0.72.3
> User-Agent: curl/8.9.1
> Accept: */*
>
* Request completely sent off
< HTTP/1.1 301 Moved Permanently
< Server: nginx
< Date: Fri, 07 Feb 2025 22:58:33 GMT
< Content-Type: text/html
< Content-Length: 162
< Connection: keep-alive
< Location: https://192.0.72.3/
<
0 162 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
* Connection #0 to host 192.0.72.3 left intact
HTTP/1.1 301 Moved Permanently
Server: nginx
Date: Fri, 07 Feb 2025 22:58:33 GMT
Content-Type: text/html
Content-Length: 162
Connection: keep-alive
Location: https://192.0.72.3/

4
Backend/Scripts/80Header1.txt Normal file
View File

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.64.106:80...
0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0

4
Backend/Scripts/80Header2.txt Normal file
View File

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.68.86:80...
0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0

4
Backend/Scripts/80Header3.txt Normal file
View File

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 141.0.68.78:80...
0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0

BIN
Models/BackupDB/ScannerResume.db
View File

Binary file not shown.

BIN
Models/CompressedDatabases.db
View File

Binary file not shown.

BIN
Models/Filtered.db
View File

Binary file not shown.

32
Models/Handler/DbHandler.cs
View File

@ -73,8 +73,8 @@ public class DbHandler
private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
" VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
" INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
" VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@ -86,11 +86,13 @@ public class DbHandler
private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
private const string ReIndexDatabasesStatement = "REINDEX;";
private const string VacuumDatabasesStatement = "VACUUM;";
@ -256,7 +258,7 @@ public class DbHandler
continue;
}
if (i >= 500_000 && !_compressing)
if (i >= 5_000_000 && !_compressing)
{
_compressing = true;
@ -457,19 +459,31 @@ public class DbHandler
{
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
SqliteCommand command;
using SqliteCommand command = new(InsertIntoResume, connection);
if (resumeObject.Operation == Operations.Insert)
{
command = new(InsertIntoResume, connection);
command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
}
else
{
command = new(UpdateResumeStatement, connection);
}
command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
command.Parameters.AddWithValue("@paused", resumeObject.Paused);
command.Parameters.AddWithValue("@completed", resumeObject.Completed);
_ = command.ExecuteNonQuery();
connection.Close();
command.Dispose();
}
private void InsertCompressedDatabase(int threadNumber, long rows)
@ -739,7 +753,7 @@ public class DbHandler
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
using SqliteCommand command = new(ReadResumeStatement, connection);
command.Parameters.AddWithValue("@threadNumber", threadNumber);
using SqliteDataReader reader = command.ExecuteReader();
@ -760,6 +774,8 @@ public class DbHandler
resumeObject.SecondByte = reader.GetInt32(4);
resumeObject.ThirdByte = reader.GetInt32(5);
resumeObject.FourthByte = reader.GetInt32(6);
resumeObject.Paused = reader.GetBoolean(7);
resumeObject.Completed = reader.GetBoolean(8);
}
return resumeObject;

3
Models/Model/Backend/ScannerResumeObject.cs
View File

@ -9,4 +9,7 @@ public class ScannerResumeObject
public int ThirdByte { get; set; }
public int FourthByte { get; set; }
public int ThreadNumber { get; set; }
public Operations Operation { get; set; }
public bool Paused { get; set; }
public bool Completed { get; set; }
}

BIN
Models/ScannerResume.db
View File

Binary file not shown.

BIN
Models/mydb.db
View File

Binary file not shown.

3
RSE.sln.DotSettings.user
View File

@ -20,4 +20,5 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>