owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Misc changes such as performance tuning, added support for more than 64 threads for the scanner.

Browse Source
This commit is contained in:
owner
2024-12-04 13:20:45 +01:00
parent a1f5b08ac3
commit 5fe35192c6
13 changed files with 281 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Models/Handler/DbHandler.cs
+107 -11
View File
@@ -1,4 +1,5 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Microsoft.Data.Sqlite;
using Models.Model.Backend;
using Models.Model.External;
@@ -24,18 +25,45 @@ public class DbHandler
" VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
" INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
" Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
" RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
" CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
" PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
" ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2," +
" @title1, @title2, @description1, @description2, @url1, @url2, @serverType1," +
" @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2," +
" @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion," +
" @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2," +
" @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
" ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
" @title1, @title2, @description1, @description2, @url1, @url2, " +
" (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
" (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
" @robotsTXT1, @robotsTXT2," +
" (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion1)," +
" (SELECT HttpId FROM HttpVersion WHERE Version = @httpVersion2)," +
" (SELECT CertificateIssuerId FROM CertificateIssuerCountry WHERE Country = @certificateIssuerCountry)," +
" (SELECT CertificateOrganizationId FROM CertificateOrganizationName WHERE Name = @certificateOrganizationName), " +
" @ipV6, " +
" (SELECT TlsId FROM TlsVersion WHERE Version = @tlsVersion)," +
" (SELECT CipherId FROM CipherSuite WHERE Suite = @cipherSuite)," +
" (SELECT KeyExchangeId FROM KeyExchangeAlgorithm WHERE Algorithm = @keyExchangeAlgorithm)," +
" (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType1)," +
" (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType2)," +
" (SELECT PublicKeyId FROM PublicKeyType WHERE Type = @publicKeyType3)," +
" (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding1)," +
" (SELECT AcceptId FROM AcceptEncoding WHERE Encoding = @acceptEncoding2)," +
" (SELECT ALPNId FROM ALPN WHERE ALPNValue = @aLPN)," +
" (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection1)," +
" (SELECT ConnectionId FROM Connection WHERE ConnectionValue = @connection2))";
private const string InsertIntoFilteredServerType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ServerType (Type) VALUES (@type)";
private const string InsertIntoFilteredHttpVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO HttpVersion (Version) VALUES (@version)";
private const string InsertIntoFilteredCertificateIssuerCountry = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateIssuerCountry (Country) VALUES (@country)";
private const string InsertIntoFilteredCertificateOrganizationName = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CertificateOrganizationName (Name) VALUES (@name)";
private const string InsertIntoFilteredTlsVersion = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO TlsVersion (Version) VALUES (@version)";
private const string InsertIntoFilteredCipherSuite = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO CipherSuite (Suite) VALUES (@suite)";
private const string InsertIntoFilteredKeyExchangeAlgorithm = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO KeyExchangeAlgorithm (Algorithm) VALUES (@algorithm)";
private const string InsertIntoFilteredPublicKeyType = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO PublicKeyType (Type) VALUES (@type)";
private const string InsertIntoFilteredAcceptEncoding = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO AcceptEncoding (Encoding) VALUES (@encoding)";
private const string InsertIntoFilteredALPN = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO ALPN (ALPNValue) VALUES (@alpnValue)";
private const string InsertIntoFilteredConnection = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; INSERT OR IGNORE INTO Connection (ConnectionValue) VALUES (@connectionValue)";
private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -270,8 +298,75 @@ public class DbHandler
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoFiltered, connection);
SqliteCommand command = new(InsertIntoFilteredServerType, connection);
command.Parameters.AddWithValue("@type", filtered.ServerType1);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredServerType, connection);
command.Parameters.AddWithValue("@type", filtered.ServerType2);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredHttpVersion, connection);
command.Parameters.AddWithValue("@version", filtered.HttpVersion1);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredHttpVersion, connection);
command.Parameters.AddWithValue("@version", filtered.HttpVersion2);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredCertificateIssuerCountry, connection);
command.Parameters.AddWithValue("@country", filtered.CertificateIssuerCountry);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredCertificateOrganizationName, connection);
command.Parameters.AddWithValue("@name", filtered.CertificateOrganizationName);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredTlsVersion, connection);
command.Parameters.AddWithValue("@version", filtered.TlsVersion);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredCipherSuite, connection);
command.Parameters.AddWithValue("@suite", filtered.CipherSuite);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredKeyExchangeAlgorithm, connection);
command.Parameters.AddWithValue("@algorithm", filtered.KeyExchangeAlgorithm);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredPublicKeyType, connection);
command.Parameters.AddWithValue("@type", filtered.PublicKeyType1);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredPublicKeyType, connection);
command.Parameters.AddWithValue("@type", filtered.PublicKeyType2);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredPublicKeyType, connection);
command.Parameters.AddWithValue("@type", filtered.PublicKeyType3);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredAcceptEncoding, connection);
command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding1);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredAcceptEncoding, connection);
command.Parameters.AddWithValue("@encoding", filtered.AcceptEncoding2);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredALPN, connection);
command.Parameters.AddWithValue("@alpnValue", filtered.ALPN);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredConnection, connection);
command.Parameters.AddWithValue("@connectionValue", filtered.Connection1);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFilteredConnection, connection);
command.Parameters.AddWithValue("@connectionValue", filtered.Connection2);
_ = command.ExecuteNonQuery();
command = new(InsertIntoFiltered, connection);
command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
@@ -304,8 +399,9 @@ public class DbHandler
command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
command.Parameters.AddWithValue("@connection1", filtered.Connection1);
command.Parameters.AddWithValue("@connection2", filtered.Connection2);
_ = command.ExecuteNonQuery();
command.Dispose();
connection.Close();
}
@@ -640,8 +736,8 @@ public class DbHandler
{
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 TEXT NOT NULL, Ip2 TEXT NOT NULL, Ip3 TEXT NOT NULL, Ip4 TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
_discardedConnectionStrings.Add(databaseName);
using SqliteConnection connection = new(databaseName);