533 lines
20 KiB
C#
533 lines
20 KiB
C#
using System.Collections.Concurrent;
|
|
using Microsoft.Data.Sqlite;
|
|
using Models.Model.Backend;
|
|
using Models.Model.External;
|
|
|
|
namespace Models.Handler;
|
|
|
|
public class DbHandler
|
|
{
|
|
private readonly ConcurrentQueue<QueueItem> _contentQueue;
|
|
private readonly ConcurrentQueue<Discarded> _discardedQueue;
|
|
|
|
private readonly string _unfilteredConnectionString;
|
|
private readonly string _discardedConnectionString;
|
|
private readonly string _filteredConnectionString;
|
|
private readonly string _resumeConnectionString;
|
|
private readonly List<string> _discardedConnectionStrings = [];
|
|
|
|
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)";
|
|
private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
|
|
private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)";
|
|
private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
|
|
|
|
private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
|
|
private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
|
|
private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;";
|
|
private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
|
|
private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
|
|
private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
|
|
|
|
private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
|
|
|
|
private const string ReIndexDatabasesStatement = "REINDEX;";
|
|
|
|
private const string VacuumDatabasesStatement = "VACUUM;";
|
|
|
|
private readonly object _readFilteredLock = new();
|
|
private readonly object _readAndDeleteResumeLock = new();
|
|
|
|
private bool _stop;
|
|
private bool _pause;
|
|
private bool _paused;
|
|
|
|
private int _contentWaitTime;
|
|
private int _discardedWaitTime;
|
|
|
|
private readonly string _basePath;
|
|
|
|
public DbHandler(ConcurrentQueue<QueueItem> contentQueue, ConcurrentQueue<Discarded> discardedQueue, string basePath)
|
|
{
|
|
_contentQueue = contentQueue;
|
|
_discardedQueue = discardedQueue;
|
|
|
|
SetContentWaitTime(10);
|
|
SetDiscardedWaitTime(10);
|
|
|
|
_basePath = basePath;
|
|
|
|
_unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
|
|
_discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
|
|
_filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
|
|
_resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
|
|
}
|
|
|
|
public void SetContentWaitTime(int waitTime)
|
|
{
|
|
_contentWaitTime = waitTime;
|
|
}
|
|
|
|
public void SetDiscardedWaitTime(int waitTime)
|
|
{
|
|
_discardedWaitTime = waitTime;
|
|
}
|
|
|
|
public void StartContent()
|
|
{
|
|
Console.WriteLine("Content DbHandler started");
|
|
|
|
while (!_stop)
|
|
{
|
|
if (_contentQueue.IsEmpty || _pause)
|
|
{
|
|
Thread.Sleep(_contentWaitTime);
|
|
_paused = true;
|
|
continue;
|
|
}
|
|
|
|
_contentQueue.TryDequeue(out QueueItem? queueItem);
|
|
|
|
if (queueItem is null) { continue; }
|
|
|
|
switch (queueItem.Operations)
|
|
{
|
|
case Operations.Insert when queueItem.Unfiltered is not null:
|
|
InsertUnfiltered(queueItem.Unfiltered);
|
|
break;
|
|
case Operations.Insert when queueItem.Filtered is not null:
|
|
InsertFiltered(queueItem.Filtered);
|
|
break;
|
|
case Operations.Insert when queueItem.ResumeObject is not null:
|
|
InsertResumeObject(queueItem.ResumeObject);
|
|
break;
|
|
case Operations.Update when queueItem.Unfiltered is not null:
|
|
UpdateUnfiltered(queueItem.Unfiltered);
|
|
break;
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("Content DbHandler stopped.");
|
|
}
|
|
|
|
public WaitHandle[] Start(int threads)
|
|
{
|
|
WaitHandle[] waitHandles = new WaitHandle[threads];
|
|
|
|
for (int i = 0; i < threads; i++)
|
|
{
|
|
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
|
|
|
|
DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
|
|
{
|
|
Handle = handle,
|
|
ThreadId = i
|
|
};
|
|
|
|
waitHandles[i] = handle;
|
|
|
|
Thread f = new (RunDiscarded!);
|
|
f.Start(discardedDbHandlerSetting);
|
|
|
|
Thread.Sleep(1000);
|
|
}
|
|
|
|
return waitHandles;
|
|
}
|
|
|
|
private void RunDiscarded(object obj)
|
|
{
|
|
DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
|
|
Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
|
|
|
|
string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
|
|
|
|
while (!_stop)
|
|
{
|
|
if (_discardedQueue.IsEmpty || _pause)
|
|
{
|
|
Thread.Sleep(_discardedWaitTime);
|
|
_paused = true;
|
|
continue;
|
|
}
|
|
|
|
_discardedQueue.TryDequeue(out Discarded? queueItem);
|
|
|
|
if (queueItem is null) { continue; }
|
|
|
|
InsertDiscarded(queueItem, connectionString);
|
|
}
|
|
|
|
discardedDbHandlerSetting.Handle!.Set();
|
|
|
|
Console.WriteLine("Content DbHandler stopped.");
|
|
}
|
|
|
|
private void InsertUnfiltered(Unfiltered unfiltered)
|
|
{
|
|
using SqliteConnection connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(InsertStatement, connection);
|
|
|
|
command.Parameters.AddWithValue("@ip", unfiltered.Ip);
|
|
command.Parameters.AddWithValue("@responseCode", unfiltered.ResponseCode);
|
|
command.Parameters.AddWithValue("@port1", unfiltered.Port1);
|
|
command.Parameters.AddWithValue("@port2", unfiltered.Port2);
|
|
command.Parameters.AddWithValue("@filtered", unfiltered.Filtered);
|
|
_ = command.ExecuteNonQuery();
|
|
|
|
connection.Close();
|
|
}
|
|
|
|
private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
|
|
{
|
|
using SqliteConnection connection = new(dbConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(InsertIntoDiscarded, connection);
|
|
|
|
command.Parameters.AddWithValue("@ip", discarded.Ip);
|
|
command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
|
|
_ = command.ExecuteNonQuery();
|
|
|
|
connection.Close();
|
|
}
|
|
|
|
private void InsertFiltered(Filtered filtered)
|
|
{
|
|
using SqliteConnection connection = new(_filteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(InsertIntoFiltered, connection);
|
|
|
|
command.Parameters.AddWithValue("@ip", filtered.Ip);
|
|
command.Parameters.AddWithValue("@port1", filtered.Port1);
|
|
command.Parameters.AddWithValue("@port2", filtered.Port2);
|
|
command.Parameters.AddWithValue("@url1", filtered.Url1);
|
|
command.Parameters.AddWithValue("@url2", filtered.Url2);
|
|
command.Parameters.AddWithValue("@title1", filtered.Title1);
|
|
command.Parameters.AddWithValue("@title2", filtered.Title2);
|
|
command.Parameters.AddWithValue("@description1", filtered.Description1);
|
|
command.Parameters.AddWithValue("@description2", filtered.Description2);
|
|
command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
|
|
command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
|
|
command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
|
|
command.Parameters.AddWithValue("@robotsTXT2", filtered.RobotsTXT2);
|
|
command.Parameters.AddWithValue("@httpVersion1", filtered.HttpVersion1);
|
|
command.Parameters.AddWithValue("@httpVersion2", filtered.HttpVersion2);
|
|
command.Parameters.AddWithValue("@certificateIssuerCountry", filtered.CertificateIssuerCountry);
|
|
command.Parameters.AddWithValue("@certificateOrganizationName", filtered.CertificateOrganizationName);
|
|
command.Parameters.AddWithValue("@ipV6", filtered.IpV6);
|
|
command.Parameters.AddWithValue("@tlsVersion", filtered.TlsVersion);
|
|
command.Parameters.AddWithValue("@cipherSuite", filtered.CipherSuite);
|
|
command.Parameters.AddWithValue("@keyExchangeAlgorithm", filtered.KeyExchangeAlgorithm);
|
|
command.Parameters.AddWithValue("@publicKeyType1", filtered.PublicKeyType1);
|
|
command.Parameters.AddWithValue("@publicKeyType2", filtered.PublicKeyType2);
|
|
command.Parameters.AddWithValue("@publicKeyType3", filtered.PublicKeyType3);
|
|
command.Parameters.AddWithValue("@acceptEncoding1", filtered.AcceptEncoding1);
|
|
command.Parameters.AddWithValue("@acceptEncoding2", filtered.AcceptEncoding2);
|
|
command.Parameters.AddWithValue("@aLPN", filtered.ALPN);
|
|
command.Parameters.AddWithValue("@connection1", filtered.Connection1);
|
|
command.Parameters.AddWithValue("@connection2", filtered.Connection2);
|
|
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
}
|
|
|
|
private void InsertResumeObject(ScannerResumeObject resumeObject)
|
|
{
|
|
using SqliteConnection connection = new(_resumeConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(InsertIntoResume, connection);
|
|
|
|
command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
|
|
command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
|
|
command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
|
|
command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
|
|
command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
|
|
command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
|
|
command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
|
|
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
}
|
|
|
|
private void UpdateUnfiltered(Unfiltered unfiltered)
|
|
{
|
|
using SqliteConnection connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(UpdateUnfilteredStatement, connection);
|
|
|
|
command.Parameters.AddWithValue("@id", unfiltered.Id);
|
|
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
}
|
|
|
|
public Unfiltered? ReadUnfilteredWithId(long id)
|
|
{
|
|
using SqliteConnection connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadUnfilteredStatement, connection);
|
|
command.Parameters.AddWithValue("@id", id);
|
|
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
if (!reader.HasRows) return null;
|
|
|
|
Unfiltered unfiltered = new();
|
|
|
|
while (reader.Read())
|
|
{
|
|
unfiltered.Id = reader.GetInt32(0);
|
|
unfiltered.Ip = reader.GetString(1);
|
|
unfiltered.Port1 = reader.GetInt32(3);
|
|
unfiltered.Port2 = reader.GetInt32(4);
|
|
unfiltered.Filtered = reader.GetInt32(5);
|
|
}
|
|
|
|
return unfiltered;
|
|
}
|
|
|
|
public long GetUnfilteredIndexes()
|
|
{
|
|
long rowId = 0;
|
|
|
|
using SqliteConnection connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadUnfilteredIdsStatement, connection);
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
|
|
if (!reader.HasRows)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
while (reader.Read())
|
|
{
|
|
rowId = reader.GetInt64(0);
|
|
}
|
|
|
|
return rowId;
|
|
}
|
|
|
|
public long GetFilteredIndexes()
|
|
{
|
|
long rowId = 0;
|
|
|
|
using SqliteConnection connection = new(_filteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
|
|
if (!reader.HasRows)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
while (reader.Read())
|
|
{
|
|
rowId = reader.GetInt64(0);
|
|
}
|
|
|
|
return rowId;
|
|
}
|
|
|
|
public long GetDiscardedIndexes()
|
|
{
|
|
long rowId = 0;
|
|
|
|
for (int i = 0; i < _discardedConnectionStrings.Count; i++)
|
|
{
|
|
using SqliteConnection connection = new(_discardedConnectionStrings[i]);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
|
|
if (!reader.HasRows)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
while (reader.Read())
|
|
{
|
|
rowId += reader.GetInt64(0);
|
|
}
|
|
}
|
|
|
|
return rowId;
|
|
}
|
|
|
|
public bool GetFilteredIp(string ip)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
public List<SearchResult?> GetSearchResults()
|
|
{
|
|
lock (_readFilteredLock)
|
|
{
|
|
using SqliteConnection connection = new(_filteredConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadFilteredStatement, connection);
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
|
|
if (!reader.HasRows)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
List<SearchResult?> results = [];
|
|
|
|
while (reader.Read())
|
|
{
|
|
SearchResult result = new();
|
|
result.Title = reader.GetString(0);
|
|
result.Url = reader.GetString(1);
|
|
|
|
results.Add(result);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
}
|
|
|
|
public ScannerResumeObject? GetResumeObject(int threadNumber)
|
|
{
|
|
lock (_readAndDeleteResumeLock)
|
|
{
|
|
using SqliteConnection connection = new(_resumeConnectionString);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
|
|
command.Parameters.AddWithValue("@threadNumber", threadNumber);
|
|
|
|
using SqliteDataReader reader = command.ExecuteReader();
|
|
|
|
if (!reader.HasRows)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
ScannerResumeObject resumeObject = new();
|
|
|
|
while (reader.Read())
|
|
{
|
|
resumeObject.ThreadNumber = reader.GetInt32(0);
|
|
resumeObject.StartRange = reader.GetInt32(1);
|
|
resumeObject.EndRange = reader.GetInt32(2);
|
|
resumeObject.FirstByte = reader.GetInt32(3);
|
|
resumeObject.SecondByte = reader.GetInt32(4);
|
|
resumeObject.ThirdByte = reader.GetInt32(5);
|
|
resumeObject.FourthByte = reader.GetInt32(6);
|
|
}
|
|
|
|
return resumeObject;
|
|
}
|
|
}
|
|
|
|
public void ReIndex()
|
|
{
|
|
_pause = true;
|
|
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
|
|
|
|
if (!_paused)
|
|
{
|
|
Thread.Sleep(5000); // Just for safety.
|
|
}
|
|
|
|
SqliteConnection connection = new(_discardedConnectionString);
|
|
connection.Open();
|
|
|
|
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection = new(_filteredConnectionString);
|
|
connection.Open();
|
|
|
|
command = new(ReIndexDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
command = new(ReIndexDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection.Dispose();
|
|
command.Dispose();
|
|
|
|
_pause = false;
|
|
_paused = false;
|
|
}
|
|
|
|
public void Vacuum()
|
|
{
|
|
_pause = true;
|
|
|
|
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
|
|
|
|
if (!_paused)
|
|
{
|
|
Thread.Sleep(5000); // Just for safety.
|
|
}
|
|
|
|
SqliteConnection connection = new(_discardedConnectionString);
|
|
connection.Open();
|
|
|
|
SqliteCommand command = new(VacuumDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection = new(_filteredConnectionString);
|
|
connection.Open();
|
|
|
|
command = new(VacuumDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection = new(_unfilteredConnectionString);
|
|
connection.Open();
|
|
|
|
command = new(VacuumDatabasesStatement, connection);
|
|
_ = command.ExecuteNonQuery();
|
|
connection.Close();
|
|
|
|
connection.Dispose();
|
|
command.Dispose();
|
|
|
|
_pause = false;
|
|
_paused = false;
|
|
}
|
|
|
|
private string CreateDiscardedDb(int threadNumber)
|
|
{
|
|
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
|
|
|
|
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
|
|
|
|
_discardedConnectionStrings.Add(databaseName);
|
|
|
|
using SqliteConnection connection = new(databaseName);
|
|
connection.Open();
|
|
|
|
using SqliteCommand command = new(createStatement, connection);
|
|
command.ExecuteNonQuery();
|
|
|
|
return databaseName;
|
|
}
|
|
|
|
public void Stop()
|
|
{
|
|
_stop = true;
|
|
}
|
|
} |