Reduced the memory usage, as well as hopefully fixing the memory leak. Also implemented database compression across rows, including Brotli compression and bitshifting.
This commit is contained in:
+49
-132
@@ -1,5 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Models.Helper;
|
||||
using Models.Model.Backend;
|
||||
@@ -75,8 +77,8 @@ public class DbHandler
|
||||
|
||||
private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
|
||||
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
|
||||
" INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
|
||||
" VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
|
||||
" INSERT INTO Discarded (PackedData)" +
|
||||
" VALUES (@packedData)";
|
||||
|
||||
private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
|
||||
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
|
||||
@@ -263,7 +265,7 @@ public class DbHandler
|
||||
Thread f = new (DiscardedDbHandler!);
|
||||
f.Start(discardedDbHandlerSetting);
|
||||
|
||||
Thread.Sleep(5000);
|
||||
Thread.Sleep(150);
|
||||
}
|
||||
|
||||
return waitHandles;
|
||||
@@ -278,7 +280,16 @@ public class DbHandler
|
||||
|
||||
int i = 0;
|
||||
|
||||
while (!_stop)
|
||||
SqliteConnection connection = new(connectionString);
|
||||
connection.Open();
|
||||
|
||||
SqliteCommand command = new(InsertIntoDiscarded, connection);
|
||||
|
||||
StringBuilder stringBuilder = new();
|
||||
|
||||
bool running = true;
|
||||
|
||||
while (!_stop && running)
|
||||
{
|
||||
if (_discardedQueue.IsEmpty || _pause)
|
||||
{
|
||||
@@ -286,31 +297,46 @@ public class DbHandler
|
||||
_paused = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i >= 5_000_000 && !_compressing)
|
||||
|
||||
if (_stop && _discardedQueue.IsEmpty)
|
||||
{
|
||||
_compressing = true;
|
||||
|
||||
i = 0;
|
||||
|
||||
InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
|
||||
|
||||
int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
|
||||
|
||||
CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
|
||||
|
||||
DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
|
||||
|
||||
_compressing = false;
|
||||
running = false;
|
||||
}
|
||||
|
||||
_discardedQueue.TryDequeue(out Discarded queueItem);
|
||||
if (i == 2048)
|
||||
{
|
||||
command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
|
||||
_ = command.ExecuteNonQuery();
|
||||
|
||||
// Re-use the command object.
|
||||
command.Parameters.Clear();
|
||||
|
||||
stringBuilder.Clear();
|
||||
|
||||
i = 0;
|
||||
}
|
||||
|
||||
InsertDiscarded(queueItem, connectionString);
|
||||
if (_discardedQueue.TryDequeue(out Discarded queueItem))
|
||||
{
|
||||
stringBuilder.Append(queueItem.Ip.PackIp());
|
||||
stringBuilder.Append(':');
|
||||
stringBuilder.Append(queueItem.ResponseCode);
|
||||
stringBuilder.Append(',');
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
if (stringBuilder.Length != 0)
|
||||
{
|
||||
command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
|
||||
_ = command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
connection.Close();
|
||||
connection.Dispose();
|
||||
command.Dispose();
|
||||
|
||||
discardedDbHandlerSetting.Handle!.Set();
|
||||
|
||||
Console.WriteLine("Discarded DbHandler stopped.");
|
||||
@@ -359,23 +385,6 @@ public class DbHandler
|
||||
connection.Close();
|
||||
}
|
||||
|
||||
private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
|
||||
{
|
||||
using SqliteConnection connection = new(dbConnectionString);
|
||||
connection.Open();
|
||||
|
||||
using SqliteCommand command = new(InsertIntoDiscarded, connection);
|
||||
|
||||
command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
|
||||
command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
|
||||
command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
|
||||
command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
|
||||
command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
|
||||
_ = command.ExecuteNonQuery();
|
||||
|
||||
connection.Close();
|
||||
}
|
||||
|
||||
private void InsertFiltered(Filtered filtered)
|
||||
{
|
||||
using SqliteConnection connection = new(_filteredConnectionString);
|
||||
@@ -544,6 +553,7 @@ public class DbHandler
|
||||
command.Parameters.AddWithValue("@filtered", 0);
|
||||
|
||||
_ = command.ExecuteNonQuery();
|
||||
|
||||
connection.Close();
|
||||
}
|
||||
|
||||
@@ -726,36 +736,6 @@ public class DbHandler
|
||||
return true;
|
||||
}
|
||||
|
||||
public List<SearchResult?> GetSearchResults()
|
||||
{
|
||||
lock (_readFilteredLock)
|
||||
{
|
||||
using SqliteConnection connection = new(_filteredConnectionString);
|
||||
connection.Open();
|
||||
|
||||
using SqliteCommand command = new(ReadFilteredStatement, connection);
|
||||
using SqliteDataReader reader = command.ExecuteReader();
|
||||
|
||||
if (!reader.HasRows)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
List<SearchResult?> results = [];
|
||||
|
||||
while (reader.Read())
|
||||
{
|
||||
SearchResult result = new();
|
||||
result.Title = reader.GetString(0);
|
||||
result.Url = reader.GetString(1);
|
||||
|
||||
results.Add(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
public ScannerResumeObject? GetResumeObject(int threadNumber)
|
||||
{
|
||||
lock (_readAndDeleteResumeLock)
|
||||
@@ -791,76 +771,13 @@ public class DbHandler
|
||||
return resumeObject;
|
||||
}
|
||||
}
|
||||
|
||||
public void ReIndex()
|
||||
{
|
||||
_pause = true;
|
||||
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
|
||||
|
||||
if (!_paused)
|
||||
{
|
||||
Thread.Sleep(5000); // Just for safety.
|
||||
}
|
||||
|
||||
SqliteConnection connection = new(_filteredConnectionString);
|
||||
connection.Open();
|
||||
|
||||
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
|
||||
_ = command.ExecuteNonQuery();
|
||||
connection.Close();
|
||||
|
||||
connection = new(_unfilteredConnectionString);
|
||||
connection.Open();
|
||||
|
||||
command = new(ReIndexDatabasesStatement, connection);
|
||||
_ = command.ExecuteNonQuery();
|
||||
connection.Close();
|
||||
|
||||
connection.Dispose();
|
||||
command.Dispose();
|
||||
|
||||
_pause = false;
|
||||
_paused = false;
|
||||
}
|
||||
|
||||
public void Vacuum()
|
||||
{
|
||||
_pause = true;
|
||||
|
||||
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
|
||||
|
||||
if (!_paused)
|
||||
{
|
||||
Thread.Sleep(5000); // Just for safety.
|
||||
}
|
||||
|
||||
SqliteConnection connection = new(_filteredConnectionString);
|
||||
connection.Open();
|
||||
|
||||
SqliteCommand command = new(VacuumDatabasesStatement, connection);
|
||||
_ = command.ExecuteNonQuery();
|
||||
connection.Close();
|
||||
|
||||
connection = new(_unfilteredConnectionString);
|
||||
connection.Open();
|
||||
|
||||
command = new(VacuumDatabasesStatement, connection);
|
||||
_ = command.ExecuteNonQuery();
|
||||
connection.Close();
|
||||
|
||||
connection.Dispose();
|
||||
command.Dispose();
|
||||
|
||||
_pause = false;
|
||||
_paused = false;
|
||||
}
|
||||
|
||||
private (string, string) CreateDiscardedDb(int threadNumber)
|
||||
{
|
||||
string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
|
||||
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
|
||||
|
||||
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
|
||||
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
|
||||
|
||||
_discardedConnectionStrings.Add(databaseName);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user