owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request 'CompressDatabase' (#43) from CompressDatabase into main

Browse Source 
Reviewed-on: #43
This commit is contained in:
Rasmus Rasmussen 2024-12-30 17:47:33 +00:00
commit 8256d6a681
7 changed files with 170 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Backend/Handler/Communication.cs
View File

@ -87,6 +87,7 @@ public class Communication
}
else
{
// This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
status.PercentageOfIpv4Scanned = 0.0000000001f;
}
@ -96,7 +97,7 @@ public class Communication
status.MyDbSize = databaseSizes.MyDbSize;
status.FilteredDbSize = databaseSizes.FilteredDbSize;
status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
rep.SendFrame(serializedResult);

6
Backend/Handler/IpScanner.cs
View File

@ -48,10 +48,11 @@ public class IpScanner
{
threadsAmount = 256 / threads;
}
WaitHandle[] waitHandle1;
WaitHandle[] waitHandle2;
// This is jank. But it was to get it production ready.
if (threads <= 64)
{
waitHandle1 = new WaitHandle[threads];
@ -77,7 +78,7 @@ public class IpScanner
ThreadNumber = i,
Handle = handle
};
if (i < 64)
{
waitHandle1[counter] = handle;
@ -131,6 +132,7 @@ public class IpScanner
fourthByte = resumeNow.FourthByte;
}
// Empty buffer so we use the lowest abstracted ping.Send() method.
byte[] buf = [];
using Ping ping = new();

BIN
Models/BackupDB/Discarded.db → Models/BackupDB/CompressedDatabases.db
View File

Binary file not shown.

156
Models/Handler/DbHandler.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Microsoft.Data.Sqlite;
using Models.Helper;
using Models.Model.Backend;
using Models.Model.External;
@ -14,9 +15,9 @@ public class DbHandler
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly string _unfilteredConnectionString;
private readonly string _discardedConnectionString;
private readonly string _filteredConnectionString;
private readonly string _resumeConnectionString;
private readonly string _compressedConnectionString;
private readonly List<string> _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
@ -74,6 +75,10 @@ public class DbHandler
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
" VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO CompressedDatabases (DbNumber, Rows) VALUES (@dbNumber, @rows)";
private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Filtered == 0;";
@ -82,6 +87,7 @@ public class DbHandler
private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
@ -95,6 +101,7 @@ public class DbHandler
private bool _stop;
private bool _pause;
private bool _paused;
private bool _compressing;
private int _contentWaitTime;
private int _discardedWaitTime;
@ -117,9 +124,9 @@ public class DbHandler
_basePath = basePath;
_unfilteredConnectionString = $"Data Source={basePath}/Models/mydb.db";
_discardedConnectionString = $"Data Source={basePath}/Models/Discarded.db";
_filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
_resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
_compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
}
public void SetContentWaitTime(int waitTime)
@ -213,7 +220,7 @@ public class DbHandler
for (int i = 0; i < threads; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
DiscardedDbHandlerSetting discardedDbHandlerSetting = new()
{
Handle = handle,
@ -225,7 +232,7 @@ public class DbHandler
Thread f = new (DiscardedDbHandler!);
f.Start(discardedDbHandlerSetting);
Thread.Sleep(1000);
Thread.Sleep(5000);
}
return waitHandles;
@ -236,7 +243,9 @@ public class DbHandler
DiscardedDbHandlerSetting discardedDbHandlerSetting = (DiscardedDbHandlerSetting)obj;
Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
string connectionString = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
(string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
int i = 0;
while (!_stop)
{
@ -247,9 +256,28 @@ public class DbHandler
continue;
}
if (i >= 50_000_000 && !_compressing)
{
_compressing = true;
i = 0;
InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
_compressing = false;
}
_discardedQueue.TryDequeue(out Discarded queueItem);
InsertDiscarded(queueItem, connectionString);
i++;
}
discardedDbHandlerSetting.Handle!.Set();
@ -257,6 +285,30 @@ public class DbHandler
Console.WriteLine("Discarded DbHandler stopped.");
}
private void DropAndCreateDiscarded(int threadNumber)
{
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
const string dropStatement = "DROP TABLE Discarded;";
const string vacuum = "VACUUM;";
using SqliteConnection connection = new(databaseName);
connection.Open();
SqliteCommand command = new(dropStatement, connection);
command.ExecuteNonQuery();
command = new(vacuum, connection);
command.ExecuteNonQuery();
command = new(createStatement, connection);
command.ExecuteNonQuery();
command.Dispose();
connection.Close();
}
private void InsertUnfiltered(Unfiltered unfiltered)
{
using SqliteConnection connection = new(_unfilteredConnectionString);
@ -423,6 +475,20 @@ public class DbHandler
_ = command.ExecuteNonQuery();
connection.Close();
}
private void InsertCompressedDatabase(int threadNumber, long rows)
{
using SqliteConnection connection = new(_compressedConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
command.Parameters.AddWithValue("@dbNumber", threadNumber);
command.Parameters.AddWithValue("@rows", rows);
_ = command.ExecuteNonQuery();
connection.Close();
}
private void UpdateUnfiltered(Unfiltered unfiltered)
{
@ -517,26 +583,75 @@ public class DbHandler
public long GetDiscardedIndexes()
{
long rowId = 0;
SqliteConnection connection;
SqliteCommand command;
SqliteDataReader reader;
for (int i = 0; i < _discardedConnectionStrings.Count; i++)
{
using SqliteConnection connection = new(_discardedConnectionStrings[i]);
connection = new(_discardedConnectionStrings[i]);
connection.Open();
using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
using SqliteDataReader reader = command.ExecuteReader();
command = new(ReadDiscardedSeqIdsStatement, connection);
reader = command.ExecuteReader();
if (!reader.HasRows)
{
return 0;
return rowId;
}
while (reader.Read())
{
rowId += reader.GetInt64(0);
}
connection.Close();
}
connection = new(_compressedConnectionString);
connection.Open();
command = new(ReadCompressedDbRowsStatement, connection);
reader = command.ExecuteReader();
if (!reader.HasRows)
{
return rowId;
}
while (reader.Read())
{
rowId += reader.GetInt64(0);
}
connection.Close();
connection.Dispose();
command.Dispose();
reader.Dispose();
return rowId;
}
private static long GetDiscardedIndexesForSpecificDb(string connectionString)
{
using SqliteConnection connection = new(connectionString);
connection.Open();
using SqliteCommand command = new(ReadDiscardedSeqIdsStatement, connection);
using SqliteDataReader reader = command.ExecuteReader();
long rowId = 0;
if (!reader.HasRows)
{
return rowId;
}
while (reader.Read())
{
rowId += reader.GetInt64(0);
}
return rowId;
}
@ -665,20 +780,13 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(_discardedConnectionString);
SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_filteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_unfilteredConnectionString);
connection.Open();
@ -704,20 +812,13 @@ public class DbHandler
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(_discardedConnectionString);
SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
SqliteCommand command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_filteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_unfilteredConnectionString);
connection.Open();
@ -732,8 +833,9 @@ public class DbHandler
_paused = false;
}
private string CreateDiscardedDb(int threadNumber)
private (string, string) CreateDiscardedDb(int threadNumber)
{
string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
@ -746,7 +848,7 @@ public class DbHandler
using SqliteCommand command = new(createStatement, connection);
command.ExecuteNonQuery();
return databaseName;
return (absolutePath, databaseName);
}
public void Stop()

22
Models/Helper/CompressionHelper.cs Normal file
View File

@ -0,0 +1,22 @@
using System.IO.Compression;
namespace Models.Helper;
public static class CompressionHelper
{
public static void CompressFile(string sourceFile, string targetFile)
{
using FileStream originalFileStream = new(sourceFile, FileMode.Open);
using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
originalFileStream.CopyTo(compressor);
}
public static void DecompressFile(string sourceFile, string targetFile)
{
using FileStream compressedFileStream = new(sourceFile, FileMode.Open);
using FileStream decompressedFileStream = File.Create($"{targetFile}.gz");
using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
decompressor.CopyTo(decompressedFileStream);
}
}

9
Models/Helper/GetDatabasesHelper.cs Normal file
View File

@ -0,0 +1,9 @@
namespace Models.Helper;
public static class GetDatabasesHelper
{
public static int GetTotalCompressedDatabases(string path)
{
return Directory.GetFiles(path, "*.gz").Length;
}
}

4
RSE.sln.DotSettings.user
View File

@ -4,8 +4,12 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>