owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Fixed a new memory leak

Browse Source
This commit is contained in:
owner
2025-04-14 13:56:57 +02:00
parent c7fcc3297f
commit 291c65ac02
7 changed files with 94 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Models/Handler/DbHandler.cs
+53 -52
View File
@@ -5,7 +5,6 @@ using System.Text;
using Microsoft.Data.Sqlite;
using Models.Helper;
using Models.Model.Backend;
using Models.Model.External;
namespace Models.Handler;
@@ -15,14 +14,13 @@ public class DbHandler
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private readonly ConcurrentQueue<Ip> _preFilteredQueue;
private readonly string _unfilteredConnectionString;
private readonly string _preFilteredConnectionString;
private readonly string _filteredConnectionString;
private readonly string _resumeConnectionString;
private readonly string _compressedConnectionString;
private readonly List<string> _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@@ -98,7 +96,7 @@ public class DbHandler
private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Id == @id ORDER BY Id ASC LIMIT 1;";
private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
@@ -109,7 +107,6 @@ public class DbHandler
private const string VacuumDatabasesStatement = "VACUUM;";
private readonly object _readFilteredLock = new();
private readonly object _readAndDeleteResumeLock = new();
private bool _stop;
@@ -126,7 +123,7 @@ public class DbHandler
ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue,
ConcurrentQueue<FilterQueueItem> preFilteredQueue,
ConcurrentQueue<Ip> preFilteredQueue,
string basePath)
{
_filteredQueue = filteredQueue;
@@ -219,7 +216,7 @@ public class DbHandler
continue;
}
_preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
_preFilteredQueue.TryDequeue(out Ip queueItem);
InsertPrefiltered(queueItem);
}
@@ -524,31 +521,17 @@ public class DbHandler
command.Dispose();
}
private void InsertCompressedDatabase(int threadNumber, long rows)
{
using SqliteConnection connection = new(_compressedConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoCompressedDbConnectionString, connection);
command.Parameters.AddWithValue("@dbNumber", threadNumber);
command.Parameters.AddWithValue("@rows", rows);
_ = command.ExecuteNonQuery();
connection.Close();
}
private void InsertPrefiltered(FilterQueueItem filterQueueItem)
private void InsertPrefiltered(Ip filterQueueItem)
{
using SqliteConnection connection = new(_preFilteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertPreFilteredStatement, connection);
command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip1);
command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip2);
command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip3);
command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip4);
command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
command.Parameters.AddWithValue("@filtered", 0);
@@ -624,42 +607,62 @@ public class DbHandler
return ids;
}
public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
public void GetPreFilterQueueItem()
{
using SqliteConnection connection = new(_preFilteredConnectionString);
connection.Open();
const string temp = "SELECT seq FROM sqlite_sequence;";
SqliteCommand command = new(ReadPreFilteredStatement, connection);
using SqliteDataReader reader = command.ExecuteReader();
filterQueueItem = new();
Ip ip = new();
long id = 0;
if (!reader.HasRows)
{
return false;
}
SqliteCommand command2 = new(UpdatePreFilteredStatement, connection);
SqliteCommand command3 = new(temp, connection);
SqliteDataReader reader = command3.ExecuteReader();
long count = 0;
while (reader.Read())
{
ip.Ip1 = reader.GetInt32(0);
ip.Ip2 = reader.GetInt32(1);
ip.Ip3 = reader.GetInt32(2);
ip.Ip4 = reader.GetInt32(3);
filterQueueItem.ResponseCode = reader.GetInt32(4);
id = reader.GetInt64(5);
count = reader.GetInt64(0);
}
filterQueueItem.Ip = ip;
reader.Close();
command = new(UpdatePreFilteredStatement, connection);
command.Parameters.AddWithValue("@id", id);
for (long i = 0; i < count; i++)
{
if (_preFilteredQueue.Count > 1000)
{
Thread.Sleep(5);
}
command.Parameters.AddWithValue("@id", i);
reader = command.ExecuteReader();
command.Parameters.Clear();
Ip ip = new();
command.ExecuteNonQuery();
command.Dispose();
return true;
if (!reader.HasRows)
{
Console.WriteLine("Uuhhh");
}
while (reader.Read())
{
ip.Ip1 = reader.GetInt32(0);
ip.Ip2 = reader.GetInt32(1);
ip.Ip3 = reader.GetInt32(2);
ip.Ip4 = reader.GetInt32(3);
ip.ResponseCode = reader.GetInt32(4);
}
reader.Close();
_preFilteredQueue.Enqueue(ip);
command2.Parameters.AddWithValue("@id", i);
command2.ExecuteNonQuery();
command2.Parameters.Clear();
}
}
private static long GetDiscardedIndexesForSpecificDb(string connectionString)
@@ -779,8 +782,6 @@ public class DbHandler
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
_discardedConnectionStrings.Add(databaseName);
using SqliteConnection connection = new(databaseName);
connection.Open();