Unfiltered object is now a struct.

This commit is contained in:
Rasmus Rasmussen 2024-11-28 14:24:33 +01:00
parent eedca27a11
commit 3d8f903d30
14 changed files with 149 additions and 77 deletions

View File

@ -55,26 +55,29 @@ public class ContentFilter
{ {
if (_stop) break; if (_stop) break;
Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i); Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(i);
if (unfiltered is null || unfiltered.Filtered == 1) continue; if (unfiltered.Filtered) continue;
unfiltered.Filtered = 1; Ip ip = unfiltered.Ip;
unfiltered.Filtered = true;
QueueItem superUnfilteredObject = new() QueueItem superUnfilteredObject = new()
{ {
Unfiltered = unfiltered, Unfiltered = unfiltered,
Operations = Operations.Update Operations = Operations.Update,
DbType = DbType.Unfiltered
}; };
_queue.Enqueue(superUnfilteredObject); _queue.Enqueue(superUnfilteredObject);
if (_dbHandler.GetFilteredIp(unfiltered.Ip)) if (_dbHandler.FilteredIpExists(unfiltered.Ip))
{ {
continue; continue;
} }
Filtered filtered = GetSiteData(unfiltered.Ip); Filtered filtered = GetSiteData(ip);
filtered.Port1 = unfiltered.Port1; filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2; filtered.Port2 = unfiltered.Port2;
@ -82,7 +85,8 @@ public class ContentFilter
QueueItem superFilteredObject = new() QueueItem superFilteredObject = new()
{ {
Filtered = filtered, Filtered = filtered,
Operations = Operations.Insert Operations = Operations.Insert,
DbType = DbType.Filtered
}; };
_queue.Enqueue(superFilteredObject); _queue.Enqueue(superFilteredObject);
@ -94,7 +98,7 @@ public class ContentFilter
((EventWaitHandle) obj).Set(); ((EventWaitHandle) obj).Set();
} }
private Filtered GetSiteData(string ip) private Filtered GetSiteData(Ip ip)
{ {
StartProcess(ip, 80); StartProcess(ip, 80);
StartProcess(ip, 443); StartProcess(ip, 443);
@ -222,15 +226,15 @@ public class ContentFilter
return siteData; return siteData;
} }
private void StartProcess(string ip, int port) private void StartProcess(Ip ip, int port)
{ {
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
//Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}");
Process proc = new(); Process proc = new();
proc.StartInfo = new() proc.StartInfo = new()
{ {
FileName = "/bin/bash", FileName = "/bin/bash",
Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt", Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
UseShellExecute = false, UseShellExecute = false,
RedirectStandardOutput = false, RedirectStandardOutput = false,
RedirectStandardError = false, RedirectStandardError = false,

View File

@ -127,16 +127,22 @@ public class IpScanner
resumeObject.FourthByte = l; resumeObject.FourthByte = l;
break; break;
} }
string ip = $"{i}.{j}.{k}.{l}"; Ip ip = new()
{
Ip1 = i,
Ip2 = j,
Ip3 = k,
Ip4 = l
};
IPStatus responseCode = IPStatus.Unknown; IPStatus responseCode = IPStatus.Unknown;
try try
{ {
// Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception. // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
// https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
_ = IPAddress.TryParse(ip, out IPAddress? address); _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
if (address is not null) if (address is not null)
{ {
responseCode = ping.Send(address, _timeout, buf, null).Status; responseCode = ping.Send(address, _timeout, buf, null).Status;
@ -149,19 +155,19 @@ public class IpScanner
if (responseCode != IPStatus.Success) if (responseCode != IPStatus.Success)
{ {
_discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode));
continue; continue;
} }
(int, int) ports = TcpClientHelper.CheckPort(ip, 80, 443); (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
if (ports is { Item1: 0, Item2: 0 }) if (ports is { Item1: 0, Item2: 0 })
{ {
_discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode));
continue; continue;
} }
_queue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode, ports)); _queue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
} }
if (_stop) if (_stop)
@ -208,21 +214,21 @@ public class IpScanner
return discarded; return discarded;
} }
private static QueueItem CreateUnfilteredQueueItem(string ip, int responseCode, (int, int) ports) private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
{ {
Unfiltered unfiltered = new() Unfiltered unfiltered = new()
{ {
Ip = ip, Ip = ip,
ResponseCode = responseCode,
Port1 = ports.Item1, Port1 = ports.Item1,
Port2 = ports.Item2, Port2 = ports.Item2,
Filtered = 0 Filtered = false
}; };
QueueItem superUnfilteredObject = new() QueueItem superUnfilteredObject = new()
{ {
Unfiltered = unfiltered, Unfiltered = unfiltered,
Operations = Operations.Insert Operations = Operations.Insert,
DbType = DbType.Unfiltered
}; };
return superUnfilteredObject; return superUnfilteredObject;

View File

@ -49,9 +49,9 @@ public class ThreadHandler
private void StartScanner() private void StartScanner()
{ {
Thread.Sleep(10000); // Let the database handler instantiate and warm up first. Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
WaitHandle[] wait = _ipScanner.Start(4); WaitHandle[] wait = _ipScanner.Start(64);
WaitHandle.WaitAll(wait); WaitHandle.WaitAll(wait);

View File

@ -1,19 +1,13 @@
DROP TABLE Unfiltered; DROP TABLE Unfiltered;
DROP TABLE Runtimes;
CREATE TABLE "Unfiltered" ( CREATE TABLE "Unfiltered" (
"Id" INTEGER NOT NULL, "Id" INTEGER NOT NULL,
"Ip" TEXT NOT NULL, "Ip1" INTEGER NOT NULL,
"ResponseCode" INTEGER NOT NULL, "Ip2" INTEGER NOT NULL,
"Ip3" INTEGER NOT NULL,
"Ip4" INTEGER NOT NULL,
"Port1" INTEGER NOT NULL, "Port1" INTEGER NOT NULL,
"Port2" INTEGER NOT NULL, "Port2" INTEGER NOT NULL,
"Filtered" INTEGER NOT NULL, "Filtered" INTEGER NOT NULL,
CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT) CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT)
); )
CREATE TABLE "Runtimes" (
"Id" INTEGER NOT NULL CONSTRAINT "PK_Runtimes" PRIMARY KEY AUTOINCREMENT,
"StartTime" TEXT NOT NULL,
"EndTime" TEXT NOT NULL,
"ThreadNumber" INTEGER NOT NULL
);

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,3 +1,4 @@
using System.Collections;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
using Models.Model.Backend; using Models.Model.Backend;
@ -16,8 +17,8 @@ public class DbHandler
private readonly string _resumeConnectionString; private readonly string _resumeConnectionString;
private readonly List<string> _discardedConnectionStrings = []; private readonly List<string> _discardedConnectionStrings = [];
private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)"; private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)"; private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)"; private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)";
private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);"; private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
@ -25,7 +26,7 @@ public class DbHandler
private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;"; private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;";
private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;"; private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
@ -90,20 +91,24 @@ public class DbHandler
if (queueItem is null) { continue; } if (queueItem is null) { continue; }
switch (queueItem.Operations) if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Unfiltered)
{ {
case Operations.Insert when queueItem.Unfiltered is not null: InsertUnfiltered(queueItem.Unfiltered);
InsertUnfiltered(queueItem.Unfiltered); }
break;
case Operations.Insert when queueItem.Filtered is not null: else if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Filtered)
InsertFiltered(queueItem.Filtered); {
break; InsertFiltered(queueItem.Filtered!);
case Operations.Insert when queueItem.ResumeObject is not null: }
InsertResumeObject(queueItem.ResumeObject);
break; else if (queueItem.Operations == Operations.Insert && queueItem.ResumeObject is not null)
case Operations.Update when queueItem.Unfiltered is not null: {
UpdateUnfiltered(queueItem.Unfiltered); InsertResumeObject(queueItem.ResumeObject);
break; }
else if (queueItem.Operations == Operations.Update && queueItem.DbType == DbType.Unfiltered)
{
UpdateUnfiltered(queueItem.Unfiltered);
} }
} }
@ -170,8 +175,10 @@ public class DbHandler
using SqliteCommand command = new(InsertStatement, connection); using SqliteCommand command = new(InsertStatement, connection);
command.Parameters.AddWithValue("@ip", unfiltered.Ip); command.Parameters.AddWithValue("@ip1", unfiltered.Ip.Ip1);
command.Parameters.AddWithValue("@responseCode", unfiltered.ResponseCode); command.Parameters.AddWithValue("@ip2", unfiltered.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", unfiltered.Ip.Ip3);
command.Parameters.AddWithValue("@ip4", unfiltered.Ip.Ip4);
command.Parameters.AddWithValue("@port1", unfiltered.Port1); command.Parameters.AddWithValue("@port1", unfiltered.Port1);
command.Parameters.AddWithValue("@port2", unfiltered.Port2); command.Parameters.AddWithValue("@port2", unfiltered.Port2);
command.Parameters.AddWithValue("@filtered", unfiltered.Filtered); command.Parameters.AddWithValue("@filtered", unfiltered.Filtered);
@ -201,7 +208,10 @@ public class DbHandler
using SqliteCommand command = new(InsertIntoFiltered, connection); using SqliteCommand command = new(InsertIntoFiltered, connection);
command.Parameters.AddWithValue("@ip", filtered.Ip); command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1);
command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3);
command.Parameters.AddWithValue("@ip4", filtered.Ip.Ip4);
command.Parameters.AddWithValue("@port1", filtered.Port1); command.Parameters.AddWithValue("@port1", filtered.Port1);
command.Parameters.AddWithValue("@port2", filtered.Port2); command.Parameters.AddWithValue("@port2", filtered.Port2);
command.Parameters.AddWithValue("@url1", filtered.Url1); command.Parameters.AddWithValue("@url1", filtered.Url1);
@ -267,7 +277,7 @@ public class DbHandler
connection.Close(); connection.Close();
} }
public Unfiltered? ReadUnfilteredWithId(long id) public Unfiltered ReadUnfilteredWithId(long id)
{ {
using SqliteConnection connection = new(_unfilteredConnectionString); using SqliteConnection connection = new(_unfilteredConnectionString);
connection.Open(); connection.Open();
@ -276,19 +286,28 @@ public class DbHandler
command.Parameters.AddWithValue("@id", id); command.Parameters.AddWithValue("@id", id);
using SqliteDataReader reader = command.ExecuteReader(); using SqliteDataReader reader = command.ExecuteReader();
if (!reader.HasRows) return null; if (!reader.HasRows) return new();
Unfiltered unfiltered = new(); Unfiltered unfiltered = new();
Ip ip = new();
while (reader.Read()) while (reader.Read())
{ {
unfiltered.Id = reader.GetInt32(0); unfiltered.Id = reader.GetInt32(0);
unfiltered.Ip = reader.GetString(1); ip.Ip1 = reader.GetInt32(1);
unfiltered.Port1 = reader.GetInt32(3); ip.Ip2 = reader.GetInt32(2);
unfiltered.Port2 = reader.GetInt32(4); ip.Ip3 = reader.GetInt32(3);
unfiltered.Filtered = reader.GetInt32(5); ip.Ip4 = reader.GetInt32(4);
//Console.WriteLine(ip + "lmo");
unfiltered.Port1 = reader.GetInt32(5);
unfiltered.Port2 = reader.GetInt32(6);
unfiltered.Filtered = reader.GetBoolean(7);
} }
//Console.WriteLine(ip + "aaaaa");
unfiltered.Ip = ip;
//Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs");
return unfiltered; return unfiltered;
} }
@ -364,13 +383,16 @@ public class DbHandler
return rowId; return rowId;
} }
public bool GetFilteredIp(string ip) public bool FilteredIpExists(Ip ip)
{ {
using SqliteConnection connection = new(_filteredConnectionString); using SqliteConnection connection = new(_filteredConnectionString);
connection.Open(); connection.Open();
using SqliteCommand command = new(ReadFilteredIpStatement, connection); using SqliteCommand command = new(ReadFilteredIpStatement, connection);
command.Parameters.AddWithValue("@ip", ip); command.Parameters.AddWithValue("@ip1", ip.Ip1);
command.Parameters.AddWithValue("@ip2", ip.Ip2);
command.Parameters.AddWithValue("@ip3", ip.Ip3);
command.Parameters.AddWithValue("@ip4", ip.Ip4);
using SqliteDataReader reader = command.ExecuteReader(); using SqliteDataReader reader = command.ExecuteReader();
@ -379,14 +401,37 @@ public class DbHandler
return false; return false;
} }
string ipAddress = ""; Ip tempIp = new();
while (reader.Read()) while (reader.Read())
{ {
ipAddress = reader.GetString(0); tempIp.Ip1 = reader.GetInt32(0);
tempIp.Ip2 = reader.GetInt32(1);
tempIp.Ip3 = reader.GetInt32(2);
tempIp.Ip4 = reader.GetInt32(3);
} }
return ipAddress == ip; if (tempIp.Ip1 != ip.Ip1)
{
return false;
}
if (tempIp.Ip2 != ip.Ip2)
{
return false;
}
if (tempIp.Ip3 != ip.Ip3)
{
return false;
}
if (tempIp.Ip4 != ip.Ip4)
{
return false;
}
return true;
} }
public List<SearchResult?> GetSearchResults() public List<SearchResult?> GetSearchResults()

View File

@ -0,0 +1,7 @@
namespace Models.Model.Backend;
public enum DbType
{
Unfiltered,
Filtered,
}

View File

@ -2,7 +2,7 @@ namespace Models.Model.Backend;
public class Filtered public class Filtered
{ {
public string Ip { get; set; } = ""; public Ip Ip { get; set; }
public string Title1 { get; set; } = ""; public string Title1 { get; set; } = "";
public string Title2 { get; set; } = ""; public string Title2 { get; set; } = "";
public string Description1 { get; set; } = ""; public string Description1 { get; set; } = "";

View File

@ -0,0 +1,17 @@
namespace Models.Model.Backend;
public struct Ip
{
public int Ip1 { get; set; }
public int Ip2 { get; set; }
public int Ip3 { get; set; }
public int Ip4 { get; set; }
public override string ToString()
{
return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
}
}

View File

@ -2,8 +2,9 @@ namespace Models.Model.Backend;
public class QueueItem public class QueueItem
{ {
public Unfiltered? Unfiltered { get; init; } public Unfiltered Unfiltered { get; init; }
public Filtered? Filtered { get; init; } public Filtered? Filtered { get; init; }
public ScannerResumeObject? ResumeObject { get; init; } public ScannerResumeObject? ResumeObject { get; init; }
public Operations Operations { get; init; } public Operations Operations { get; init; }
public DbType DbType { get; init; }
} }

View File

@ -1,16 +1,14 @@
namespace Models.Model.Backend; namespace Models.Model.Backend;
public class Unfiltered public struct Unfiltered
{ {
public int Id { get; set; } public int Id { get; set; }
public string Ip { get; set; } = ""; public Ip Ip { get; set; }
public int ResponseCode { get; init; }
public int Port1 { get; set; } public int Port1 { get; set; }
public int Port2 { get; set; } public int Port2 { get; set; }
public int Filtered { get; set; } public bool Filtered { get; set; }
} }

Binary file not shown.