From 3d8f903d307ac8a77c0cbfe342552c9efe73034b Mon Sep 17 00:00:00 2001 From: Rasmus Date: Thu, 28 Nov 2024 14:24:33 +0100 Subject: [PATCH] Unfiltered object is now a struct. --- Backend/Handler/ContentFilter.cs | 26 ++++--- Backend/Handler/IpScanner.cs | 30 +++++---- Backend/Handler/ThreadHandler.cs | 4 +- Backend/Sql/DropAndCreateMyDB.sql | 16 ++--- Models/BackupDB/Filtered.db | Bin 12288 -> 12288 bytes Models/BackupDB/mydb.db | Bin 16384 -> 16384 bytes Models/Filtered.db | Bin 12288 -> 12288 bytes Models/Handler/DbHandler.cs | 105 ++++++++++++++++++++--------- Models/Model/Backend/DbType.cs | 7 ++ Models/Model/Backend/Filtered.cs | 2 +- Models/Model/Backend/IP.cs | 17 +++++ Models/Model/Backend/QueueItem.cs | 3 +- Models/Model/Backend/Unfiltered.cs | 16 ++--- Models/mydb.db | Bin 16384 -> 16384 bytes 14 files changed, 149 insertions(+), 77 deletions(-) create mode 100644 Models/Model/Backend/DbType.cs create mode 100644 Models/Model/Backend/IP.cs diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 5b66860..0284a9a 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -55,26 +55,29 @@ public class ContentFilter { if (_stop) break; - Unfiltered? unfiltered = _dbHandler.ReadUnfilteredWithId(i); + Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(i); - if (unfiltered is null || unfiltered.Filtered == 1) continue; + if (unfiltered.Filtered) continue; - unfiltered.Filtered = 1; + Ip ip = unfiltered.Ip; + + unfiltered.Filtered = true; QueueItem superUnfilteredObject = new() { Unfiltered = unfiltered, - Operations = Operations.Update + Operations = Operations.Update, + DbType = DbType.Unfiltered }; _queue.Enqueue(superUnfilteredObject); - if (_dbHandler.GetFilteredIp(unfiltered.Ip)) + if (_dbHandler.FilteredIpExists(unfiltered.Ip)) { continue; } - Filtered filtered = GetSiteData(unfiltered.Ip); + Filtered filtered = GetSiteData(ip); filtered.Port1 = unfiltered.Port1; filtered.Port2 = unfiltered.Port2; @@ -82,7 +85,8 @@ public class ContentFilter QueueItem superFilteredObject = new() { Filtered = filtered, - Operations = Operations.Insert + Operations = Operations.Insert, + DbType = DbType.Filtered }; _queue.Enqueue(superFilteredObject); @@ -94,7 +98,7 @@ public class ContentFilter ((EventWaitHandle) obj).Set(); } - private Filtered GetSiteData(string ip) + private Filtered GetSiteData(Ip ip) { StartProcess(ip, 80); StartProcess(ip, 443); @@ -222,15 +226,15 @@ public class ContentFilter return siteData; } - private void StartProcess(string ip, int port) + private void StartProcess(Ip ip, int port) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; - + //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}"); Process proc = new(); proc.StartInfo = new() { FileName = "/bin/bash", - Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt", + Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index 7838c73..90ae450 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -127,16 +127,22 @@ public class IpScanner resumeObject.FourthByte = l; break; } - - string ip = $"{i}.{j}.{k}.{l}"; - + + Ip ip = new() + { + Ip1 = i, + Ip2 = j, + Ip3 = k, + Ip4 = l + }; + IPStatus responseCode = IPStatus.Unknown; try { // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception. // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages - _ = IPAddress.TryParse(ip, out IPAddress? address); + _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address); if (address is not null) { responseCode = ping.Send(address, _timeout, buf, null).Status; @@ -149,19 +155,19 @@ public class IpScanner if (responseCode != IPStatus.Success) { - _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); + _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode)); continue; } - (int, int) ports = TcpClientHelper.CheckPort(ip, 80, 443); + (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443); if (ports is { Item1: 0, Item2: 0 }) { - _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); + _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode)); continue; } - _queue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode, ports)); + _queue.Enqueue(CreateUnfilteredQueueItem(ip, ports)); } if (_stop) @@ -208,21 +214,21 @@ public class IpScanner return discarded; } - private static QueueItem CreateUnfilteredQueueItem(string ip, int responseCode, (int, int) ports) + private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) { Unfiltered unfiltered = new() { Ip = ip, - ResponseCode = responseCode, Port1 = ports.Item1, Port2 = ports.Item2, - Filtered = 0 + Filtered = false }; QueueItem superUnfilteredObject = new() { Unfiltered = unfiltered, - Operations = Operations.Insert + Operations = Operations.Insert, + DbType = DbType.Unfiltered }; return superUnfilteredObject; diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 6fe360a..82316ef 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -49,9 +49,9 @@ public class ThreadHandler private void StartScanner() { - Thread.Sleep(10000); // Let the database handler instantiate and warm up first. + Thread.Sleep(5000); // Let the database handler instantiate and warm up first. - WaitHandle[] wait = _ipScanner.Start(4); + WaitHandle[] wait = _ipScanner.Start(64); WaitHandle.WaitAll(wait); diff --git a/Backend/Sql/DropAndCreateMyDB.sql b/Backend/Sql/DropAndCreateMyDB.sql index b7f6379..b637bb7 100644 --- a/Backend/Sql/DropAndCreateMyDB.sql +++ b/Backend/Sql/DropAndCreateMyDB.sql @@ -1,19 +1,13 @@ DROP TABLE Unfiltered; -DROP TABLE Runtimes; CREATE TABLE "Unfiltered" ( "Id" INTEGER NOT NULL, - "Ip" TEXT NOT NULL, - "ResponseCode" INTEGER NOT NULL, + "Ip1" INTEGER NOT NULL, + "Ip2" INTEGER NOT NULL, + "Ip3" INTEGER NOT NULL, + "Ip4" INTEGER NOT NULL, "Port1" INTEGER NOT NULL, "Port2" INTEGER NOT NULL, "Filtered" INTEGER NOT NULL, CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT) -); - -CREATE TABLE "Runtimes" ( - "Id" INTEGER NOT NULL CONSTRAINT "PK_Runtimes" PRIMARY KEY AUTOINCREMENT, - "StartTime" TEXT NOT NULL, - "EndTime" TEXT NOT NULL, - "ThreadNumber" INTEGER NOT NULL -); \ No newline at end of file +) \ No newline at end of file diff --git a/Models/BackupDB/Filtered.db b/Models/BackupDB/Filtered.db index 48380b104b163fc09888ac27e175777d56742c0a..93bce66fc88ab3425ba107d6a5f3d5009b42d7ad 100644 GIT binary patch delta 168 zcmZojXh@hKEy%{ez`zW|Fu*ZU#~3K67ca%jqs+j>KA(a6BK!P}h5GDV9s11d;Ja57R-P*Cvn3vqRK4N~y)4^i+7_3_c+;#Bf1Fvcxqf+V%Mo~My{ c@n%}%m~D?6LpM%f_m{%y!_$}OuR)5d`EeUHWobK;o=u( zEEQK&WNg)+yq;Hrt0{n)9mH+P26BTK4JRw`iP?G<7%3?zD0upXxVpOrDfsz^DENi? z_~>wPDtQ(dm&NHc`hID5w`N#S0W-=3UCbFV1&#W5ENSi4#J(nq-;T z#T6A9TUaO8@CL~%afY}?geds=197O2j}8~7Qc!AfL4ICws&jry>g4mhVw27I9KA(a6BK!P}h5GDV9s11d;Ja57R-P*Cvn3vqRK4N~y)4^i+7_3_c+;#Bf1Fvcxqf+V%Mo~My{ c@ _discardedConnectionStrings = []; - private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip, ResponseCode, Port1, Port2, Filtered) VALUES (@ip, @responseCode, @port1, @port2, @filtered)"; - private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)"; + private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)"; + private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)"; private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)"; private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);"; @@ -25,7 +26,7 @@ public class DbHandler private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;"; private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; - private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;"; + private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; @@ -90,20 +91,24 @@ public class DbHandler if (queueItem is null) { continue; } - switch (queueItem.Operations) + if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Unfiltered) { - case Operations.Insert when queueItem.Unfiltered is not null: - InsertUnfiltered(queueItem.Unfiltered); - break; - case Operations.Insert when queueItem.Filtered is not null: - InsertFiltered(queueItem.Filtered); - break; - case Operations.Insert when queueItem.ResumeObject is not null: - InsertResumeObject(queueItem.ResumeObject); - break; - case Operations.Update when queueItem.Unfiltered is not null: - UpdateUnfiltered(queueItem.Unfiltered); - break; + InsertUnfiltered(queueItem.Unfiltered); + } + + else if (queueItem.Operations == Operations.Insert && queueItem.DbType == DbType.Filtered) + { + InsertFiltered(queueItem.Filtered!); + } + + else if (queueItem.Operations == Operations.Insert && queueItem.ResumeObject is not null) + { + InsertResumeObject(queueItem.ResumeObject); + } + + else if (queueItem.Operations == Operations.Update && queueItem.DbType == DbType.Unfiltered) + { + UpdateUnfiltered(queueItem.Unfiltered); } } @@ -170,8 +175,10 @@ public class DbHandler using SqliteCommand command = new(InsertStatement, connection); - command.Parameters.AddWithValue("@ip", unfiltered.Ip); - command.Parameters.AddWithValue("@responseCode", unfiltered.ResponseCode); + command.Parameters.AddWithValue("@ip1", unfiltered.Ip.Ip1); + command.Parameters.AddWithValue("@ip2", unfiltered.Ip.Ip2); + command.Parameters.AddWithValue("@ip3", unfiltered.Ip.Ip3); + command.Parameters.AddWithValue("@ip4", unfiltered.Ip.Ip4); command.Parameters.AddWithValue("@port1", unfiltered.Port1); command.Parameters.AddWithValue("@port2", unfiltered.Port2); command.Parameters.AddWithValue("@filtered", unfiltered.Filtered); @@ -201,7 +208,10 @@ public class DbHandler using SqliteCommand command = new(InsertIntoFiltered, connection); - command.Parameters.AddWithValue("@ip", filtered.Ip); + command.Parameters.AddWithValue("@ip1", filtered.Ip.Ip1); + command.Parameters.AddWithValue("@ip2", filtered.Ip.Ip2); + command.Parameters.AddWithValue("@ip3", filtered.Ip.Ip3); + command.Parameters.AddWithValue("@ip4", filtered.Ip.Ip4); command.Parameters.AddWithValue("@port1", filtered.Port1); command.Parameters.AddWithValue("@port2", filtered.Port2); command.Parameters.AddWithValue("@url1", filtered.Url1); @@ -267,7 +277,7 @@ public class DbHandler connection.Close(); } - public Unfiltered? ReadUnfilteredWithId(long id) + public Unfiltered ReadUnfilteredWithId(long id) { using SqliteConnection connection = new(_unfilteredConnectionString); connection.Open(); @@ -276,19 +286,28 @@ public class DbHandler command.Parameters.AddWithValue("@id", id); using SqliteDataReader reader = command.ExecuteReader(); - if (!reader.HasRows) return null; + if (!reader.HasRows) return new(); Unfiltered unfiltered = new(); + Ip ip = new(); while (reader.Read()) { unfiltered.Id = reader.GetInt32(0); - unfiltered.Ip = reader.GetString(1); - unfiltered.Port1 = reader.GetInt32(3); - unfiltered.Port2 = reader.GetInt32(4); - unfiltered.Filtered = reader.GetInt32(5); + ip.Ip1 = reader.GetInt32(1); + ip.Ip2 = reader.GetInt32(2); + ip.Ip3 = reader.GetInt32(3); + ip.Ip4 = reader.GetInt32(4); + //Console.WriteLine(ip + "lmo"); + unfiltered.Port1 = reader.GetInt32(5); + unfiltered.Port2 = reader.GetInt32(6); + unfiltered.Filtered = reader.GetBoolean(7); } + //Console.WriteLine(ip + "aaaaa"); + unfiltered.Ip = ip; + //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs"); + return unfiltered; } @@ -364,13 +383,16 @@ public class DbHandler return rowId; } - public bool GetFilteredIp(string ip) + public bool FilteredIpExists(Ip ip) { using SqliteConnection connection = new(_filteredConnectionString); connection.Open(); using SqliteCommand command = new(ReadFilteredIpStatement, connection); - command.Parameters.AddWithValue("@ip", ip); + command.Parameters.AddWithValue("@ip1", ip.Ip1); + command.Parameters.AddWithValue("@ip2", ip.Ip2); + command.Parameters.AddWithValue("@ip3", ip.Ip3); + command.Parameters.AddWithValue("@ip4", ip.Ip4); using SqliteDataReader reader = command.ExecuteReader(); @@ -379,14 +401,37 @@ public class DbHandler return false; } - string ipAddress = ""; + Ip tempIp = new(); while (reader.Read()) { - ipAddress = reader.GetString(0); + tempIp.Ip1 = reader.GetInt32(0); + tempIp.Ip2 = reader.GetInt32(1); + tempIp.Ip3 = reader.GetInt32(2); + tempIp.Ip4 = reader.GetInt32(3); } - return ipAddress == ip; + if (tempIp.Ip1 != ip.Ip1) + { + return false; + } + + if (tempIp.Ip2 != ip.Ip2) + { + return false; + } + + if (tempIp.Ip3 != ip.Ip3) + { + return false; + } + + if (tempIp.Ip4 != ip.Ip4) + { + return false; + } + + return true; } public List GetSearchResults() diff --git a/Models/Model/Backend/DbType.cs b/Models/Model/Backend/DbType.cs new file mode 100644 index 0000000..29cdab8 --- /dev/null +++ b/Models/Model/Backend/DbType.cs @@ -0,0 +1,7 @@ +namespace Models.Model.Backend; + +public enum DbType +{ + Unfiltered, + Filtered, +} \ No newline at end of file diff --git a/Models/Model/Backend/Filtered.cs b/Models/Model/Backend/Filtered.cs index bc948fa..5499d3e 100644 --- a/Models/Model/Backend/Filtered.cs +++ b/Models/Model/Backend/Filtered.cs @@ -2,7 +2,7 @@ namespace Models.Model.Backend; public class Filtered { - public string Ip { get; set; } = ""; + public Ip Ip { get; set; } public string Title1 { get; set; } = ""; public string Title2 { get; set; } = ""; public string Description1 { get; set; } = ""; diff --git a/Models/Model/Backend/IP.cs b/Models/Model/Backend/IP.cs new file mode 100644 index 0000000..9abe538 --- /dev/null +++ b/Models/Model/Backend/IP.cs @@ -0,0 +1,17 @@ +namespace Models.Model.Backend; + +public struct Ip +{ + public int Ip1 { get; set; } + + public int Ip2 { get; set; } + + public int Ip3 { get; set; } + + public int Ip4 { get; set; } + + public override string ToString() + { + return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}"; + } +} \ No newline at end of file diff --git a/Models/Model/Backend/QueueItem.cs b/Models/Model/Backend/QueueItem.cs index b0693b8..471e7ca 100644 --- a/Models/Model/Backend/QueueItem.cs +++ b/Models/Model/Backend/QueueItem.cs @@ -2,8 +2,9 @@ namespace Models.Model.Backend; public class QueueItem { - public Unfiltered? Unfiltered { get; init; } + public Unfiltered Unfiltered { get; init; } public Filtered? Filtered { get; init; } public ScannerResumeObject? ResumeObject { get; init; } public Operations Operations { get; init; } + public DbType DbType { get; init; } } \ No newline at end of file diff --git a/Models/Model/Backend/Unfiltered.cs b/Models/Model/Backend/Unfiltered.cs index 89d2520..566e856 100644 --- a/Models/Model/Backend/Unfiltered.cs +++ b/Models/Model/Backend/Unfiltered.cs @@ -1,16 +1,14 @@ namespace Models.Model.Backend; -public class Unfiltered +public struct Unfiltered { public int Id { get; set; } - - public string Ip { get; set; } = ""; - - public int ResponseCode { get; init; } - + + public Ip Ip { get; set; } + public int Port1 { get; set; } - + public int Port2 { get; set; } - - public int Filtered { get; set; } + + public bool Filtered { get; set; } } \ No newline at end of file diff --git a/Models/mydb.db b/Models/mydb.db index 8ad61fdd412e0581b69fd4f48e17630e8d5f7cba..ef45e70ffa2d8792f084268584b3c401d3456d4a 100644 GIT binary patch delta 198 zcmZo@U~Fh$oFFa8$-uzC0>n%}%m~D?6LpM%f_m{%y!_$}OuR)5d`EeUHWobK;o=u( zEEQK&WNg)+yq;Hrt0{n)9mH+P26BTK4JRw`iP?G<7%3?zD0upXxVpOrDfsz^DENi? z_~>wPDtQ(dm&NHc`hID5w`N#S0W-=3UCbFV1&#W5ENSi4#J(nq-;T z#T6A9TUaO8@CL~%afY}?geds=197O2j}8~7Qc!AfL4ICws&jry>g4mhVw27I9