From f02203c14c3dd68cb544b72df81461eadc9d29c2 Mon Sep 17 00:00:00 2001 From: Rasmus Date: Wed, 27 Nov 2024 15:53:00 +0100 Subject: [PATCH] Fixed duplicate insertions into the filtered database. --- Backend/Handler/ContentFilter.cs | 13 ++++++------- Backend/Handler/ThreadHandler.cs | 4 ++-- Backend/Helper/HttpClientHelper.cs | 12 ++++++------ Backend/Scripts/GetDomainNamePort443.sh | 2 +- Backend/Scripts/GetDomainNamePort80.sh | 3 ++- Models/Filtered.db | Bin 12288 -> 0 bytes Models/Handler/DbHandler.cs | 23 ++++++++++++++++++++++- Models/ScannerResume.db | Bin 8192 -> 0 bytes Models/mydb.db | Bin 16384 -> 0 bytes 9 files changed, 39 insertions(+), 18 deletions(-) delete mode 100644 Models/Filtered.db delete mode 100644 Models/ScannerResume.db delete mode 100644 Models/mydb.db diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 0be4d0e..5b66860 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Diagnostics; -using System.Text; using Backend.Helper; using Models.Handler; using Models.Model.Backend; @@ -15,18 +14,18 @@ public class ContentFilter private readonly string _getDomainPort443; private bool _stop; private int _timeOut; - private string _basePath; + private readonly string _basePath; public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler, string basePath) { _queue = queue; _dbHandler = dbHandler; _basePath = basePath; - + _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; - SetTimeout(60000); + SetTimeout(3000); } public void SetTimeout(int timeOut) @@ -39,6 +38,7 @@ public class ContentFilter WaitHandle[] waitHandles = new WaitHandle[1]; EventWaitHandle handle = new(false, EventResetMode.ManualReset); waitHandles[0] = handle; + Thread f = new (Filter!); f.Start(handle); @@ -229,12 +229,11 @@ public class ContentFilter Process proc = new(); proc.StartInfo = new() { - FileName = fileName, - Arguments = $"{ip}", + FileName = "/bin/bash", + Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, - RedirectStandardInput = false, CreateNoWindow = true }; diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 9267459..e4c457a 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -51,7 +51,7 @@ public class ThreadHandler { Thread.Sleep(10000); // Let the database handler instantiate and warm up first. - WaitHandle[] wait = _ipScanner.Start(2); + WaitHandle[] wait = _ipScanner.Start(64); WaitHandle.WaitAll(wait); @@ -78,7 +78,7 @@ public class ThreadHandler private void StartDiscardedDbHandler() { - WaitHandle[] wait = _dbHandler.Start(2); + WaitHandle[] wait = _dbHandler.Start(4); WaitHandle.WaitAll(wait); diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs index a1d7133..3e1b2a3 100644 --- a/Backend/Helper/HttpClientHelper.cs +++ b/Backend/Helper/HttpClientHelper.cs @@ -18,14 +18,14 @@ public static class HttpClientHelper client.DefaultRequestHeaders.Accept.Clear(); HttpResponseMessage? response = null; - + try { response = await client.GetAsync("/"); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } if (response is null || !response.IsSuccessStatusCode) @@ -35,7 +35,7 @@ public static class HttpClientHelper return await response.Content.ReadAsStringAsync(); } - + public static async Task HasRobotsTxt(string url, int port) { using HttpClient client = new(); @@ -57,9 +57,9 @@ public static class HttpClientHelper { response = await client.GetAsync("/robots.txt"); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } return response is not null && response.IsSuccessStatusCode; diff --git a/Backend/Scripts/GetDomainNamePort443.sh b/Backend/Scripts/GetDomainNamePort443.sh index 5177341..308c5f8 100755 --- a/Backend/Scripts/GetDomainNamePort443.sh +++ b/Backend/Scripts/GetDomainNamePort443.sh @@ -1,3 +1,3 @@ #!/bin/bash -curl -k -vvI -m 10 https://"$1" &> 443Header.txt +curl -k -vvI -m 10 https://"$1" &> "$2" diff --git a/Backend/Scripts/GetDomainNamePort80.sh b/Backend/Scripts/GetDomainNamePort80.sh index 59f8260..5ffa2b7 100755 --- a/Backend/Scripts/GetDomainNamePort80.sh +++ b/Backend/Scripts/GetDomainNamePort80.sh @@ -1,3 +1,4 @@ #!/bin/bash -curl -k -vvI -m 10 http://"$1" &> 80Header.txt +curl -k -vvI -m 10 http://"$1" &> "$2" +#80Header.txt diff --git a/Models/Filtered.db b/Models/Filtered.db deleted file mode 100644 index 48380b104b163fc09888ac27e175777d56742c0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeH~L2KJE7>4b1ZArJmZiPWXc$b8h!CKgPUZ!=XspBO|X?7`&qb35!c4fIN`vW`f zx9uNI>6$fZzT9>yuRyl^z3(TkJ>!R$~ z#RYMBtRScOpi;Tre3+Y3TncKu zeT<&yF3D7OOKPL8RT3K+lw~D#E3Y!6X9UV6Ye$dcO#O)GSVOya2zO9i zyeGmUE$=5wh%Hr2r0!LA9~0`Cobm6;B+katPsh1dW-=vgQ8i3eV$GWtV#Po)Suvat zL-Ln177S z?~6_l+E4bJhoZUHoS)6@p&tbVKmY_l00ck)1V8`;KmY_l00cnb2@}Bi{|PTHUJV35 Y00ck)1V8`;KmY_l00ck)1nvp^0R_7|w*UYD diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 8ca07ad..4c25fda 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -25,6 +25,7 @@ public class DbHandler private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;"; private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; + private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; @@ -365,7 +366,27 @@ public class DbHandler public bool GetFilteredIp(string ip) { - return true; + using SqliteConnection connection = new(_filteredConnectionString); + connection.Open(); + + using SqliteCommand command = new(ReadFilteredIpStatement, connection); + command.Parameters.AddWithValue("@ip", ip); + + using SqliteDataReader reader = command.ExecuteReader(); + + if (!reader.HasRows) + { + return false; + } + + string ipAddress = ""; + + while (reader.Read()) + { + ipAddress = reader.GetString(0); + } + + return ipAddress == ip; } public List GetSearchResults() diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db deleted file mode 100644 index a44725c94c534967899d2392373bfd3dd9f212b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI1K}#D!6oB8%Mzfp5osBsZ6=cY%g(6D-z$UVa5K@zD&UI~86ETTyHiw=XQlUMz z;1AG?=hkCS=^rR1$KLu2gdpT1(zhEIDl~fY^oE_;+3(G~8NNLYQz;c1zAN|IUeodA zD=>gDFr@^5OROAgRF<$4i$3tp{^?KaUc$upm&Vu#7IPgV&--(AQ>&nmB372 z;XH(D1e1sc@a;b2&_meg3XF}os!i4~p_fN;m6?s^DmX|NzhRNj$3lh;GQ5i)C~%ur eCQQTJK{~90hsls*GE;?7imAAhV25TLZvFzJ^MOqO diff --git a/Models/mydb.db b/Models/mydb.db deleted file mode 100644 index 8ad61fdd412e0581b69fd4f48e17630e8d5f7cba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI#-A=+V6bJB4Pyz|+l_Vy_(klqWM127#sb*me93_z}GF*uyY```iz>in)QG72u zWMM4e1CW1{rCrb3_58M*(|a~1p&;X#@7f``Q_dAtRqhE<6eTCGti0lOAqUBOH4gmK z=alQ=Wl=tmDSj(wrPCkTp+Nuw5P$##AOHafKmY;|fWSV1k8&ngs#Mgk3D)k2=?4$bQ(XHU!`W``5AZy1eeeoph8y~v5gq$Ps+ z%nL+w=7`o0b&pO;^}0G(=>&_ZtZ0XUSS*G2DpI$ZEvHnl>e;Sn$d$GCv6?HD%j)OV zYENtFg%ei7S|jlbP`q3wrUzMqoRw_3f7dCvYI zfAfXym`isgd`i-1D%7IdTo(Ok5P$##AOHafKmY;|fB*y_009UbumI}+1OB*34FV8= t00bZa0SG_<0uX=z1R$^$K>d$j0Rj+!00bZa0SG_<0uX=z1R!wu0>50^w*CMB