diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 0be4d0e..5b66860 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Diagnostics; -using System.Text; using Backend.Helper; using Models.Handler; using Models.Model.Backend; @@ -15,18 +14,18 @@ public class ContentFilter private readonly string _getDomainPort443; private bool _stop; private int _timeOut; - private string _basePath; + private readonly string _basePath; public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler, string basePath) { _queue = queue; _dbHandler = dbHandler; _basePath = basePath; - + _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; - SetTimeout(60000); + SetTimeout(3000); } public void SetTimeout(int timeOut) @@ -39,6 +38,7 @@ public class ContentFilter WaitHandle[] waitHandles = new WaitHandle[1]; EventWaitHandle handle = new(false, EventResetMode.ManualReset); waitHandles[0] = handle; + Thread f = new (Filter!); f.Start(handle); @@ -229,12 +229,11 @@ public class ContentFilter Process proc = new(); proc.StartInfo = new() { - FileName = fileName, - Arguments = $"{ip}", + FileName = "/bin/bash", + Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, - RedirectStandardInput = false, CreateNoWindow = true }; diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 9267459..e4c457a 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -51,7 +51,7 @@ public class ThreadHandler { Thread.Sleep(10000); // Let the database handler instantiate and warm up first. - WaitHandle[] wait = _ipScanner.Start(2); + WaitHandle[] wait = _ipScanner.Start(64); WaitHandle.WaitAll(wait); @@ -78,7 +78,7 @@ public class ThreadHandler private void StartDiscardedDbHandler() { - WaitHandle[] wait = _dbHandler.Start(2); + WaitHandle[] wait = _dbHandler.Start(4); WaitHandle.WaitAll(wait); diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs index a1d7133..3e1b2a3 100644 --- a/Backend/Helper/HttpClientHelper.cs +++ b/Backend/Helper/HttpClientHelper.cs @@ -18,14 +18,14 @@ public static class HttpClientHelper client.DefaultRequestHeaders.Accept.Clear(); HttpResponseMessage? response = null; - + try { response = await client.GetAsync("/"); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } if (response is null || !response.IsSuccessStatusCode) @@ -35,7 +35,7 @@ public static class HttpClientHelper return await response.Content.ReadAsStringAsync(); } - + public static async Task HasRobotsTxt(string url, int port) { using HttpClient client = new(); @@ -57,9 +57,9 @@ public static class HttpClientHelper { response = await client.GetAsync("/robots.txt"); } - catch (Exception e) + catch { - Console.WriteLine(e); + // } return response is not null && response.IsSuccessStatusCode; diff --git a/Backend/Scripts/GetDomainNamePort443.sh b/Backend/Scripts/GetDomainNamePort443.sh index 5177341..308c5f8 100755 --- a/Backend/Scripts/GetDomainNamePort443.sh +++ b/Backend/Scripts/GetDomainNamePort443.sh @@ -1,3 +1,3 @@ #!/bin/bash -curl -k -vvI -m 10 https://"$1" &> 443Header.txt +curl -k -vvI -m 10 https://"$1" &> "$2" diff --git a/Backend/Scripts/GetDomainNamePort80.sh b/Backend/Scripts/GetDomainNamePort80.sh index 59f8260..5ffa2b7 100755 --- a/Backend/Scripts/GetDomainNamePort80.sh +++ b/Backend/Scripts/GetDomainNamePort80.sh @@ -1,3 +1,4 @@ #!/bin/bash -curl -k -vvI -m 10 http://"$1" &> 80Header.txt +curl -k -vvI -m 10 http://"$1" &> "$2" +#80Header.txt diff --git a/Models/Filtered.db b/Models/Filtered.db deleted file mode 100644 index 48380b1..0000000 Binary files a/Models/Filtered.db and /dev/null differ diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 8ca07ad..4c25fda 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -25,6 +25,7 @@ public class DbHandler private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;"; private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; + private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; @@ -365,7 +366,27 @@ public class DbHandler public bool GetFilteredIp(string ip) { - return true; + using SqliteConnection connection = new(_filteredConnectionString); + connection.Open(); + + using SqliteCommand command = new(ReadFilteredIpStatement, connection); + command.Parameters.AddWithValue("@ip", ip); + + using SqliteDataReader reader = command.ExecuteReader(); + + if (!reader.HasRows) + { + return false; + } + + string ipAddress = ""; + + while (reader.Read()) + { + ipAddress = reader.GetString(0); + } + + return ipAddress == ip; } public List GetSearchResults() diff --git a/Models/ScannerResume.db b/Models/ScannerResume.db deleted file mode 100644 index a44725c..0000000 Binary files a/Models/ScannerResume.db and /dev/null differ diff --git a/Models/mydb.db b/Models/mydb.db deleted file mode 100644 index 8ad61fd..0000000 Binary files a/Models/mydb.db and /dev/null differ