Backend/Handler/ContentFilter.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Text;
 using Backend.Helper;
 using Models.Handler;
 using Models.Model.Backend;
@@ -14,18 +15,18 @@ public class ContentFilter
     private readonly string _getDomainPort443;
     private bool _stop;
     private int _timeOut;
-    private readonly string _basePath;
+    private string _basePath;
     
     public ContentFilter(ConcurrentQueue<QueueItem> queue, DbHandler dbHandler, string basePath)
     {
         _queue = queue;
         _dbHandler = dbHandler;
         _basePath = basePath;
-        
+
         _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
         _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
         
-        SetTimeout(3000);
+        SetTimeout(60000);
     }
     
     public void SetTimeout(int timeOut)
@@ -38,7 +39,6 @@ public class ContentFilter
         WaitHandle[] waitHandles = new WaitHandle[1];
         EventWaitHandle handle = new(false, EventResetMode.ManualReset);
         waitHandles[0] = handle;
-        
         Thread f = new (Filter!);
         f.Start(handle);
         
@@ -229,11 +229,12 @@ public class ContentFilter
         Process proc = new();
         proc.StartInfo = new()
         {
-            FileName = "/bin/bash",
-            Arguments = $"{fileName} {ip} {_basePath}/Backend/Scripts/{port}Header.txt",
+            FileName = fileName,
+            Arguments = $"{ip}",
             UseShellExecute = false,
             RedirectStandardOutput = false,
             RedirectStandardError = false,
+            RedirectStandardInput = false,
             CreateNoWindow = true
         };
         

Backend/Handler/ThreadHandler.cs

@@ -51,7 +51,7 @@ public class ThreadHandler
     {
         Thread.Sleep(10000); // Let the database handler instantiate and warm up first.
         
-        WaitHandle[] wait = _ipScanner.Start(64);
+        WaitHandle[] wait = _ipScanner.Start(2);
         
         WaitHandle.WaitAll(wait);
         
@@ -78,7 +78,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(4);
+        WaitHandle[] wait = _dbHandler.Start(2);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -18,14 +18,14 @@ public static class HttpClientHelper
         client.DefaultRequestHeaders.Accept.Clear();
         
         HttpResponseMessage? response = null;
-        
+
         try
         {
             response = await client.GetAsync("/");
         }
-        catch
+        catch (Exception e)
         {
-            //
+            Console.WriteLine(e);
         }
         
         if (response is null || !response.IsSuccessStatusCode)
@@ -35,7 +35,7 @@ public static class HttpClientHelper
         
         return await response.Content.ReadAsStringAsync();
     }
-    
+
     public static async Task<bool> HasRobotsTxt(string url, int port)
     {
         using HttpClient client = new();
@@ -57,9 +57,9 @@ public static class HttpClientHelper
         {
             response = await client.GetAsync("/robots.txt");
         }
-        catch
+        catch (Exception e)
         {
-            //
+            Console.WriteLine(e);
         }
         
         return response is not null && response.IsSuccessStatusCode;

Backend/Scripts/GetDomainNamePort443.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-curl -k -vvI -m 10 https://"$1" &> "$2"
+curl -k -vvI -m 10 https://"$1" &> 443Header.txt

Backend/Scripts/GetDomainNamePort80.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 
-curl -k -vvI -m 10 http://"$1" &> "$2"
-#80Header.txt
+curl -k -vvI -m 10 http://"$1" &> 80Header.txt

Models/Handler/DbHandler.cs

@@ -25,7 +25,6 @@ public class DbHandler
     private const string ReadUnfilteredIdsStatement = "SELECT Id FROM Unfiltered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
     private const string ReadFilteredStatement = "SELECT Title2, Url2 FROM Filtered WHERE (Url2 NOT NULL AND Url2 != '') AND (Title2 NOT NULL AND Title2 != '') ORDER BY Url2 DESC;";
     private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
-    private const string ReadFilteredIpStatement = "SELECT Ip FROM Filtered WHERE Ip == @ip ORDER BY Ip DESC LIMIT 1;";
     private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
     private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
     
@@ -366,27 +365,7 @@ public class DbHandler
 
     public bool GetFilteredIp(string ip)
     {
-        using SqliteConnection connection = new(_filteredConnectionString);
-        connection.Open();
-        
-        using SqliteCommand command = new(ReadFilteredIpStatement, connection);
-        command.Parameters.AddWithValue("@ip", ip);
-        
-        using SqliteDataReader reader = command.ExecuteReader();
-        
-        if (!reader.HasRows)
-        {
-            return false;
-        }
-        
-        string ipAddress = "";
-        
-        while (reader.Read())
-        {
-            ipAddress = reader.GetString(0);
-        }
-        
-        return ipAddress == ip;
+        return true;
     }
     
     public List<SearchResult?> GetSearchResults()