Misc changes such as performance tuning, added support for more than 64 threads for the scanner.

Backend/Backend.csproj

@@ -6,8 +6,8 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
 
-        <Platform>x64</Platform>
-        <Optimize>true</Optimize>
+        <!--<Platform>x64</Platform>-->
+        <Optimize>false</Optimize>
     </PropertyGroup>
     
     <ItemGroup>

Backend/Handler/ContentFilter.cs

@@ -161,19 +161,33 @@ public class ContentFilter
         
         for (int i = 0; i < ports.Length; i++)
         {
-            string? html;
+            string? html = "";
             
             if (ports[i] == 80)
             {
                 if (string.IsNullOrWhiteSpace(url1)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
             }
             else
             {
                 if (string.IsNullOrWhiteSpace(url2)) continue;
-                
-                html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result;
+
+                try
+                {
+                    html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                }
             }
             
             if (string.IsNullOrWhiteSpace(html)) continue;
@@ -182,8 +196,8 @@ public class ContentFilter
             if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
             if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
             if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
-            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; }
-            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; }
+            if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
+            if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
         }
         
         Filtered siteData = new()

Backend/Handler/IpScanner.cs

@@ -41,15 +41,30 @@ public class IpScanner
         _timeout = milliseconds;
     }
     
-    public WaitHandle[] Start(int threads)
+    public List<WaitHandle[]> Start(int threads)
     {
         int threadsAmount = 0;
         if (threads % 2 == 0)
         {
             threadsAmount = 256 / threads;
         }
+
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
         
-        WaitHandle[] waitHandles = new WaitHandle[threads];
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
+        
+        int counter = 0;
+        int counter2 = 0;
         
         for (int i = 0; i < threads; i++)
         {
@@ -62,16 +77,33 @@ public class IpScanner
                 ThreadNumber = i,
                 Handle = handle
             };
-            
-            waitHandles[i] = handle;
+
+            if (i < 64)
+            {
+                waitHandle1[counter] = handle;
+                counter++;
+            }
+            else
+            {
+                waitHandle2[counter2] = handle;
+                counter2++;
+            }
             
             Thread f = new (Scan!);
             f.Start(scanSettings);
             
             Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(1000);
+            Thread.Sleep(100);
         }
         
+        List<WaitHandle[]> waitHandles = new();
+        
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
+        
+        waitHandles.Add(waitHandle1);
+        waitHandles.Add(waitHandle2);
+        
         return waitHandles;
     }
     
@@ -193,7 +225,8 @@ public class IpScanner
                 resumeObject.FirstByte = i;
                 break;
             }
-            //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
+            
+            Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
         }
         
         _resumeQueue.Enqueue(resumeObject);

Backend/Handler/ThreadHandler.cs

@@ -59,9 +59,12 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        WaitHandle[] wait = _ipScanner.Start(64);
-        
-        WaitHandle.WaitAll(wait);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
+
+        for (int i = 0; i < wait.Count; i++)
+        {
+            WaitHandle.WaitAll(wait[i]);
+        }
         
         Console.WriteLine("Scanner finished");
         
@@ -70,6 +73,8 @@ public class ThreadHandler
     
     private void StartContentFilter()
     {
+        Thread.Sleep(5000);
+        
         WaitHandle[] wait = _contentFilter.Start();
         
         WaitHandle.WaitAll(wait);
@@ -83,12 +88,12 @@ public class ThreadHandler
     {
         _dbHandler.UnfilteredDbHandler();
     }
-
+    
     private void StartFilteredDbHandler()
     {
         _dbHandler.FilteredDbHandler();
     }
-
+    
     private void StartResumeDbHandler()
     {
         _dbHandler.ResumeDbHandler();
@@ -96,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(2);
+        WaitHandle[] wait = _dbHandler.Start(4);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -18,9 +18,9 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
-        client.Timeout = TimeSpan.FromSeconds(30);
+        client.Timeout = TimeSpan.FromSeconds(1);
         
-        HttpResponseMessage? response = null;
+        HttpResponseMessage? response;
         
         try
         {
@@ -28,10 +28,10 @@ public static class HttpClientHelper
         }
         catch
         {
-            //
+            return "";
         }
         
-        if (response is null || !response.IsSuccessStatusCode)
+        if (!response.IsSuccessStatusCode)
         {
             return "";
         }
@@ -53,6 +53,7 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(1);
         
         HttpResponseMessage? response = null;