Removed Agility pack and changed a few things

Backend/Backend.csproj

@@ -16,7 +16,6 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
-      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/IpScanner.cs

@@ -32,7 +32,7 @@ public class IpScanner
         _discardedQueue = discardedQueue;
         _unfilteredQueue = unfilteredQueue;
         _resumeQueue = resumeQueue;
-        
+
         SetTimeout(64);
     }
     
@@ -48,12 +48,24 @@ public class IpScanner
         {
             threadsAmount = 256 / threads;
         }
-
+        
-        WaitHandle[] waitHandle = new WaitHandle[64];
+        WaitHandle[] waitHandle1;
+        WaitHandle[] waitHandle2;
+        
+        // This is jank. But it was to get it production ready.
+        if (threads <= 64)
+        {
+            waitHandle1 = new WaitHandle[threads];
+            waitHandle2 = new WaitHandle[threads];
+        }
+        else
+        {
+            waitHandle1 = new WaitHandle[64];
+            waitHandle2 = new WaitHandle[64];
+        }
         
         int counter = 0;
-        
+        int counter2 = 0;
-        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -67,27 +79,32 @@ public class IpScanner
                 Handle = handle
             };
             
-            if (counter < 64)
+            if (i < 64)
             {
-                waitHandle[counter] = handle;
+                waitHandle1[counter] = handle;
                 counter++;
-                
+            }
-                Thread f = new (Scan!);
+            else
-                f.Start(scanSettings);
+            {
-                
+                waitHandle2[counter2] = handle;
-                Console.WriteLine($"Scanner thread ({i}) started");
+                counter2++;
-                Thread.Sleep(100);
-                
-                continue;
             }
             
-            counter = 0;
+            Thread f = new (Scan!);
+            f.Start(scanSettings);
             
-            waitHandles.Add(waitHandle);
+            Console.WriteLine($"Scanner thread ({i}) started");
-            
+            Thread.Sleep(100);
-            waitHandle = new WaitHandle[64];
         }
         
+        List<WaitHandle[]> waitHandles = new();
+        
+        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
+        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
+        
+        waitHandles.Add(waitHandle1);
+        waitHandles.Add(waitHandle2);
+        
         return waitHandles;
     }
     
@@ -166,8 +183,7 @@ public class IpScanner
                             _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             if (address is not null)
                             {
-                                responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = ping.Send(address, _timeout, buf, null).Status;
-                                Thread.Sleep(_timeout);
                             }
                         }
                         catch

Backend/Handler/ThreadHandler.cs

@@ -59,8 +59,8 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(256);
+        List<WaitHandle[]> wait = _ipScanner.Start(128);
-
+        
         for (int i = 0; i < wait.Count; i++)
         {
             WaitHandle.WaitAll(wait[i]);
@@ -101,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(5);
+        WaitHandle[] wait = _dbHandler.Start(4);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -1,5 +1,4 @@
 using System.Text.RegularExpressions;
-using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
@@ -18,25 +17,11 @@ public static partial class HttpClientHelper
         
         if (port == 80)
         {
-            try
+            client.BaseAddress = new($"http://{url}");
-            {
-                client.BaseAddress = new($"http://{url}");
-            }
-            catch
-            {
-                //
-            }
         }
         else
         {
-            try
+            client.BaseAddress = new($"https://{url}");
-            {
-                client.BaseAddress = new($"https://{url}");
-            }
-            catch
-            {
-                //
-            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
@@ -101,25 +86,11 @@ public static partial class HttpClientHelper
         
         if (port == 80)
         {
-            try
+            client.BaseAddress = new($"http://{url}");
-            {
-                client.BaseAddress = new($"http://{url}");
-            }
-            catch
-            {
-                //
-            }
         }
         else
         {
-            try
+            client.BaseAddress = new($"https://{url}");
-            {
-                client.BaseAddress = new($"https://{url}");
-            }
-            catch
-            {
-                //
-            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();

Models/Handler/DbHandler.cs

@@ -244,6 +244,16 @@ public class DbHandler
         Console.WriteLine($"Discarded DbHandler started with thread: ({discardedDbHandlerSetting.ThreadId})");
         
         (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
+
+        // Compress the databases if they're already filled.
+        if (GetDiscardedIndexesForSpecificDb(connectionString) >= 50_000_000 && !_compressing)
+        {
+            _compressing = true;
+            
+            Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
+            
+            _compressing = false;
+        }
         
         int i = 0;
         
@@ -262,13 +272,7 @@ public class DbHandler
                 
                 i = 0;
                 
-                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
-                
-                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
                 
                 _compressing = false;
             }
@@ -284,6 +288,18 @@ public class DbHandler
         
         Console.WriteLine("Discarded DbHandler stopped.");
     }
+
+    private void Compress(int threadNumber, string absolutePath, string connectionString)
+    {
+        InsertCompressedDatabase(threadNumber, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+        int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+        CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+        DropAndCreateDiscarded(threadNumber);
+    }
+    
     
     private void DropAndCreateDiscarded(int threadNumber)
     {
@@ -583,7 +599,7 @@ public class DbHandler
     public long GetDiscardedIndexes()
     {
         long rowId = 0;
-
+        
         SqliteConnection connection;
         SqliteCommand command;
         SqliteDataReader reader;

Proxy/Program.cs

@@ -60,7 +60,7 @@ progressApi.MapGet("/", (IMemoryCache memoryCache) =>
     return scanningStatus;
 });
 
-
+/*
 RouteGroupBuilder searchApi = app.MapGroup("/search");
 searchApi.MapGet("/{term}", (string term) =>
 {
@@ -78,7 +78,7 @@ searchApi.MapGet("/{term}", (string term) =>
     
     return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-
+*/
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]

frontend/app.vue

@@ -9,12 +9,8 @@ useHead({
 </script>
 
 <template>
-  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+  <NuxtLayout>
-    <ul>
+    <NuxtPage page-key="static" />
-      <li><nuxt-link to="/">Search</nuxt-link></li>
+  </NuxtLayout>
-      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
-    </ul>
-  </div>
 
-  <NuxtPage page-key="static" />
 </template>

frontend/nuxt.config.ts

@@ -13,9 +13,5 @@ export default defineNuxtConfig({
     },
   },
 
-  nitro: {
-    static: true,
-  },
-
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
 });

frontend/pages/index.vue

@@ -1,15 +1,12 @@
 <script setup lang="ts">
-const value = ref<string | null>(null)
+const route = useRoute()
-
-// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
 </script>
 
-
 <template>
   <div class="h-screen flex items-center justify-center">
     <div class="flex space-x-2">
-      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
-      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
+      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
     </div>
   </div>
 </template>

frontend/pages/progress.vue

@@ -39,51 +39,23 @@ fetchMyData();
 
 <template>
   <div class="h-screen grid place-items-center">
-    <table class="table-auto border-gray-300">
+    <div v-if="dict">
-      <thead>
+      <table class="table-auto border-gray-300">
-        <tr>
+        <thead>
-          <th class="px-4 py-2">Metric</th>
+          <tr>
-          <th class="px-4 py-2">Value</th>
+            <th class="px-4 py-2">Metric</th>
-        </tr>
+            <th class="px-4 py-2">Value</th>
-      </thead>
+          </tr>
+        </thead>
 
-      <tbody v-if="dict">
+        <tbody>
-        <tr v-for="d in dict">
+          <tr v-for="d in dict">
-          <td class="px-4 py-2">{{d.description}}</td>
+            <td class="px-4 py-2">{{d.description}}</td>
-          <td class="px-4 py-2">{{d.value}}</td>
+            <td class="px-4 py-2">{{d.value}}</td>
-        </tr>
+          </tr>
-      </tbody>
+        </tbody>
-      <tbody v-else>
+      </table>
-      <tr>
+    </div>
-        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Total filtered</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Total Discarded</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Amount of Ipv4 left</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Discarded db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Filtered db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      <tr>
-        <td class="px-4 py-2">Unfiltered db size</td>
-        <td class="px-4 py-2"></td>
-      </tr>
-      </tbody>
-    </table>
   </div>
 </template>
 

frontend/pages/searchResult.vue

@@ -1,8 +0,0 @@
-<script setup lang="ts">
-const route = useRoute();
-  console.log(route.query);
-</script>
-
-<template>
-  
-</template>

frontend/types/global.d.ts

@@ -6,22 +6,12 @@ interface ProgressType {
     discardedDbSize: bigint;
     filteredDbSize: bigint;
     myDbSize: bigint;
-};
+}
 
 interface ProgressDictionary {
     description: string;
     value: string;
-};
+}
-
-interface SearchResult {
-    title: string;
-    description: string;
-    url: string;
-};
-
-interface SearchResults {
-    results: SearchResult[];
-};
 
 type CacheEntry<T> = {
     data: T;