Add stuff

Reviewed-on: #43

Backend/Backend.csproj

@@ -16,6 +16,7 @@
     
     <ItemGroup>
       <PackageReference Include="FuzzySharp" Version="2.0.2" />
+      <PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
       <PackageReference Include="NetMQ" Version="4.0.1.13" />
     </ItemGroup>
     

Backend/Handler/IpScanner.cs

@@ -49,23 +49,11 @@ public class IpScanner
             threadsAmount = 256 / threads;
         }
 
-        WaitHandle[] waitHandle1;
-        WaitHandle[] waitHandle2;
-        
-        // This is jank. But it was to get it production ready.
-        if (threads <= 64)
-        {
-            waitHandle1 = new WaitHandle[threads];
-            waitHandle2 = new WaitHandle[threads];
-        }
-        else
-        {
-            waitHandle1 = new WaitHandle[64];
-            waitHandle2 = new WaitHandle[64];
-        }
+        WaitHandle[] waitHandle = new WaitHandle[64];
         
         int counter = 0;
-        int counter2 = 0;
+        
+        List<WaitHandle[]> waitHandles = [];
         
         for (int i = 0; i < threads; i++)
         {
@@ -79,32 +67,27 @@ public class IpScanner
                 Handle = handle
             };
             
-            if (i < 64)
+            if (counter < 64)
             {
-                waitHandle1[counter] = handle;
+                waitHandle[counter] = handle;
                 counter++;
-            }
-            else
-            {
-                waitHandle2[counter2] = handle;
-                counter2++;
+                
+                Thread f = new (Scan!);
+                f.Start(scanSettings);
+                
+                Console.WriteLine($"Scanner thread ({i}) started");
+                Thread.Sleep(100);
+                
+                continue;
             }
             
-            Thread f = new (Scan!);
-            f.Start(scanSettings);
+            counter = 0;
             
-            Console.WriteLine($"Scanner thread ({i}) started");
-            Thread.Sleep(100);
+            waitHandles.Add(waitHandle);
+            
+            waitHandle = new WaitHandle[64];
         }
         
-        List<WaitHandle[]> waitHandles = new();
-        
-        Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length);
-        Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length);
-        
-        waitHandles.Add(waitHandle1);
-        waitHandles.Add(waitHandle2);
-        
         return waitHandles;
     }
     
@@ -183,7 +166,8 @@ public class IpScanner
                             _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
                             if (address is not null)
                             {
-                                responseCode = ping.Send(address, _timeout, buf, null).Status;
+                                responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
+                                Thread.Sleep(_timeout);
                             }
                         }
                         catch

Backend/Handler/ThreadHandler.cs

@@ -59,7 +59,7 @@ public class ThreadHandler
     {
         Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
         
-        List<WaitHandle[]> wait = _ipScanner.Start(128);
+        List<WaitHandle[]> wait = _ipScanner.Start(256);
 
         for (int i = 0; i < wait.Count; i++)
         {
@@ -101,7 +101,7 @@ public class ThreadHandler
     
     private void StartDiscardedDbHandler()
     {
-        WaitHandle[] wait = _dbHandler.Start(4);
+        WaitHandle[] wait = _dbHandler.Start(5);
         
         WaitHandle.WaitAll(wait);
         

Backend/Helper/HttpClientHelper.cs

@@ -1,4 +1,5 @@
 using System.Text.RegularExpressions;
+using HtmlAgilityPack;
 
 namespace Backend.Helper;
 
@@ -17,11 +18,25 @@ public static partial class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
@@ -86,11 +101,25 @@ public static partial class HttpClientHelper
         
         if (port == 80)
         {
-            client.BaseAddress = new($"http://{url}");
+            try
+            {
+                client.BaseAddress = new($"http://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         else
         {
-            client.BaseAddress = new($"https://{url}");
+            try
+            {
+                client.BaseAddress = new($"https://{url}");
+            }
+            catch
+            {
+                //
+            }
         }
         
         client.DefaultRequestHeaders.Accept.Clear();

Models/Handler/DbHandler.cs

@@ -245,16 +245,6 @@ public class DbHandler
         
         (string absolutePath, string connectionString) = CreateDiscardedDb(discardedDbHandlerSetting.ThreadId);
         
-        // Compress the databases if they're already filled.
-        if (GetDiscardedIndexesForSpecificDb(connectionString) >= 50_000_000 && !_compressing)
-        {
-            _compressing = true;
-            
-            Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
-            
-            _compressing = false;
-        }
-        
         int i = 0;
         
         while (!_stop)
@@ -272,7 +262,13 @@ public class DbHandler
                 
                 i = 0;
                 
-                Compress(discardedDbHandlerSetting.ThreadId, absolutePath, connectionString);
+                InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
+                
+                int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
+                
+                CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
+                
+                DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
                 
                 _compressing = false;
             }
@@ -289,18 +285,6 @@ public class DbHandler
         Console.WriteLine("Discarded DbHandler stopped.");
     }
     
-    private void Compress(int threadNumber, string absolutePath, string connectionString)
-    {
-        InsertCompressedDatabase(threadNumber, GetDiscardedIndexesForSpecificDb(connectionString));
-                
-        int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
-                
-        CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
-                
-        DropAndCreateDiscarded(threadNumber);
-    }
-    
-    
     private void DropAndCreateDiscarded(int threadNumber)
     {
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";

Proxy/Program.cs

@@ -60,7 +60,7 @@ progressApi.MapGet("/", (IMemoryCache memoryCache) =>
     return scanningStatus;
 });
 
-/*
+
 RouteGroupBuilder searchApi = app.MapGroup("/search");
 searchApi.MapGet("/{term}", (string term) =>
 {
@@ -78,7 +78,7 @@ searchApi.MapGet("/{term}", (string term) =>
     
     return JsonSerializer.Deserialize<SearchResults?>(msg);
 });
-*/
+
 app.Run();
 
 [JsonSerializable(typeof(ScanningStatus))]

frontend/app.vue

@@ -9,8 +9,12 @@ useHead({
 </script>
 
 <template>
-  <NuxtLayout>
-    <NuxtPage page-key="static" />
-  </NuxtLayout>
+  <div class="navbar navbar-expand-md navbar-dark bg-dark">
+    <ul>
+      <li><nuxt-link to="/">Search</nuxt-link></li>
+      <li><nuxt-link to="/progress">Progress</nuxt-link></li>
+    </ul>
+  </div>
 
+  <NuxtPage page-key="static" />
 </template>

frontend/nuxt.config.ts

@@ -13,5 +13,9 @@ export default defineNuxtConfig({
     },
   },
 
+  nitro: {
+    static: true,
+  },
+
   modules: ['@nuxtjs/tailwindcss', 'nuxt-purgecss'],
 });

frontend/pages/index.vue

@@ -1,12 +1,15 @@
 <script setup lang="ts">
-const route = useRoute()
+const value = ref<string | null>(null)
+
+// https://nuxt.com/docs/api/components/nuxt-link#nuxtlink
 </script>
 
+
 <template>
   <div class="h-screen flex items-center justify-center">
     <div class="flex space-x-2">
-      <input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
-      <button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
+      <input v-model="value" placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
+      <nuxt-link :to="`/searchResult?search=${value}`" class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</nuxt-link>
     </div>
   </div>
 </template>

frontend/pages/progress.vue

@@ -39,23 +39,51 @@ fetchMyData();
 
 <template>
   <div class="h-screen grid place-items-center">
-    <div v-if="dict">
-      <table class="table-auto border-gray-300">
-        <thead>
-          <tr>
-            <th class="px-4 py-2">Metric</th>
-            <th class="px-4 py-2">Value</th>
-          </tr>
-        </thead>
+    <table class="table-auto border-gray-300">
+      <thead>
+        <tr>
+          <th class="px-4 py-2">Metric</th>
+          <th class="px-4 py-2">Value</th>
+        </tr>
+      </thead>
 
-        <tbody>
-          <tr v-for="d in dict">
-            <td class="px-4 py-2">{{d.description}}</td>
-            <td class="px-4 py-2">{{d.value}}</td>
-          </tr>
-        </tbody>
-      </table>
-    </div>
+      <tbody v-if="dict">
+        <tr v-for="d in dict">
+          <td class="px-4 py-2">{{d.description}}</td>
+          <td class="px-4 py-2">{{d.value}}</td>
+        </tr>
+      </tbody>
+      <tbody v-else>
+      <tr>
+        <td class="px-4 py-2">Percentage of Ipv4 scanned</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total filtered</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Total Discarded</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Amount of Ipv4 left</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Discarded db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Filtered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      <tr>
+        <td class="px-4 py-2">Unfiltered db size</td>
+        <td class="px-4 py-2"></td>
+      </tr>
+      </tbody>
+    </table>
   </div>
 </template>
 

frontend/pages/searchResult.vue

@@ -0,0 +1,8 @@
+<script setup lang="ts">
+const route = useRoute();
+  console.log(route.query);
+</script>
+
+<template>
+  
+</template>

frontend/types/global.d.ts

@@ -6,12 +6,22 @@ interface ProgressType {
     discardedDbSize: bigint;
     filteredDbSize: bigint;
     myDbSize: bigint;
-}
+};
 
 interface ProgressDictionary {
     description: string;
     value: string;
-}
+};
+
+interface SearchResult {
+    title: string;
+    description: string;
+    url: string;
+};
+
+interface SearchResults {
+    results: SearchResult[];
+};
 
 type CacheEntry<T> = {
     data: T;