Changed GET to HEAD for checking robots txt.

Reviewed-on: #17

Backend/Handler/ContentFilter.cs

@@ -229,7 +229,7 @@ public class ContentFilter
     private void StartProcess(Ip ip, int port)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
-        //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}");
+        
         Process proc = new();
         proc.StartInfo = new()
         {

Backend/Handler/IpScanner.cs

@@ -205,13 +205,11 @@ public class IpScanner
     
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
-        Discarded discarded = new()
+        return new()
         {
             Ip = ip,
             ResponseCode = responseCode
         };
-        
-        return discarded;
     }
     
     private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
@@ -224,14 +222,12 @@ public class IpScanner
             Filtered = false
         };
         
-        QueueItem superUnfilteredObject = new()
+        return new()
         {
             Unfiltered = unfiltered,
             Operations = Operations.Insert,
             DbType = DbType.Unfiltered
         };
-        
-        return superUnfilteredObject;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -28,22 +28,22 @@ public class ThreadHandler
     
     public void Start()
     {
-        Thread scanner = new(StartScanner);
+        //Thread scanner = new(StartScanner);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
-        Thread discarded = new(StartDiscardedDbHandler);
+        //Thread discarded = new(StartDiscardedDbHandler);
         Thread communication = new(StartCommunicationHandler);
         
-        scanner.Start();
+        //scanner.Start();
         indexer.Start();
         database.Start();
-        discarded.Start();
+        //discarded.Start();
         communication.Start();
         
-        scanner.Join();
+        //scanner.Join();
         indexer.Join();
         database.Join();
-        discarded.Join();
+        //discarded.Join();
         communication.Join();
     }
     

Backend/Helper/HttpClientHelper.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+
 namespace Backend.Helper;
 
 public static class HttpClientHelper
@@ -16,6 +18,7 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
@@ -54,8 +57,8 @@ public static class HttpClientHelper
         HttpResponseMessage? response = null;
         
         try
-        {
-            response = await client.GetAsync("/robots.txt");
+        {//
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
         {

Models/Handler/DbHandler.cs

@@ -299,15 +299,12 @@ public class DbHandler
             ip.Ip2 = reader.GetInt32(2);
             ip.Ip3 = reader.GetInt32(3);
             ip.Ip4 = reader.GetInt32(4);
-            //Console.WriteLine(ip + "lmo");
             unfiltered.Port1 = reader.GetInt32(5);
             unfiltered.Port2 = reader.GetInt32(6);
             unfiltered.Filtered = reader.GetBoolean(7);
         }
         
-        //Console.WriteLine(ip + "aaaaa");
         unfiltered.Ip = ip;
-        //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs");
         
         return unfiltered;
     }

RSE.sln.DotSettings.user

@@ -2,8 +2,10 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000006pdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fd17d4b69379a42eb90f15b17ef6c846a5400_003F39_003F1eeed291_003F02000006pdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F096da63313a8436fb75089601a728c765200_003Fbb_003Ff6383783_003F02000007pdb1Low_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F357247e747614c8297021dd08d79da3d5200_003Ff2_003Fc4ae8c87_003F02000007pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Fpdb2Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F069d64e89a7540319e3e5355db9fd8fa8600_003Fac_003F604d330e_003F0200000Fpdb2Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000010pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F80cf8253b0c2409096d0c0e368300af28200_003Fdd_003F8098fc19_003F02000010pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb4Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F1f6cfdd1e3a14f6390237f6ab98b06af8000_003F0d_003Febdca535_003F02000011pdb4Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000013pdb13Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F8d5d3174bfdc4554bbaac3d5e7c564a29800_003F64_003F42d93916_003F02000013pdb13Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttributes_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9a4cb3a98df697f884fa5f45bc79e4291fa3e948c7f42560e14678e12055e_003FAttributes_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttribute_002ECoreCLR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbb9be2cb8efb72a7d2286fbdd10a293b55b3e2a912f49fac6a7719673268e4b_003FAttribute_002ECoreCLR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -11,6 +13,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5029ca69753a60cb407b1053f5834320dadee066_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F99b5d033f8bfb58a6a753ca88a3d956f2228bd48_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F13cbfe6856867dd1ed4e39575e46d816dae2a146a8ceec8f76b5897f5e0fe_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F60038225224a9318597b3232280390933d3c3fe85d9ed859a7e4825915eb1c_003FHttpClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpContent_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a55e5df9328b51021ba85b8f12ff49eca8772dba9772c0d61d5e28edf255c_003FHttpContent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpMethodAttribute_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2e4732e712a98f9ac0d080ac68669b3db6d98781e0d5ad89886cb6bcc890b18_003FHttpMethodAttribute_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIFormatterResolver_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc7d856f45ddf907f9face6c2bdba7836cc70fed4bb2b8ebd83ee6917584af3_003FIFormatterResolver_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>