Changed GET to HEAD for checking robots txt.

Reviewed-on: #17

Backend/Handler/ContentFilter.cs

@@ -229,7 +229,7 @@ public class ContentFilter
     private void StartProcess(Ip ip, int port)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
-        //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}");
+        
         Process proc = new();
         proc.StartInfo = new()
         {

Backend/Handler/IpScanner.cs

@@ -155,7 +155,7 @@ public class IpScanner
                         
                         if (responseCode != IPStatus.Success)
                         {
-                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode));
+                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
                             continue;
                         }
                         
@@ -163,7 +163,7 @@ public class IpScanner
                         
                         if (ports is { Item1: 0, Item2: 0 })
                         {
-                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip.ToString(), (int)responseCode));
+                            _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
                             continue;
                         }
                         
@@ -203,15 +203,13 @@ public class IpScanner
         scanSettings.Handle!.Set();
     }
     
-    private static Discarded CreateDiscardedQueueItem(string ip, int responseCode)
+    private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
-        Discarded discarded = new()
+        return new()
         {
             Ip = ip,
             ResponseCode = responseCode
         };
-        
-        return discarded;
     }
     
     private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
@@ -224,14 +222,12 @@ public class IpScanner
             Filtered = false
         };
         
-        QueueItem superUnfilteredObject = new()
+        return new()
         {
             Unfiltered = unfiltered,
             Operations = Operations.Insert,
             DbType = DbType.Unfiltered
         };
-        
-        return superUnfilteredObject;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -28,22 +28,22 @@ public class ThreadHandler
     
     public void Start()
     {
-        Thread scanner = new(StartScanner);
+        //Thread scanner = new(StartScanner);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
-        Thread discarded = new(StartDiscardedDbHandler);
+        //Thread discarded = new(StartDiscardedDbHandler);
         Thread communication = new(StartCommunicationHandler);
         
-        scanner.Start();
+        //scanner.Start();
         indexer.Start();
         database.Start();
-        discarded.Start();
+        //discarded.Start();
         communication.Start();
         
-        scanner.Join();
+        //scanner.Join();
         indexer.Join();
         database.Join();
-        discarded.Join();
+        //discarded.Join();
         communication.Join();
     }
     

Backend/Helper/HttpClientHelper.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+
 namespace Backend.Helper;
 
 public static class HttpClientHelper
@@ -16,6 +18,7 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
@@ -54,8 +57,8 @@ public static class HttpClientHelper
         HttpResponseMessage? response = null;
         
         try
-        {
-            response = await client.GetAsync("/robots.txt");
+        {//
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
         {

Models/Handler/DbHandler.cs

@@ -19,7 +19,7 @@ public class DbHandler
     
     private const string InsertStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
     private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2, Description1, Description2, Url1, Url2, ServerType1, ServerType2, RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry, CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm, PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2, ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @title1, @title2, @description1, @description2, @url1, @url2, @serverType1, @serverType2, @robotsTXT1, @robotsTXT2, @httpVersion1, @httpVersion2, @certificateIssuerCountry, @certificateOrganizationName, @ipV6, @tlsVersion, @cipherSuite, @keyExchangeAlgorithm, @publicKeyType1, @publicKeyType2, @publicKeyType3, @acceptEncoding1, @acceptEncoding2, @aLPN, @connection1, @connection2)";
-    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip, ResponseCode) VALUES (@ip, @responseCode)";
+    private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode) VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
     private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte) VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
     
     private const string ReadUnfilteredStatement = "SELECT * FROM Unfiltered WHERE Id = @id;";
@@ -156,9 +156,7 @@ public class DbHandler
                 continue;
             }
             
-            _discardedQueue.TryDequeue(out Discarded? queueItem);
-            
-            if (queueItem is null) { continue; }
+            _discardedQueue.TryDequeue(out Discarded queueItem);
             
             InsertDiscarded(queueItem, connectionString);
         }
@@ -194,7 +192,10 @@ public class DbHandler
         
         using SqliteCommand command = new(InsertIntoDiscarded, connection);
         
-        command.Parameters.AddWithValue("@ip", discarded.Ip);
+        command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
+        command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
+        command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
+        command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
         command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
         _ = command.ExecuteNonQuery();
         
@@ -298,15 +299,12 @@ public class DbHandler
             ip.Ip2 = reader.GetInt32(2);
             ip.Ip3 = reader.GetInt32(3);
             ip.Ip4 = reader.GetInt32(4);
-            //Console.WriteLine(ip + "lmo");
             unfiltered.Port1 = reader.GetInt32(5);
             unfiltered.Port2 = reader.GetInt32(6);
             unfiltered.Filtered = reader.GetBoolean(7);
         }
         
-        //Console.WriteLine(ip + "aaaaa");
         unfiltered.Ip = ip;
-        //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs");
         
         return unfiltered;
     }
@@ -579,7 +577,7 @@ public class DbHandler
     {
         string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
         
-        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
+        const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 TEXT NOT NULL, Ip2 TEXT NOT NULL, Ip3 TEXT NOT NULL, Ip4 TEXT NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
 
         _discardedConnectionStrings.Add(databaseName);
         

Models/Model/Backend/Discarded.cs

@@ -1,7 +1,8 @@
 namespace Models.Model.Backend;
 
-public class Discarded
+public struct Discarded
 {
-    public string Ip { get; set; } = "";
+    public Ip Ip { get; set; }
+    
     public int ResponseCode { get; set; }
 }

RSE.sln.DotSettings.user

@@ -2,8 +2,10 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000006pdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fd17d4b69379a42eb90f15b17ef6c846a5400_003F39_003F1eeed291_003F02000006pdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F096da63313a8436fb75089601a728c765200_003Fbb_003Ff6383783_003F02000007pdb1Low_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000007pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F357247e747614c8297021dd08d79da3d5200_003Ff2_003Fc4ae8c87_003F02000007pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Fpdb2Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F069d64e89a7540319e3e5355db9fd8fa8600_003Fac_003F604d330e_003F0200000Fpdb2Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000010pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F80cf8253b0c2409096d0c0e368300af28200_003Fdd_003F8098fc19_003F02000010pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb4Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F1f6cfdd1e3a14f6390237f6ab98b06af8000_003F0d_003Febdca535_003F02000011pdb4Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000013pdb13Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F8d5d3174bfdc4554bbaac3d5e7c564a29800_003F64_003F42d93916_003F02000013pdb13Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttributes_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9a4cb3a98df697f884fa5f45bc79e4291fa3e948c7f42560e14678e12055e_003FAttributes_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AAttribute_002ECoreCLR_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbb9be2cb8efb72a7d2286fbdd10a293b55b3e2a912f49fac6a7719673268e4b_003FAttribute_002ECoreCLR_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -11,6 +13,7 @@
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5029ca69753a60cb407b1053f5834320dadee066_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ECommunicationObjectFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F99b5d033f8bfb58a6a753ca88a3d956f2228bd48_003FFormatters_002EMessagePack_002EGeneratedMessagePackResolver_002EModels_002EModel_002EExternal_002ESearchResultsFormatter_002Eg_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFuture_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F13cbfe6856867dd1ed4e39575e46d816dae2a146a8ceec8f76b5897f5e0fe_003FFuture_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F60038225224a9318597b3232280390933d3c3fe85d9ed859a7e4825915eb1c_003FHttpClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpContent_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a55e5df9328b51021ba85b8f12ff49eca8772dba9772c0d61d5e28edf255c_003FHttpContent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpMethodAttribute_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F2e4732e712a98f9ac0d080ac68669b3db6d98781e0d5ad89886cb6bcc890b18_003FHttpMethodAttribute_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIFormatterResolver_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc7d856f45ddf907f9face6c2bdba7836cc70fed4bb2b8ebd83ee6917584af3_003FIFormatterResolver_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>