Changed GET to HEAD for checking robots txt.

Backend/Handler/ContentFilter.cs

@@ -229,7 +229,7 @@ public class ContentFilter
     private void StartProcess(Ip ip, int port)
     {
         string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
-        //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}");
+        
         Process proc = new();
         proc.StartInfo = new()
         {

Backend/Handler/IpScanner.cs

@@ -205,13 +205,11 @@ public class IpScanner
     
     private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
     {
-        Discarded discarded = new()
+        return new()
         {
             Ip = ip,
             ResponseCode = responseCode
         };
-        
-        return discarded;
     }
     
     private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
@@ -224,14 +222,12 @@ public class IpScanner
             Filtered = false
         };
         
-        QueueItem superUnfilteredObject = new()
+        return new()
         {
             Unfiltered = unfiltered,
             Operations = Operations.Insert,
             DbType = DbType.Unfiltered
         };
-        
-        return superUnfilteredObject;
     }
     
     public void Stop()

Backend/Handler/ThreadHandler.cs

@@ -28,22 +28,22 @@ public class ThreadHandler
     
     public void Start()
     {
-        Thread scanner = new(StartScanner);
+        //Thread scanner = new(StartScanner);
         Thread indexer = new(StartContentFilter);
         Thread database = new(StartDbHandler);
-        Thread discarded = new(StartDiscardedDbHandler);
+        //Thread discarded = new(StartDiscardedDbHandler);
         Thread communication = new(StartCommunicationHandler);
         
-        scanner.Start();
+        //scanner.Start();
         indexer.Start();
         database.Start();
-        discarded.Start();
+        //discarded.Start();
         communication.Start();
         
-        scanner.Join();
+        //scanner.Join();
         indexer.Join();
         database.Join();
-        discarded.Join();
+        //discarded.Join();
         communication.Join();
     }
     

Backend/Helper/HttpClientHelper.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+
 namespace Backend.Helper;
 
 public static class HttpClientHelper
@@ -16,6 +18,7 @@ public static class HttpClientHelper
         }
         
         client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(30);
         
         HttpResponseMessage? response = null;
         
@@ -54,8 +57,8 @@ public static class HttpClientHelper
         HttpResponseMessage? response = null;
         
         try
-        {
-            response = await client.GetAsync("/robots.txt");
+        {//
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
         }
         catch
         {

Backend/Sql/CountDuplicatedData.sql

@@ -1,3 +0,0 @@
-SELECT CertificateIssuerCountry, COUNT(*) c FROM Filtered GROUP BY CertificateIssuerCountry HAVING c > 1 ORDER BY c DESC;
-
---SELECT ServerType1, ServerType2 FROM Filtered WHERE (ServerType1 NOT NULL AND ServerType1 != '') AND (ServerType2 NOT NULL AND ServerType2 != '') ORDER BY ServerType2 DESC;

Backend/Sql/DropAndCreateDiscarded.sql

@@ -1,8 +0,0 @@
-DROP TABLE Discarded;
-
-CREATE TABLE "Discarded" (
-	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
-	"ResponseCode"	INTEGER NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
-);

Backend/Sql/DropAndCreateFiltered.sql

@@ -1,37 +0,0 @@
-VACUUM;
-
-DROP TABLE Filtered;
-
-CREATE TABLE "Filtered" (
-	"Id"	INTEGER NOT NULL,
-	"Ip"	TEXT NOT NULL,
-	"Port1"	INTEGER NOT NULL,
-	"Port2"	INTEGER NOT NULL,
-	"Title1"	TEXT NOT NULL,
-	"Title2"	TEXT NOT NULL,
-	"Description1"	TEXT NOT NULL,
-	"Description2"	TEXT NOT NULL,
-	"Url1"	TEXT NOT NULL,
-	"Url2"	TEXT NOT NULL,
-	"ServerType1"	TEXT NOT NULL,
-	"ServerType2"	TEXT NOT NULL,
-	"RobotsTXT1"	TEXT NOT NULL,
-	"RobotsTXT2"	TEXT NOT NULL,
-	"HttpVersion1"	TEXT NOT NULL,
-	"HttpVersion2"	TEXT NOT NULL,
-	"CertificateIssuerCountry"	TEXT NOT NULL,
-	"CertificateOrganizationName"	TEXT NOT NULL,
-	"IpV6"	TEXT NOT NULL,
-	"TlsVersion"	TEXT NOT NULL,
-	"CipherSuite"	TEXT NOT NULL,
-	"KeyExchangeAlgorithm"	TEXT NOT NULL,
-	"PublicKeyType1"	TEXT NOT NULL,
-	"PublicKeyType2"	TEXT NOT NULL,
-	"PublicKeyType3"	TEXT NOT NULL,
-	"AcceptEncoding1"	TEXT NOT NULL,
-	"AcceptEncoding2"	TEXT NOT NULL,
-	"ALPN"	TEXT NOT NULL,
-	"Connection1" TEXT NOT NULL,
-	"Connection2" TEXT NOT NULL,
-	PRIMARY KEY("Id" AUTOINCREMENT)
-)

Backend/Sql/DropAndCreateMyDB.sql

@@ -1,13 +0,0 @@
-DROP TABLE Unfiltered;
-
-CREATE TABLE "Unfiltered" (
-	"Id"	INTEGER NOT NULL,
-	"Ip1"	INTEGER NOT NULL,
-	"Ip2"   INTEGER NOT NULL,
-	"Ip3"   INTEGER NOT NULL,
-	"Ip4"   INTEGER NOT NULL,
-	"Port1"	INTEGER NOT NULL,
-	"Port2"	INTEGER NOT NULL,
-	"Filtered"	INTEGER NOT NULL,
-	CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT)
-)

Backend/Sql/UpdateUnfilteredFilteredRow.sql

@@ -1 +0,0 @@
-UPDATE Unfiltered SET Filtered = 0 WHERE Filtered != 0;