Changed GET to HEAD for checking robots txt.

Models/Handler/DbHandler.cs

@@ -299,15 +299,12 @@ public class DbHandler
             ip.Ip2 = reader.GetInt32(2);
             ip.Ip3 = reader.GetInt32(3);
             ip.Ip4 = reader.GetInt32(4);
-            //Console.WriteLine(ip + "lmo");
             unfiltered.Port1 = reader.GetInt32(5);
             unfiltered.Port2 = reader.GetInt32(6);
             unfiltered.Filtered = reader.GetBoolean(7);
         }
         
-        //Console.WriteLine(ip + "aaaaa");
         unfiltered.Ip = ip;
-        //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs");
         
         return unfiltered;
     }

Models/Sql/CountDuplicatedData.sql

@@ -0,0 +1,3 @@
+SELECT CertificateIssuerCountry, COUNT(*) c FROM Filtered GROUP BY CertificateIssuerCountry HAVING c > 1 ORDER BY c DESC;
+
+--SELECT ServerType1, ServerType2 FROM Filtered WHERE (ServerType1 NOT NULL AND ServerType1 != '') AND (ServerType2 NOT NULL AND ServerType2 != '') ORDER BY ServerType2 DESC;

Models/Sql/DropAndCreateDiscarded.sql

@@ -0,0 +1,8 @@
+DROP TABLE Discarded;
+
+CREATE TABLE "Discarded" (
+	"Id"	INTEGER NOT NULL,
+	"Ip"	TEXT NOT NULL,
+	"ResponseCode"	INTEGER NOT NULL,
+	PRIMARY KEY("Id" AUTOINCREMENT)
+);

Models/Sql/DropAndCreateFiltered.sql

@@ -0,0 +1,37 @@
+VACUUM;
+
+DROP TABLE Filtered;
+
+CREATE TABLE "Filtered" (
+	"Id"	INTEGER NOT NULL,
+	"Ip"	TEXT NOT NULL,
+	"Port1"	INTEGER NOT NULL,
+	"Port2"	INTEGER NOT NULL,
+	"Title1"	TEXT NOT NULL,
+	"Title2"	TEXT NOT NULL,
+	"Description1"	TEXT NOT NULL,
+	"Description2"	TEXT NOT NULL,
+	"Url1"	TEXT NOT NULL,
+	"Url2"	TEXT NOT NULL,
+	"ServerType1"	TEXT NOT NULL,
+	"ServerType2"	TEXT NOT NULL,
+	"RobotsTXT1"	TEXT NOT NULL,
+	"RobotsTXT2"	TEXT NOT NULL,
+	"HttpVersion1"	TEXT NOT NULL,
+	"HttpVersion2"	TEXT NOT NULL,
+	"CertificateIssuerCountry"	TEXT NOT NULL,
+	"CertificateOrganizationName"	TEXT NOT NULL,
+	"IpV6"	TEXT NOT NULL,
+	"TlsVersion"	TEXT NOT NULL,
+	"CipherSuite"	TEXT NOT NULL,
+	"KeyExchangeAlgorithm"	TEXT NOT NULL,
+	"PublicKeyType1"	TEXT NOT NULL,
+	"PublicKeyType2"	TEXT NOT NULL,
+	"PublicKeyType3"	TEXT NOT NULL,
+	"AcceptEncoding1"	TEXT NOT NULL,
+	"AcceptEncoding2"	TEXT NOT NULL,
+	"ALPN"	TEXT NOT NULL,
+	"Connection1" TEXT NOT NULL,
+	"Connection2" TEXT NOT NULL,
+	PRIMARY KEY("Id" AUTOINCREMENT)
+)

Models/Sql/DropAndCreateMyDB.sql

@@ -0,0 +1,13 @@
+DROP TABLE Unfiltered;
+
+CREATE TABLE "Unfiltered" (
+	"Id"	INTEGER NOT NULL,
+	"Ip1"	INTEGER NOT NULL,
+	"Ip2"   INTEGER NOT NULL,
+	"Ip3"   INTEGER NOT NULL,
+	"Ip4"   INTEGER NOT NULL,
+	"Port1"	INTEGER NOT NULL,
+	"Port2"	INTEGER NOT NULL,
+	"Filtered"	INTEGER NOT NULL,
+	CONSTRAINT "PK_Unfiltered" PRIMARY KEY("Id" AUTOINCREMENT)
+)

Models/Sql/UpdateUnfilteredFilteredRow.sql

@@ -0,0 +1 @@
+UPDATE Unfiltered SET Filtered = 0 WHERE Filtered != 0;