From 3034e66126917fe1f60b5d29aac16eae5db64f03 Mon Sep 17 00:00:00 2001 From: Rasmus Date: Fri, 29 Nov 2024 11:02:51 +0100 Subject: [PATCH] Changed GET to HEAD for checking robots txt. --- Backend/Handler/ContentFilter.cs | 2 +- Backend/Handler/IpScanner.cs | 8 ++------ Backend/Handler/ThreadHandler.cs | 12 ++++++------ Backend/Helper/HttpClientHelper.cs | 7 +++++-- Models/Handler/DbHandler.cs | 3 --- {Backend => Models}/Sql/CountDuplicatedData.sql | 0 {Backend => Models}/Sql/DropAndCreateDiscarded.sql | 0 {Backend => Models}/Sql/DropAndCreateFiltered.sql | 0 {Backend => Models}/Sql/DropAndCreateMyDB.sql | 0 .../Sql/UpdateUnfilteredFilteredRow.sql | 0 RSE.sln.DotSettings.user | 3 +++ 11 files changed, 17 insertions(+), 18 deletions(-) rename {Backend => Models}/Sql/CountDuplicatedData.sql (100%) rename {Backend => Models}/Sql/DropAndCreateDiscarded.sql (100%) rename {Backend => Models}/Sql/DropAndCreateFiltered.sql (100%) rename {Backend => Models}/Sql/DropAndCreateMyDB.sql (100%) rename {Backend => Models}/Sql/UpdateUnfilteredFilteredRow.sql (100%) diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 0284a9a..85f09df 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -229,7 +229,7 @@ public class ContentFilter private void StartProcess(Ip ip, int port) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; - //Console.WriteLine($"{ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4}"); + Process proc = new(); proc.StartInfo = new() { diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index b175b9c..c470b39 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -205,13 +205,11 @@ public class IpScanner private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) { - Discarded discarded = new() + return new() { Ip = ip, ResponseCode = responseCode }; - - return discarded; } private static QueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) @@ -224,14 +222,12 @@ public class IpScanner Filtered = false }; - QueueItem superUnfilteredObject = new() + return new() { Unfiltered = unfiltered, Operations = Operations.Insert, DbType = DbType.Unfiltered }; - - return superUnfilteredObject; } public void Stop() diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 82316ef..a16d2ef 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -28,22 +28,22 @@ public class ThreadHandler public void Start() { - Thread scanner = new(StartScanner); + //Thread scanner = new(StartScanner); Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); - Thread discarded = new(StartDiscardedDbHandler); + //Thread discarded = new(StartDiscardedDbHandler); Thread communication = new(StartCommunicationHandler); - scanner.Start(); + //scanner.Start(); indexer.Start(); database.Start(); - discarded.Start(); + //discarded.Start(); communication.Start(); - scanner.Join(); + //scanner.Join(); indexer.Join(); database.Join(); - discarded.Join(); + //discarded.Join(); communication.Join(); } diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs index 3e1b2a3..0d8b36b 100644 --- a/Backend/Helper/HttpClientHelper.cs +++ b/Backend/Helper/HttpClientHelper.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; + namespace Backend.Helper; public static class HttpClientHelper @@ -16,6 +18,7 @@ public static class HttpClientHelper } client.DefaultRequestHeaders.Accept.Clear(); + client.Timeout = TimeSpan.FromSeconds(30); HttpResponseMessage? response = null; @@ -54,8 +57,8 @@ public static class HttpClientHelper HttpResponseMessage? response = null; try - { - response = await client.GetAsync("/robots.txt"); + {// + response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt")); } catch { diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 248b0d7..63754ee 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -299,15 +299,12 @@ public class DbHandler ip.Ip2 = reader.GetInt32(2); ip.Ip3 = reader.GetInt32(3); ip.Ip4 = reader.GetInt32(4); - //Console.WriteLine(ip + "lmo"); unfiltered.Port1 = reader.GetInt32(5); unfiltered.Port2 = reader.GetInt32(6); unfiltered.Filtered = reader.GetBoolean(7); } - //Console.WriteLine(ip + "aaaaa"); unfiltered.Ip = ip; - //Console.WriteLine(unfiltered.Ip + "adfgdgfdgfsdfgs"); return unfiltered; } diff --git a/Backend/Sql/CountDuplicatedData.sql b/Models/Sql/CountDuplicatedData.sql similarity index 100% rename from Backend/Sql/CountDuplicatedData.sql rename to Models/Sql/CountDuplicatedData.sql diff --git a/Backend/Sql/DropAndCreateDiscarded.sql b/Models/Sql/DropAndCreateDiscarded.sql similarity index 100% rename from Backend/Sql/DropAndCreateDiscarded.sql rename to Models/Sql/DropAndCreateDiscarded.sql diff --git a/Backend/Sql/DropAndCreateFiltered.sql b/Models/Sql/DropAndCreateFiltered.sql similarity index 100% rename from Backend/Sql/DropAndCreateFiltered.sql rename to Models/Sql/DropAndCreateFiltered.sql diff --git a/Backend/Sql/DropAndCreateMyDB.sql b/Models/Sql/DropAndCreateMyDB.sql similarity index 100% rename from Backend/Sql/DropAndCreateMyDB.sql rename to Models/Sql/DropAndCreateMyDB.sql diff --git a/Backend/Sql/UpdateUnfilteredFilteredRow.sql b/Models/Sql/UpdateUnfilteredFilteredRow.sql similarity index 100% rename from Backend/Sql/UpdateUnfilteredFilteredRow.sql rename to Models/Sql/UpdateUnfilteredFilteredRow.sql diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index 738137f..8d56886 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -2,8 +2,10 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -11,6 +13,7 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded