Fixed duplicate insertions into the filtered database.

2024-11-27 15:53:00 +01:00
parent 7ad691c439
commit f02203c14c
9 changed files with 39 additions and 18 deletions
@@ -18,14 +18,14 @@ public static class HttpClientHelper
        client.DefaultRequestHeaders.Accept.Clear();
        
        HttpResponseMessage? response = null;
-
+        
        try
        {
            response = await client.GetAsync("/");
        }
-        catch (Exception e)
+        catch
        {
-            Console.WriteLine(e);
+            //
        }
        
        if (response is null || !response.IsSuccessStatusCode)
@@ -35,7 +35,7 @@ public static class HttpClientHelper
        
        return await response.Content.ReadAsStringAsync();
    }
-
+    
    public static async Task<bool> HasRobotsTxt(string url, int port)
    {
        using HttpClient client = new();
@@ -57,9 +57,9 @@ public static class HttpClientHelper
        {
            response = await client.GetAsync("/robots.txt");
        }
-        catch (Exception e)
+        catch
        {
-            Console.WriteLine(e);
+            //
        }
        
        return response is not null && response.IsSuccessStatusCode;