Changed GET to HEAD for checking robots txt.

2024-11-29 11:02:51 +01:00
parent 701ffff27e
commit 3034e66126
11 changed files with 17 additions and 18 deletions
@@ -1,3 +1,5 @@
+using System.Diagnostics;
+
 namespace Backend.Helper;

 public static class HttpClientHelper
@@ -16,6 +18,7 @@ public static class HttpClientHelper
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
+        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response = null;
        
@@ -54,8 +57,8 @@ public static class HttpClientHelper
        HttpResponseMessage? response = null;
        
        try
-        {
-            response = await client.GetAsync("/robots.txt");
+        {//
+            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
        {