using System.Diagnostics; namespace Backend.Helper; public static class HttpClientHelper { public static async Task GetHtml(string url, int port) { using HttpClient client = new(); if (port == 80) { client.BaseAddress = new($"http://{url}"); } else { client.BaseAddress = new($"https://{url}"); } client.DefaultRequestHeaders.Accept.Clear(); client.Timeout = TimeSpan.FromSeconds(1); HttpResponseMessage? response; try { response = await client.GetAsync("/"); } catch { return ""; } if (!response.IsSuccessStatusCode) { return ""; } return await response.Content.ReadAsStringAsync(); } public static async Task HasRobotsTxt(string url, int port) { using HttpClient client = new(); if (port == 80) { client.BaseAddress = new($"http://{url}"); } else { client.BaseAddress = new($"https://{url}"); } client.DefaultRequestHeaders.Accept.Clear(); client.Timeout = TimeSpan.FromSeconds(1); HttpResponseMessage? response = null; try {// response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt")); } catch { // } return response is not null && response.IsSuccessStatusCode; } }