RSE/Backend/Helper/HttpClientHelper.cs

namespace Backend.Helper;

public static class HttpClientHelper
{
    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
    
    public static async Task<string> GetHtml(string url, int port)
    {
        using HttpClient client = new();
        
        if (port == 80)
        {
            client.BaseAddress = new($"http://{url}");
        }
        else
        {
            client.BaseAddress = new($"https://{url}");
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response;
        
        try
        {
            response = await client.GetAsync("/");
        }
        catch
        {
            return "";
        }
        
        if (!response.IsSuccessStatusCode)
        {
            return "";
        }
        
        return await response.Content.ReadAsStringAsync();
    }
    
    public static async Task<bool> HasRobotsTxt(string url, int port)
    {
        using HttpClient client = new();
        
        if (port == 80)
        {
            client.BaseAddress = new($"http://{url}");
        }
        else
        {
            client.BaseAddress = new($"https://{url}");
        }
        
        client.DefaultRequestHeaders.Accept.Clear();
        client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
        client.Timeout = TimeSpan.FromSeconds(30);
        
        HttpResponseMessage? response = null;
        
        try
        {
            response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
        }
        catch
        {
            //
        }
        
        return response is not null && response.IsSuccessStatusCode;
    }
}