Title and description extraction now only look in the head section of the HTML document

2024-12-23 14:09:51 +01:00
parent 580d687f62
commit e9fd738f1f
8 changed files with 63 additions and 38 deletions
@@ -1,10 +1,18 @@
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
+
 namespace Backend.Helper;

-public static class HttpClientHelper
+public static partial class HttpClientHelper
 {
+    // Reddit, for example, will block the GET request if you don't have a user agent.
    private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+    private const string TitlePattern = "<title>(.*)</title>";
+    private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
+    private const string StartHeadTag = "<head>";
+    private const string EndHeadTag = "</head>";
    
-    public static async Task<string> GetHtml(string url, int port)
+    public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
    {
        using HttpClient client = new();
        
@@ -29,15 +37,48 @@ public static class HttpClientHelper
        }
        catch
        {
-            return "";
+            return ("", "");
        }
        
        if (!response.IsSuccessStatusCode)
        {
-            return "";
+            return ("", "");
+        }
+
+        string html = await response.Content.ReadAsStringAsync();
+        
+        int firstIndex = 0;
+        int lastIndex = 0;
+        
+        if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
+        {
+            firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
+            lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
        }
        
-        return await response.Content.ReadAsStringAsync();
+        string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
+        html = "";
+        
+        string title = "";
+        string description = "";
+        
+        Regex titleRegex = TitleRegEx();
+        Match titleMatch = titleRegex.Match(head);
+        
+        if (titleMatch.Success)
+        {
+            title = titleMatch.Groups[1].Value;
+        }
+        
+        Regex descriptionRegex = DexcriptionRegEx();
+        Match descriptionMatch = descriptionRegex.Match(head);
+        
+        if (descriptionMatch.Success)
+        {
+            description = descriptionMatch.Groups[1].Value;
+        }
+        
+        return (title, description);
    }
    
    public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -70,4 +111,9 @@ public static class HttpClientHelper
        
        return response is not null && response.IsSuccessStatusCode;
    }
+
+    [GeneratedRegex(TitlePattern)]
+    private static partial Regex TitleRegEx();
+    [GeneratedRegex(DescriptionPattern)]
+    private static partial Regex DexcriptionRegEx();
 }