owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Title and description extraction now only look in the head section of the HTML document

Browse Source
This commit is contained in:
owner
2024-12-23 14:09:51 +01:00
parent 580d687f62
commit e9fd738f1f
8 changed files with 63 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Backend/Backend.csproj
+1
View File
@@ -16,6 +16,7 @@
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>
Backend/Handler/ContentFilter.cs
+8 -12
View File
@@ -161,15 +161,15 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
string? html = "";
if (ports[i] == 80)
{
if (string.IsNullOrWhiteSpace(url1)) continue;
try
{
html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
title1 = temp.Item1;
description1 = temp.Item2;
}
catch
{
@@ -179,10 +179,12 @@ public class ContentFilter
else
{
if (string.IsNullOrWhiteSpace(url2)) continue;
try
{
html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
title2 = temp.Item1;
description2 = temp.Item2;
}
catch
{
@@ -190,12 +192,6 @@ public class ContentFilter
}
}
if (string.IsNullOrWhiteSpace(html)) continue;
if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
}
Backend/Handler/ThreadHandler.cs
+1 -1
View File
@@ -59,7 +59,7 @@ public class ThreadHandler
{
Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
List<WaitHandle[]> wait = _ipScanner.Start(32);
List<WaitHandle[]> wait = _ipScanner.Start(128);
for (int i = 0; i < wait.Count; i++)
{
Backend/Helper/HttpClientHelper.cs
+51 -5
View File
@@ -1,10 +1,18 @@
using System.Text.RegularExpressions;
using HtmlAgilityPack;
namespace Backend.Helper;
public static class HttpClientHelper
public static partial class HttpClientHelper
{
// Reddit, for example, will block the GET request if you don't have a user agent.
private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
private const string TitlePattern = "<title>(.*)</title>";
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
private const string StartHeadTag = "<head>";
private const string EndHeadTag = "</head>";
public static async Task<string> GetHtml(string url, int port)
public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
{
using HttpClient client = new();
@@ -29,15 +37,48 @@ public static class HttpClientHelper
}
catch
{
return "";
return ("", "");
}
if (!response.IsSuccessStatusCode)
{
return "";
return ("", "");
}
string html = await response.Content.ReadAsStringAsync();
int firstIndex = 0;
int lastIndex = 0;
if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
{
firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
}
return await response.Content.ReadAsStringAsync();
string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
html = "";
string title = "";
string description = "";
Regex titleRegex = TitleRegEx();
Match titleMatch = titleRegex.Match(head);
if (titleMatch.Success)
{
title = titleMatch.Groups[1].Value;
}
Regex descriptionRegex = DexcriptionRegEx();
Match descriptionMatch = descriptionRegex.Match(head);
if (descriptionMatch.Success)
{
description = descriptionMatch.Groups[1].Value;
}
return (title, description);
}
public static async Task<bool> HasRobotsTxt(string url, int port)
@@ -70,4 +111,9 @@ public static class HttpClientHelper
return response is not null && response.IsSuccessStatusCode;
}
[GeneratedRegex(TitlePattern)]
private static partial Regex TitleRegEx();
[GeneratedRegex(DescriptionPattern)]
private static partial Regex DexcriptionRegEx();
}