owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Made a lot of changes. Enhanced memory usage.

Browse Source
This commit is contained in:
owner
2025-02-16 09:38:00 +01:00
parent 8d659d4261
commit c70b42c9d7
169 changed files with 3628 additions and 1304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Backend/Helper/FilesystemHelper.cs
-43
View File
@@ -1,43 +0,0 @@
using Models.Model.Backend;
namespace Backend.Helper;
public static class FilesystemHelper
{
public static DatabaseSizes GetDatabaseSizes(string basePath)
{
DatabaseSizes databaseSizes = new();
FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
fileInfo = new($"{basePath}/Models/Filtered.db");
databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
return databaseSizes;
}
private static long GetDiscardedDbSizes(string basePath)
{
string folder = $"{basePath}/Models";
string[] files = Directory.GetFiles(folder, "*.db");
long size = 0;
for (int i = 0; i < files.Length; i++)
{
if (!files[i].Contains("Discarded")) continue;
FileInfo fileInfo = new(files[i]);
size += fileInfo.Length;
}
return size;
}
private static double ToSize(this long value, SizeUnits unit)
{
return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
}
}
Backend/Helper/FilterHelper.cs
+3 -41
View File
@@ -2,12 +2,10 @@ using System.Text.RegularExpressions;
namespace Backend.Helper;
public static partial class FilterHelper
public static class FilterHelper
{
// https://stackoverflow.com/a/56116499
private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
private const string TitlePattern = "<title>(.*)</title>";
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
public static void GetDomain(string data, out string result)
{
@@ -42,32 +40,6 @@ public static partial class FilterHelper
result = GetSubstring(data, start, end);
}
public static void GetTitle(string data, out string result)
{
result = "";
Regex titleRegex = MyRegex1();
Match match = titleRegex.Match(data);
if (match.Success)
{
result = match.Groups[1].Value;
}
}
public static void GetDescription(string data, out string result)
{
result = "";
Regex titleRegex = MyRegex2();
Match match = titleRegex.Match(data);
if (match.Success)
{
result = match.Groups[1].Value;
}
}
public static void GetHttpVersion(string data, out string result)
{
result = "";
@@ -198,7 +170,7 @@ public static partial class FilterHelper
case 0:
return;
case 2:
result = $"{temp[1].Trim()} UUHHHMMM";
result = $"{temp[1].Trim()}";
return;
default:
result = temp[2].Trim();
@@ -241,17 +213,7 @@ public static partial class FilterHelper
private static bool ValidateUrl(string url)
{
Regex rgx = MyRegex();
Regex rgx = new(DomainPattern);
return rgx.IsMatch(url);
}
//Generate the RegEx at compile time.
[GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
private static partial Regex MyRegex();
[GeneratedRegex(TitlePattern)]
private static partial Regex MyRegex1();
[GeneratedRegex(DescriptionPattern)]
private static partial Regex MyRegex2();
}
Backend/Helper/HttpClientHelper.cs
-93
View File
@@ -1,5 +1,4 @@
using System.Text.RegularExpressions;
using HtmlAgilityPack;
namespace Backend.Helper;
@@ -7,93 +6,6 @@ public static partial class HttpClientHelper
{
// Reddit, for example, will block the GET request if you don't have a user agent.
private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
private const string TitlePattern = "<title>(.*)</title>";
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
private const string StartHeadTag = "<head>";
private const string EndHeadTag = "</head>";
public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
{
using HttpClient client = new();
if (port == 80)
{
try
{
client.BaseAddress = new($"http://{url}");
}
catch
{
//
}
}
else
{
try
{
client.BaseAddress = new($"https://{url}");
}
catch
{
//
}
}
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
client.Timeout = TimeSpan.FromSeconds(30);
HttpResponseMessage? response;
try
{
response = await client.GetAsync("/");
}
catch
{
return ("", "");
}
if (!response.IsSuccessStatusCode)
{
return ("", "");
}
string html = await response.Content.ReadAsStringAsync();
int firstIndex = 0;
int lastIndex = 0;
if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
{
firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
}
string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
html = "";
string title = "";
string description = "";
Regex titleRegex = TitleRegEx();
Match titleMatch = titleRegex.Match(head);
if (titleMatch.Success)
{
title = titleMatch.Groups[1].Value;
}
Regex descriptionRegex = DexcriptionRegEx();
Match descriptionMatch = descriptionRegex.Match(head);
if (descriptionMatch.Success)
{
description = descriptionMatch.Groups[1].Value;
}
return (title, description);
}
public static async Task<bool> HasRobotsTxt(string url, int port)
{
@@ -139,9 +51,4 @@ public static partial class HttpClientHelper
return response is not null && response.IsSuccessStatusCode;
}
[GeneratedRegex(TitlePattern)]
private static partial Regex TitleRegEx();
[GeneratedRegex(DescriptionPattern)]
private static partial Regex DexcriptionRegEx();
}
Backend/Helper/SearchHelper.cs
-44
View File
@@ -1,44 +0,0 @@
using FuzzySharp;
using Models.Handler;
using Models.Model.External;
namespace Backend.Helper;
public static class SearchHelper
{
public static SearchResults Search(string searchText, DbHandler dbHandler)
{
if (string.IsNullOrWhiteSpace(searchText))
{
SearchResult searchResult = new();
searchResult.Description = "asd";
searchResult.Title = "asd";
searchResult.Url = "asd";
SearchResults searchResults = new();
searchResults.Results =
[
searchResult
];
return searchResults;
}
List<SearchResult?> temp = dbHandler.GetSearchResults();
SearchResults searchResultsList = new();
for (int i = 0; i < temp.Count; i++)
{
if (temp[i] is null) continue;
SearchResult result = new();
if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
result.Url = temp[i]?.Url;
result.Title = temp[i]?.Title;
searchResultsList.Results.Add(result);
}
return searchResultsList;
}
}
Backend/Helper/TcpClientHelper.cs
+2 -1
View File
@@ -1,3 +1,4 @@
using System.Net;
using System.Net.Sockets;
using Models.Model.Backend;
@@ -15,7 +16,7 @@ public static class TcpClientHelper
try
{
socket.Connect(ip.ToString(), ports[i]);
socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
socket.Close();
// If the connection is not successful, update the ports array with 0.
}