Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 owner
|
bb8f55a4f1 |
@@ -16,7 +16,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FuzzySharp" Version="2.0.2" />
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
|
||||
<PackageReference Include="NetMQ" Version="4.0.1.13" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -161,37 +161,41 @@ public class ContentFilter
|
||||
|
||||
for (int i = 0; i < ports.Length; i++)
|
||||
{
|
||||
string? html = "";
|
||||
|
||||
if (ports[i] == 80)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url1)) continue;
|
||||
|
||||
|
||||
try
|
||||
{
|
||||
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
|
||||
title1 = temp.Item1;
|
||||
description1 = temp.Item2;
|
||||
html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult();
|
||||
}
|
||||
catch
|
||||
catch (Exception e)
|
||||
{
|
||||
//
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url2)) continue;
|
||||
|
||||
|
||||
try
|
||||
{
|
||||
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
|
||||
title2 = temp.Item1;
|
||||
description2 = temp.Item2;
|
||||
html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult();
|
||||
}
|
||||
catch
|
||||
catch (Exception e)
|
||||
{
|
||||
//
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(html)) continue;
|
||||
|
||||
if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); }
|
||||
if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); }
|
||||
if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); }
|
||||
if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); }
|
||||
if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
|
||||
if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
|
||||
}
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Backend.Helper;
|
||||
|
||||
public static partial class HttpClientHelper
|
||||
public static class HttpClientHelper
|
||||
{
|
||||
// Reddit, for example, will block the GET request if you don't have a user agent.
|
||||
private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
|
||||
private const string TitlePattern = "<title>(.*)</title>";
|
||||
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
|
||||
private const string StartHeadTag = "<head>";
|
||||
private const string EndHeadTag = "</head>";
|
||||
|
||||
public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
|
||||
public static async Task<string> GetHtml(string url, int port)
|
||||
{
|
||||
using HttpClient client = new();
|
||||
|
||||
@@ -26,8 +18,7 @@ public static partial class HttpClientHelper
|
||||
}
|
||||
|
||||
client.DefaultRequestHeaders.Accept.Clear();
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
client.Timeout = TimeSpan.FromSeconds(1);
|
||||
|
||||
HttpResponseMessage? response;
|
||||
|
||||
@@ -37,48 +28,15 @@ public static partial class HttpClientHelper
|
||||
}
|
||||
catch
|
||||
{
|
||||
return ("", "");
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return ("", "");
|
||||
}
|
||||
|
||||
string html = await response.Content.ReadAsStringAsync();
|
||||
|
||||
int firstIndex = 0;
|
||||
int lastIndex = 0;
|
||||
|
||||
if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
|
||||
{
|
||||
firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
|
||||
lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
|
||||
return "";
|
||||
}
|
||||
|
||||
string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
|
||||
html = "";
|
||||
|
||||
string title = "";
|
||||
string description = "";
|
||||
|
||||
Regex titleRegex = TitleRegEx();
|
||||
Match titleMatch = titleRegex.Match(head);
|
||||
|
||||
if (titleMatch.Success)
|
||||
{
|
||||
title = titleMatch.Groups[1].Value;
|
||||
}
|
||||
|
||||
Regex descriptionRegex = DexcriptionRegEx();
|
||||
Match descriptionMatch = descriptionRegex.Match(head);
|
||||
|
||||
if (descriptionMatch.Success)
|
||||
{
|
||||
description = descriptionMatch.Groups[1].Value;
|
||||
}
|
||||
|
||||
return (title, description);
|
||||
return await response.Content.ReadAsStringAsync();
|
||||
}
|
||||
|
||||
public static async Task<bool> HasRobotsTxt(string url, int port)
|
||||
@@ -95,13 +53,12 @@ public static partial class HttpClientHelper
|
||||
}
|
||||
|
||||
client.DefaultRequestHeaders.Accept.Clear();
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
|
||||
client.Timeout = TimeSpan.FromSeconds(30);
|
||||
client.Timeout = TimeSpan.FromSeconds(1);
|
||||
|
||||
HttpResponseMessage? response = null;
|
||||
|
||||
try
|
||||
{
|
||||
{//
|
||||
response = await client.SendAsync(new(HttpMethod.Head, "/robots.txt"));
|
||||
}
|
||||
catch
|
||||
@@ -111,9 +68,4 @@ public static partial class HttpClientHelper
|
||||
|
||||
return response is not null && response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
[GeneratedRegex(TitlePattern)]
|
||||
private static partial Regex TitleRegEx();
|
||||
[GeneratedRegex(DescriptionPattern)]
|
||||
private static partial Regex DexcriptionRegEx();
|
||||
}
|
||||
+22
-14
@@ -2,7 +2,6 @@ using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using AspNetCoreRateLimit;
|
||||
using Microsoft.AspNetCore.HttpOverrides;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Models.Model.External;
|
||||
using NetMQ;
|
||||
using NetMQ.Sockets;
|
||||
@@ -20,7 +19,23 @@ builder.Services.AddCors(options =>
|
||||
options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
|
||||
});
|
||||
|
||||
builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
|
||||
// Add memory cache and rate limiting services
|
||||
builder.Services.AddMemoryCache();
|
||||
builder.Services.Configure<IpRateLimitOptions>(options =>
|
||||
{
|
||||
options.GeneralRules =
|
||||
[
|
||||
new()
|
||||
{
|
||||
Endpoint = "*", // Apply to all endpoints
|
||||
Period = "10s", // Rate limiting window of 10 second
|
||||
Limit = 5 // Maximum 5 requests per 10 seconds
|
||||
}
|
||||
];
|
||||
});
|
||||
|
||||
builder.Services.AddInMemoryRateLimiting();
|
||||
builder.Services.AddSingleton<IRateLimitConfiguration, RateLimitConfiguration>();
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
|
||||
@@ -29,17 +44,14 @@ app.UseForwardedHeaders(new()
|
||||
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
|
||||
});
|
||||
|
||||
|
||||
app.UseIpRateLimiting(); // Apply IP-based rate limiting middleware
|
||||
|
||||
app.UseCors(myAllowSpecificOrigins);
|
||||
|
||||
RouteGroupBuilder progressApi = app.MapGroup("/progress");
|
||||
progressApi.MapGet("/", (IMemoryCache memoryCache) =>
|
||||
progressApi.MapGet("/", () =>
|
||||
{
|
||||
const string cacheKey = "progress_status";
|
||||
if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
|
||||
{
|
||||
return scanningStatus;
|
||||
}
|
||||
|
||||
CommunicationObject communicationObject = new()
|
||||
{
|
||||
Command = CommunicationCommand.GetScanningProgress
|
||||
@@ -53,11 +65,7 @@ progressApi.MapGet("/", (IMemoryCache memoryCache) =>
|
||||
byte[] msg = client.ReceiveFrameBytes();
|
||||
client.Close();
|
||||
|
||||
scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
|
||||
|
||||
memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
|
||||
|
||||
return scanningStatus;
|
||||
return JsonSerializer.Deserialize<ScanningStatus>(msg);
|
||||
});
|
||||
|
||||
/*
|
||||
|
||||
@@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Shared", "Shared\Shared.csproj", "{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -30,5 +32,9 @@ Global
|
||||
{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DEB1411C-F45A-40DA-92F8-D9B9929DBA5B}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
|
||||
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Shared;
|
||||
|
||||
public class Class1
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
+5
-1
@@ -4,7 +4,11 @@ useHead({
|
||||
meta: [
|
||||
{ name: 'description', content: 'Just a search engine for fun' },
|
||||
{ name: 'lang', content: 'en'}
|
||||
]
|
||||
],
|
||||
|
||||
link: [
|
||||
{ rel: 'icon', type: 'image/jpeg', href: '/favicon.jpg' }
|
||||
],
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
@@ -3,10 +3,9 @@ const route = useRoute()
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="h-screen flex items-center justify-center">
|
||||
<div class="flex space-x-2">
|
||||
<input placeholder=".NET C#" class="text-center bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg p-2.5 w-192"/>
|
||||
<button class="text-gray-900 bg-gray-50 border border-gray-300 px-4 py-2 rounded-lg">Search</button>
|
||||
</div>
|
||||
<div>
|
||||
<h1>Nuxt Routing set up successfully!</h1>
|
||||
<p>Current route: {{ route.path }}</p>
|
||||
<a href="https://nuxt.com/docs/getting-started/routing" target="_blank">Learn more about Nuxt Routing</a>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -21,7 +21,6 @@ const fetchMyData = async () => {
|
||||
},
|
||||
5 // Cache max age in seconds
|
||||
);
|
||||
|
||||
} catch (err) {
|
||||
error.value = (err as Error).message;
|
||||
} finally {
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 885 B |
@@ -9,13 +9,7 @@ export default {
|
||||
"./error.vue",
|
||||
],
|
||||
theme: {
|
||||
extend: {
|
||||
width: {
|
||||
'128': '32rem',
|
||||
'192' : '48rem',
|
||||
'256': '64rem',
|
||||
}
|
||||
},
|
||||
extend: {},
|
||||
},
|
||||
plugins: [],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user