owner/RSE
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request 'Analyzer' (#46) from Analyzer into main

Browse Source 
Reviewed-on: #46
This commit is contained in:
Rasmus Rasmussen 2025-04-05 09:30:20 +00:00
commit ebf5c11fdf
48 changed files with 1101 additions and 1296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,6 +7,7 @@
**/**/bin/*
**/obj/*
**/**/obj/*
*.gz
# User-specific stuff
.idea/**/workspace.xml

14
Analyze/Analyze.csproj Normal file
View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="CompressedDatabases\" />
</ItemGroup>
</Project>

3
Analyze/Program.cs Normal file
View File

@ -0,0 +1,3 @@
// See https://aka.ms/new-console-template for more information
Console.WriteLine("Hello, World!");

13
Backend/Backend.csproj
View File

@ -2,22 +2,15 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<!--<Platform>x64</Platform>-->
<Optimize>false</Optimize>
<PublishAot>false</PublishAot>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Models\Models.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FuzzySharp" Version="2.0.2" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.71" />
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>
</Project>

201
Backend/Handler/Communication.cs
View File

@ -1,201 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Backend.Helper;
using Models.Handler;
using Models.Model.Backend;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
namespace Backend.Handler;
public class Communication
{
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private bool _isRunning = true;
private string _basePath;
public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath)
{
_dbHandler = dbHandler;
_threadHandler = threadHandler;
_ipScanner = ipScanner;
_contentFilter = contentFilter;
_basePath = basePath;
}
public WaitHandle[] Start()
{
WaitHandle[] waitHandles = new WaitHandle[1];
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
waitHandles[0] = handle;
Thread thread = new(Server!);
thread.Start(handle);
return waitHandles;
}
private void Server(object obj)
{
using ResponseSocket rep = new();
//rep.Options.IPv4Only = true;
rep.Bind("tcp://127.0.0.1:5556");
while (_isRunning)
{
byte[] message = rep.ReceiveFrameBytes();
CommunicationObject? communicationObject = JsonSerializer.Deserialize<CommunicationObject>(message);
//rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
if (communicationObject is null)
{
continue;
}
OnServerOnReceiveReady(communicationObject, rep);
}
((EventWaitHandle) obj).Set();
}
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize<TValue>(TValue, JsonSerializerOptions)")]
private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep)
{
switch (communicationObject.Command)
{
case CommunicationCommand.GetScanningProgress:
{
DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath);
long discardedIndexes = _dbHandler.GetDiscardedIndexes();
ScanningStatus status = new();
// 4294967296 is all Ipv4 addresses.
if (discardedIndexes != 0)
{
status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100;
}
else
{
// This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float.
status.PercentageOfIpv4Scanned = 0.0000000001f;
}
status.AmountOfIpv4Left = 4294967296 - discardedIndexes;
status.TotalFiltered = _dbHandler.GetFilteredIndexes();
status.TotalDiscarded = discardedIndexes;
status.MyDbSize = databaseSizes.MyDbSize;
status.FilteredDbSize = databaseSizes.FilteredDbSize;
status.DiscardedDbSize = databaseSizes.DiscardedDbSize;
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status);
rep.SendFrame(serializedResult);
break;
}
case CommunicationCommand.DbReindex:
{
_dbHandler.ReIndex();
SendStringResponse(rep, "All Dbs have been reindexed.");
break;
}
case CommunicationCommand.DbVacuum:
{
_dbHandler.Vacuum();
SendStringResponse(rep, "All Dbs have been vacuumed.");
break;
}
case CommunicationCommand.GetSearches:
{
if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm))
{
SendSearchResponse(rep, communicationObject.SearchTerm);
}
break;
}
case CommunicationCommand.StopScanning:
{
_isRunning = false;
break;
}
case CommunicationCommand.ChangeRuntimeVariable:
{
if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break;
if (communicationObject.Variable == RuntimeVariable.DbContent.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetContentWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_dbHandler.SetDiscardedWaitTime(value);
}
if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_ipScanner.SetTimeout(value);
}
if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString())
{
int value = int.Parse(communicationObject.VariableValue);
_contentFilter.SetTimeout(value);
}
rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success"));
break;
}
}
}
private static void SendStringResponse(ResponseSocket rep, string response)
{
byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response);
rep.SendFrame(serializedResult);
}
private static void SendSearchResponse(ResponseSocket rep, string searchTerm)
{
//SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler);
SearchResults result = new()
{
Results = []
};
SearchResult lol = new()
{
Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'",
};
result.Results.Add(lol);
string serializedResult = JsonSerializer.Serialize(result);
rep.SendFrame(serializedResult);
}
}

149
Backend/Handler/ContentFilter.cs
View File

@ -6,27 +6,44 @@ using Models.Model.Backend;
namespace Backend.Handler;
public class Content
{
public int Port1 { get; set; }
public int Port2 { get; set; }
public Ip Ip { get; set; }
}
public class ContentThread
{
public int ThreadId { get; set; }
public EventWaitHandle? EventWaitHandle { get; set; }
}
public class ContentFilter
{
private readonly ConcurrentQueue<Filtered> _queue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Content?> _contentQueue = new();
private readonly DbHandler _dbHandler;
private readonly ThreadHandler _threadHandler;
private readonly string _getDomainPort80;
private readonly string _getDomainPort443;
private bool _stop;
private int _timeOut;
private readonly string _basePath;
public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath)
public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
{
_queue = queue;
_dbHandler = dbHandler;
_basePath = basePath;
_threadHandler = threadHandler;
_unfilteredQueue = unfilteredQueue;
_getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
_getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
SetTimeout(3000);
}
@ -52,11 +69,23 @@ public class ContentFilter
while (!_stop)
{
List<long> indexes = _dbHandler.GetUnfilteredIndexes();
if (indexes.Count == 0)
{
_stop = true;
_threadHandler.Stop();
break;
}
for (int i = 0; i < indexes.Count; i++)
{
if (_stop) break;
if (_contentQueue.Count >= 500)
{
Thread.Sleep(1000);
}
Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
if (unfiltered.Filtered) continue;
@ -77,13 +106,13 @@ public class ContentFilter
{
continue;
}
Content content = new();
content.Ip = ip;
content.Port1 = unfiltered.Port1;
content.Port2 = unfiltered.Port2;
Filtered filtered = GetSiteData(ip);
filtered.Port1 = unfiltered.Port1;
filtered.Port2 = unfiltered.Port2;
_queue.Enqueue(filtered);
_contentQueue.Enqueue(content);
}
Thread.Sleep(_timeOut);
@ -91,18 +120,65 @@ public class ContentFilter
((EventWaitHandle) obj).Set();
}
private Filtered GetSiteData(Ip ip)
public WaitHandle[] StartFilterThread(int threads)
{
StartProcess(ip, 80);
StartProcess(ip, 443);
WaitHandle[] waitHandle = new WaitHandle[threads];
for (int i = 0; i < threads; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
ContentThread contentThread = new();
contentThread.ThreadId = i;
contentThread.EventWaitHandle = handle;
waitHandle[i] = handle;
Thread thread = new(FilterThread!);
thread.Start(contentThread);
Thread.Sleep(8);
}
return waitHandle;
}
private void FilterThread(object obj)
{
Console.WriteLine("Filter Thread started.");
ContentThread thread = (ContentThread) obj;
while (!_stop)
{
if (_contentQueue.IsEmpty)
{
Thread.Sleep(1000);
}
_contentQueue.TryDequeue(out Content? content);
if (content is null)
{
continue;
}
Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
filtered.Port1 = content.Port1;
filtered.Port2 = content.Port2;
_queue.Enqueue(filtered);
}
thread.EventWaitHandle!.Set();
}
private Filtered GetSiteData(Ip ip, int threadId)
{
StartProcess(ip, 80, threadId);
StartProcess(ip, 443, threadId);
string url1 = "";
string url2 = "";
string title1 = "";
string title2 = "";
string description1 = "";
string description2 = "";
bool robotsTxt1 = false;
bool robotsTxt2 = false;
string serverType1 = "";
@ -128,7 +204,7 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt");
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
while (streamReader.Peek() != -1)
{
@ -161,37 +237,6 @@ public class ContentFilter
for (int i = 0; i < ports.Length; i++)
{
if (ports[i] == 80)
{
if (string.IsNullOrWhiteSpace(url1)) continue;
try
{
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult();
title1 = temp.Item1;
description1 = temp.Item2;
}
catch
{
//
}
}
else
{
if (string.IsNullOrWhiteSpace(url2)) continue;
try
{
(string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult();
title2 = temp.Item1;
description2 = temp.Item2;
}
catch
{
//
}
}
if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
}
@ -201,10 +246,6 @@ public class ContentFilter
Ip = ip,
Url1 = url1,
Url2 = url2,
Title1 = title1,
Title2 = title2,
Description1 = description1,
Description2 = description2,
ServerType1 = serverType1,
ServerType2 = serverType2,
RobotsTXT1 = robotsTxt1,
@ -230,7 +271,7 @@ public class ContentFilter
return siteData;
}
private void StartProcess(Ip ip, int port)
private void StartProcess(Ip ip, int port, int threadId)
{
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
@ -238,7 +279,7 @@ public class ContentFilter
proc.StartInfo = new()
{
FileName = "/bin/bash",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt",
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
UseShellExecute = false,
RedirectStandardOutput = false,
RedirectStandardError = false,

227
Backend/Handler/IpFilterHandler.cs Normal file
View File

@ -0,0 +1,227 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Backend.Helper;
using Models.Handler;
using Models.Model.Backend;
namespace Backend.Handler;
public class IpFilterHandler
{
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private DbHandler _dbHandler;
private ThreadHandler _threadHandler;
private bool _stop;
private bool _fillerStop;
private bool _stopAutoscaledThreads;
private int _timeout;
public IpFilterHandler(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
ConcurrentQueue<FilterQueueItem> preFilteredQueue,
DbHandler dbHandler, ThreadHandler threadHandler)
{
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_preFilteredQueue = preFilteredQueue;
_dbHandler = dbHandler;
_threadHandler = threadHandler;
_timeout = 16;
}
public List<WaitHandle[]> Start(int threadCount)
{
WaitHandle[] waitHandle = new WaitHandle[64];
int counter = 0;
List<WaitHandle[]> waitHandles = [];
for (int i = 0; i < threadCount; i++)
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
if (counter < 64)
{
waitHandle[counter] = handle;
counter++;
Thread f = new (Filter!);
f.Start(handle);
Console.WriteLine($"Filter thread ({i}) started");
Thread.Sleep(16);
continue;
}
counter = 0;
waitHandles.Add(waitHandle);
waitHandle = new WaitHandle[64];
}
return waitHandles;
}
public void AutoScaler()
{
int i = 0;
int j = 0;
while (!_stop)
{
if (_preFilteredQueue.Count >= 2000)
{
if (i == 10)
{
_stopAutoscaledThreads = false;
Console.WriteLine("Autoscaler started");
while (!_stopAutoscaledThreads)
{
if (_preFilteredQueue.Count <= 2000)
{
if (j == 1000)
{
_stopAutoscaledThreads = true;
}
j++;
Thread.Sleep(128);
}
else
{
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
Thread f = new (Filter_AutoScaler!);
f.Start(handle);
Thread.Sleep(16);
}
}
}
i++;
}
else
{
i = 0;
j = 0;
}
Thread.Sleep(128);
}
}
private void Filter(object obj)
{
int counter = 0;
while (!_stop)
{
if (_preFilteredQueue.IsEmpty && _fillerStop)
{
if (counter == 100)
{
_threadHandler.Stop();
_stop = true;
}
counter++;
Thread.Sleep(128);
}
_preFilteredQueue.TryDequeue(out FilterQueueItem item);
(int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
}
((EventWaitHandle) obj).Set();
}
public void FillFilterQueue()
{
Console.WriteLine("Fill FilterQueue started.");
while (!_stop)
{
if (_preFilteredQueue.Count > 500) continue;
if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item))
{
_preFilteredQueue.Enqueue(item);
}
else
{
_fillerStop = true;
}
}
}
private void Filter_AutoScaler(object obj)
{
while (!_stopAutoscaledThreads)
{
if (_preFilteredQueue.IsEmpty)
{
Thread.Sleep(_timeout);
continue;
}
_preFilteredQueue.TryDequeue(out FilterQueueItem item);
(int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports));
}
((EventWaitHandle) obj).Set();
}
private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
{
return new()
{
Ip = ip,
ResponseCode = responseCode
};
}
private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
{
Unfiltered unfiltered = new()
{
Ip = ip,
Port1 = ports.Item1,
Port2 = ports.Item2,
Filtered = false
};
return new()
{
Unfiltered = unfiltered,
Operations = Operations.Insert
};
}
public void Stop()
{
_stop = true;
}
}

229
Backend/Handler/IpScanner.cs
View File

@ -1,7 +1,11 @@
using System.Buffers.Binary;
using System.Collections.Concurrent;
using System.Net;
using System.Net.NetworkInformation;
using System.Numerics;
using System.Runtime.InteropServices;
using Backend.Helper;
using Models.Experimental;
using Models.Handler;
using Models.Model.Backend;
@ -18,27 +22,22 @@ public class ScanSettings
public class IpScanner
{
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly DbHandler _dbHandler;
private bool _stop;
private int _timeout;
private readonly int _timeout;
public IpScanner(ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler
)
public IpScanner(ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, DbHandler dbHandler,
ConcurrentQueue<FilterQueueItem> preFilteredQueue)
{
_dbHandler = dbHandler;
_preFilteredQueue = preFilteredQueue;
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_resumeQueue = resumeQueue;
SetTimeout(64);
}
public void SetTimeout(int milliseconds)
{
_timeout = milliseconds;
_timeout = 32;
}
public List<WaitHandle[]> Start(int threads)
@ -76,7 +75,7 @@ public class IpScanner
f.Start(scanSettings);
Console.WriteLine($"Scanner thread ({i}) started");
Thread.Sleep(100);
Thread.Sleep(128);
continue;
}
@ -108,16 +107,26 @@ public class IpScanner
if (resumeNow is not null)
{
if (resumeNow.Completed)
{
return;
}
scanSettings.Start = resumeNow.FirstByte;
scanSettings.End = resumeNow.EndRange;
secondByte = resumeNow.SecondByte;
thirdByte = resumeNow.ThirdByte;
fourthByte = resumeNow.FourthByte;
}
else
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert);
}
// Empty buffer so we use the lowest abstracted ping.Send() method.
byte[] buf = [];
using Ping ping = new();
int x = 0;
for (int i = scanSettings.Start; i < scanSettings.End; i++)
{
@ -136,19 +145,34 @@ public class IpScanner
if (i == 192 && k == 2) continue;
if (i == 192 && j == 88 && k == 99) continue;
if (_discardedQueue.Count >= 2000)
if (_discardedQueue.Count >= 20_000)
{
Thread.Sleep(500);
Thread.Sleep(1000);
}
if (_preFilteredQueue.Count >= 20_000)
{
Thread.Sleep(1000);
}
for (int l = fourthByte; l < 256; l++)
{
if (x == 75_000)
{
CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update);
x = 0;
}
x++;
if (_stop)
{
resumeObject.FourthByte = l;
break;
}
Ip ip = new()
{
Ip1 = i,
@ -156,19 +180,29 @@ public class IpScanner
Ip3 = k,
Ip4 = l
};
IPStatus responseCode = IPStatus.Unknown;
try
{
// Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception.
// https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages
_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
if (address is not null)
//_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address);
responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout);
/*if (l % 2 == 0)
{
responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status;
Thread.Sleep(_timeout);
responseCode = IPStatus.Success;
}
else
{
responseCode = IPStatus.TimedOut;
}*/
//responseCode = IPStatus.TimedOut;
//Thread.Sleep(0);
}
catch
{
@ -181,15 +215,7 @@ public class IpScanner
continue;
}
(int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443);
if (ports is { Item1: 0, Item2: 0 })
{
_discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode));
continue;
}
_unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports));
_preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode));
}
if (_stop)
@ -211,14 +237,42 @@ public class IpScanner
resumeObject.FirstByte = i;
break;
}
Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})");
}
Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped.");
if (_stop)
{
resumeObject.Paused = true;
}
else
{
resumeObject.Completed = true;
}
resumeObject.Operation = Operations.Update;
_resumeQueue.Enqueue(resumeObject);
scanSettings.Handle!.Set();
}
private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation)
{
ScannerResumeObject resumeObject = new();
resumeObject.ThreadNumber = threadNumber;
resumeObject.StartRange = startRange;
resumeObject.EndRange = endRange;
resumeObject.FirstByte = firstByte;
resumeObject.SecondByte = secondByte;
resumeObject.ThirdByte = thirdByte;
resumeObject.FourthByte = fourthByte;
resumeObject.Paused = paused;
resumeObject.Completed = completed;
resumeObject.Operation = operation;
_resumeQueue.Enqueue(resumeObject);
}
private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode)
{
@ -229,25 +283,114 @@ public class IpScanner
};
}
private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports)
private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode)
{
Unfiltered unfiltered = new()
FilterQueueItem filterQueueItem = new()
{
Ip = ip,
Port1 = ports.Item1,
Port2 = ports.Item2,
Filtered = false
ResponseCode = responseCode
};
return new()
{
Unfiltered = unfiltered,
Operations = Operations.Insert
};
return filterQueueItem;
}
public void Stop()
{
_stop = true;
}
private static unsafe IPAddress Parse(ReadOnlySpan<char> ipSpan)
{
int length = ipSpan.Length;
long nonCanonical;
fixed (char* name = &MemoryMarshal.GetReference<char>(ipSpan))
nonCanonical = ParseNonCanonical(name, 0, ref length, true);
return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8));
}
private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile)
{
long* numPtr = stackalloc long[4];
long num1 = 0;
bool flag = false;
int index1 = 0;
int index2;
for (index2 = start; index2 < end; ++index2)
{
char ch = name[index2];
num1 = 0L;
int num2 = 10;
if (ch == '0')
{
num2 = 8;
++index2;
flag = true;
if (index2 < end)
{
switch (name[index2])
{
case 'X':
case 'x':
num2 = 16;
++index2;
flag = false;
break;
}
}
}
for (; index2 < end; ++index2)
{
char c = name[index2];
int num3;
if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c))
num3 = (int) c - 48;
else if (num2 == 8 && '0' <= c && c <= '7')
num3 = (int) c - 48;
else if (num2 == 16 && 'a' <= c && c <= 'f')
num3 = (int) c + 10 - 97;
else if (num2 == 16 && 'A' <= c && c <= 'F')
num3 = (int) c + 10 - 65;
else
break;
num1 = num1 * (long) num2 + (long) num3;
if (num1 > (long) uint.MaxValue)
return -1;
flag = true;
}
if (index2 < end && name[index2] == '.')
{
if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue)
return -1;
numPtr[index1] = num1;
++index1;
flag = false;
}
else
break;
}
if (!flag)
return -1;
if (index2 < end)
{
char ch;
if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#'))
return -1;
end = index2;
}
numPtr[index1] = num1;
switch (index1)
{
case 0:
return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0];
case 1:
return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L;
case 2:
return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue;
case 3:
return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue;
default:
return -1;
}
}
}

185
Backend/Handler/ThreadHandler.cs
View File

@ -7,57 +7,108 @@ namespace Backend.Handler;
public class ThreadHandler
{
private readonly DbHandler _dbHandler;
private readonly Communication _communication;
private readonly IpScanner _ipScanner;
private readonly ContentFilter _contentFilter;
private readonly IpFilterHandler _ipFilterHandler;
private bool _communicationStopped;
private bool _ipScannerStopped;
private bool _contentFilterStopped;
private bool _ipFilterStopped;
private bool _stage1 = true;
private bool _stage2;
private bool _stage3;
ConcurrentQueue<Filtered> filteredQueue = new();
ConcurrentQueue<Discarded> discardedQueue = new();
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
ConcurrentQueue<FilterQueueItem> preFilteredQueue = new();
public ThreadHandler(string path)
{
ConcurrentQueue<Filtered> filteredQueue = new();
ConcurrentQueue<Discarded> discardedQueue = new();
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue = new();
ConcurrentQueue<ScannerResumeObject> scannerResumeQueue = new();
_dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path);
_ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler);
_contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path);
_communication = new(_dbHandler, this, _ipScanner, _contentFilter, path);
_dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path);
_ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue);
_contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this);
_ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this);
}
public void Start()
{
Thread scanner = new(StartScanner);
Thread ipFilter = new(StartIpFilter);
Thread indexer = new(StartContentFilter);
Thread database = new(StartDbHandler);
Thread discarded = new(StartDiscardedDbHandler);
Thread filtered = new(StartFilteredDbHandler);
Thread resume = new(StartResumeDbHandler);
Thread communication = new(StartCommunicationHandler);
scanner.Start();
indexer.Start();
database.Start();
discarded.Start();
filtered.Start();
resume.Start();
communication.Start();
scanner.Join();
indexer.Join();
database.Join();
discarded.Join();
filtered.Join();
resume.Join();
communication.Join();
Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler);
Thread contentFilterThread = new(StartContentFilterThread);
Thread prefilterDb = new(StartPreFilterDbHandler);
Thread fillIpFilterQueue = new(StartFillIpFilterQueue);
//Thread check = new(CheckQueue);
if (_stage1)
{
discarded.Start(); // de-queues from discardedQueue
prefilterDb.Start(); // de-queues from preFilteredQueue
scanner.Start(); // en-queues to discardedQueue and preFilteredQueue
resume.Start(); // de-queues from resumeQueue
scanner.Join();
Stop();
discarded.Join();
prefilterDb.Join();
resume.Join();
}
if (_stage2)
{
database.Start(); // de-queues from unfilteredQueue
discarded.Start(); // de-queues from discardedQueue
ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue
ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue
fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue
database.Join();
discarded.Join();
ipFilter.Join();
ipFilterAutoScaler.Join();
fillIpFilterQueue.Join();
}
if (_stage3)
{
filtered.Start(); // de-queues from filteredQueue
database.Start(); // de-queues from unfilteredQueue
indexer.Start(); // en-queues to unfilteredQueue and contentQueue
contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue
contentFilterThread.Join();
filtered.Join();
database.Join();
indexer.Join();
}
}
private void CheckQueue()
{
while (true)
{
Console.Clear();
Console.WriteLine($"filteredQueue.Count: {filteredQueue.Count}");
Console.WriteLine($"discardedQueue.Count: {discardedQueue.Count}");
Console.WriteLine($"unfilteredQueue.Count: {unfilteredQueue.Count}");
Console.WriteLine($"scannerResumeQueue.Count: {scannerResumeQueue.Count}");
Console.WriteLine($"preFilteredQueue.Count: {preFilteredQueue.Count}");
Thread.Sleep(5);
}
}
private void StartScanner()
{
Thread.Sleep(5000); // Let the database handler instantiate and warm up first.
Thread.Sleep(15000); // Let the database handler instantiate and warm up first.
List<WaitHandle[]> wait = _ipScanner.Start(256);
@ -67,8 +118,6 @@ public class ThreadHandler
}
Console.WriteLine("Scanner finished");
_ipScannerStopped = true;
}
private void StartContentFilter()
@ -83,6 +132,39 @@ public class ThreadHandler
_contentFilterStopped = true;
}
private void StartContentFilterThread()
{
WaitHandle[] wait = _contentFilter.StartFilterThread(64);
WaitHandle.WaitAll(wait);
}
private void StartIpFilterAutoScaler()
{
_ipFilterHandler.AutoScaler();
}
private void StartFillIpFilterQueue()
{
_ipFilterHandler.FillFilterQueue();
}
private void StartIpFilter()
{
Thread.Sleep(1000);
List<WaitHandle[]> wait = _ipFilterHandler.Start(256);
for (int i = 0; i < wait.Count; i++)
{
WaitHandle.WaitAll(wait[i]);
}
Console.WriteLine("Ip filter finished");
_ipFilterStopped = true;
}
private void StartDbHandler()
{
@ -93,6 +175,11 @@ public class ThreadHandler
{
_dbHandler.FilteredDbHandler();
}
private void StartPreFilterDbHandler()
{
_dbHandler.PrefilteredDbHandler();
}
private void StartResumeDbHandler()
{
@ -101,42 +188,26 @@ public class ThreadHandler
private void StartDiscardedDbHandler()
{
WaitHandle[] wait = _dbHandler.Start(5);
WaitHandle[] wait = _dbHandler.Start(4);
WaitHandle.WaitAll(wait);
Console.WriteLine("Discarded DbHandler finished");
}
private void StartCommunicationHandler()
{
WaitHandle[] wait = _communication.Start();
WaitHandle.WaitAll(wait);
Console.WriteLine("Communicator finished");
_communicationStopped = true;
Stop();
}
private void Stop()
public void Stop()
{
Console.WriteLine("Stopping...");
_ipScanner.Stop();
_contentFilter.Stop();
_ipFilterHandler.Stop();
Console.WriteLine("Stopping Extra...");
bool stopping = true;
Thread.Sleep(30_000);
while (stopping)
{
if (_communicationStopped && _ipScannerStopped && _contentFilterStopped)
{
_dbHandler.Stop();
stopping = false;
}
Thread.Sleep(3000);
}
Console.WriteLine("Stopping Super Extra...");
_dbHandler.Stop();
Console.WriteLine("Stopped.");
}
}

43
Backend/Helper/FilesystemHelper.cs
View File

@ -1,43 +0,0 @@
using Models.Model.Backend;
namespace Backend.Helper;
public static class FilesystemHelper
{
public static DatabaseSizes GetDatabaseSizes(string basePath)
{
DatabaseSizes databaseSizes = new();
FileInfo fileInfo = new($"{basePath}/Models/mydb.db");
databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB);
fileInfo = new($"{basePath}/Models/Filtered.db");
databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB);
return databaseSizes;
}
private static long GetDiscardedDbSizes(string basePath)
{
string folder = $"{basePath}/Models";
string[] files = Directory.GetFiles(folder, "*.db");
long size = 0;
for (int i = 0; i < files.Length; i++)
{
if (!files[i].Contains("Discarded")) continue;
FileInfo fileInfo = new(files[i]);
size += fileInfo.Length;
}
return size;
}
private static double ToSize(this long value, SizeUnits unit)
{
return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00"));
}
}

44
Backend/Helper/FilterHelper.cs
View File

@ -2,12 +2,10 @@ using System.Text.RegularExpressions;
namespace Backend.Helper;
public static partial class FilterHelper
public static class FilterHelper
{
// https://stackoverflow.com/a/56116499
private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$";
private const string TitlePattern = "<title>(.*)</title>";
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
public static void GetDomain(string data, out string result)
{
@ -42,32 +40,6 @@ public static partial class FilterHelper
result = GetSubstring(data, start, end);
}
public static void GetTitle(string data, out string result)
{
result = "";
Regex titleRegex = MyRegex1();
Match match = titleRegex.Match(data);
if (match.Success)
{
result = match.Groups[1].Value;
}
}
public static void GetDescription(string data, out string result)
{
result = "";
Regex titleRegex = MyRegex2();
Match match = titleRegex.Match(data);
if (match.Success)
{
result = match.Groups[1].Value;
}
}
public static void GetHttpVersion(string data, out string result)
{
result = "";
@ -198,7 +170,7 @@ public static partial class FilterHelper
case 0:
return;
case 2:
result = $"{temp[1].Trim()} UUHHHMMM";
result = $"{temp[1].Trim()}";
return;
default:
result = temp[2].Trim();
@ -241,17 +213,7 @@ public static partial class FilterHelper
private static bool ValidateUrl(string url)
{
Regex rgx = MyRegex();
Regex rgx = new(DomainPattern);
return rgx.IsMatch(url);
}
//Generate the RegEx at compile time.
[GeneratedRegex(DomainPattern, RegexOptions.IgnoreCase | RegexOptions.Compiled, "en-US")]
private static partial Regex MyRegex();
[GeneratedRegex(TitlePattern)]
private static partial Regex MyRegex1();
[GeneratedRegex(DescriptionPattern)]
private static partial Regex MyRegex2();
}

93
Backend/Helper/HttpClientHelper.cs
View File

@ -1,5 +1,4 @@
using System.Text.RegularExpressions;
using HtmlAgilityPack;
namespace Backend.Helper;
@ -7,93 +6,6 @@ public static partial class HttpClientHelper
{
// Reddit, for example, will block the GET request if you don't have a user agent.
private const string UserAgentHeader = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
private const string TitlePattern = "<title>(.*)</title>";
private const string DescriptionPattern = "<meta name=\"description\" content=\"(.*?)\"";
private const string StartHeadTag = "<head>";
private const string EndHeadTag = "</head>";
public static async Task<(string, string)> GetTitleAndDescription(string url, int port)
{
using HttpClient client = new();
if (port == 80)
{
try
{
client.BaseAddress = new($"http://{url}");
}
catch
{
//
}
}
else
{
try
{
client.BaseAddress = new($"https://{url}");
}
catch
{
//
}
}
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader);
client.Timeout = TimeSpan.FromSeconds(30);
HttpResponseMessage? response;
try
{
response = await client.GetAsync("/");
}
catch
{
return ("", "");
}
if (!response.IsSuccessStatusCode)
{
return ("", "");
}
string html = await response.Content.ReadAsStringAsync();
int firstIndex = 0;
int lastIndex = 0;
if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag))
{
firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal);
lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal);
}
string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString();
html = "";
string title = "";
string description = "";
Regex titleRegex = TitleRegEx();
Match titleMatch = titleRegex.Match(head);
if (titleMatch.Success)
{
title = titleMatch.Groups[1].Value;
}
Regex descriptionRegex = DexcriptionRegEx();
Match descriptionMatch = descriptionRegex.Match(head);
if (descriptionMatch.Success)
{
description = descriptionMatch.Groups[1].Value;
}
return (title, description);
}
public static async Task<bool> HasRobotsTxt(string url, int port)
{
@ -139,9 +51,4 @@ public static partial class HttpClientHelper
return response is not null && response.IsSuccessStatusCode;
}
[GeneratedRegex(TitlePattern)]
private static partial Regex TitleRegEx();
[GeneratedRegex(DescriptionPattern)]
private static partial Regex DexcriptionRegEx();
}

44
Backend/Helper/SearchHelper.cs
View File

@ -1,44 +0,0 @@
using FuzzySharp;
using Models.Handler;
using Models.Model.External;
namespace Backend.Helper;
public static class SearchHelper
{
public static SearchResults Search(string searchText, DbHandler dbHandler)
{
if (string.IsNullOrWhiteSpace(searchText))
{
SearchResult searchResult = new();
searchResult.Description = "asd";
searchResult.Title = "asd";
searchResult.Url = "asd";
SearchResults searchResults = new();
searchResults.Results =
[
searchResult
];
return searchResults;
}
List<SearchResult?> temp = dbHandler.GetSearchResults();
SearchResults searchResultsList = new();
for (int i = 0; i < temp.Count; i++)
{
if (temp[i] is null) continue;
SearchResult result = new();
if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue;
result.Url = temp[i]?.Url;
result.Title = temp[i]?.Title;
searchResultsList.Results.Add(result);
}
return searchResultsList;
}
}

12
Backend/Helper/TcpClientHelper.cs
View File

@ -1,20 +1,24 @@
using System.Net;
using System.Net.Sockets;
using Models.Model.Backend;
namespace Backend.Helper;
public static class TcpClientHelper
{
public static (int, int) CheckPort(string ip, params int[] ports)
public static (int, int) CheckPort(Ip ip, params int[] ports)
{
// This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached,
// and it would just return a result.error, for example.
for (int i = 0; i < ports.Length; i++) {
using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
socket.SendTimeout = 250;
try
{
using TcpClient client = new();
client.Connect(ip, ports[i]);
// If the connection is successful, update the result array with the port number
socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i]));
socket.Close();
// If the connection is not successful, update the ports array with 0.
}
catch
{

122
Manager/Commands.cs
View File

@ -1,122 +0,0 @@
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using Models.Model.Backend;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
namespace Manager;
public static class Commands
{
public static void GetProgress()
{
Console.WriteLine("Getting progress ...");
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.GetScanningProgress;
ScanningStatus temp = GetProgress(communicationObject);
Console.WriteLine($"Total filtered: {temp.TotalFiltered:n0}");
Console.WriteLine($"Total discarded: {temp.TotalDiscarded:n0}");
Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}");
Console.WriteLine($"Total Ips left: {temp.AmountOfIpv4Left:n0}");
Console.WriteLine($"Filtered DB size: {temp.FilteredDbSize} Kb");
Console.WriteLine($"Discarded DB size: {temp.DiscardedDbSize} Kb");
Console.WriteLine($"Mydb DB size: {temp.MyDbSize} Kb");
}
public static void StopServer()
{
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.StopScanning;
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void Vacuum()
{
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.DbVacuum;
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void ReIndex()
{
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.DbReindex;
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value)
{
CommunicationObject communicationObject = new()
{
Command = CommunicationCommand.ChangeRuntimeVariable,
Variable = runtimeVariable.ToString(),
VariableValue = value
};
Console.WriteLine(SendAndRecieveStringMessage(communicationObject));
}
public static void GetHelp()
{
Console.WriteLine("Available commands:");
Console.WriteLine(" stop - stops the server");
Console.WriteLine(" clear - clears the console");
Console.WriteLine(" q - quits the program");
Console.WriteLine(" p - print the progress information of the scanner");
Console.WriteLine(" g - manual garbage collect on the server");
Console.WriteLine(" r - manual reindex the databases");
Console.WriteLine(" v - manual vacuum the databases");
Console.WriteLine(" R - change runtime variable");
Console.WriteLine(" lR - list runtime variables");
Console.WriteLine(" help - shows this help");
Console.WriteLine();
}
public static void PrintRuntimeVariables()
{
Console.WriteLine("Runtime variables:");
Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()} - Sets the timeout in milliseconds for the scanner");
Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()} - Sets the timeout in milliseconds for the content filter");
Console.WriteLine($"{RuntimeVariable.DbContent.ToString()} - Sets the wait time in milliseconds for the content database if the queue is empty");
Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()} - Sets the wait time in milliseconds for the discarded database if the queue is empty");
}
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject)
{
//byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options);
byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(lol);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return JsonSerializer.Deserialize<string>(msg);
}
[RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
[RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes<TValue>(TValue, JsonSerializerOptions)")]
private static ScanningStatus GetProgress(CommunicationObject communicationObject)
{
byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(lol);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
return JsonSerializer.Deserialize<ScanningStatus>(msg);
}
}

25
Manager/Manager.csproj
View File

@ -1,25 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<!--<Platform>x64</Platform>
<Optimize>true</Optimize>
<PublishAot>true</PublishAot>
<PublishTrimmed>true</PublishTrimmed>
<TrimMode>link</TrimMode>
<TrimmerRemoveSymbols>true</TrimmerRemoveSymbols>-->
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Models\Models.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>
</Project>

90
Manager/Program.cs
View File

@ -1,90 +0,0 @@
using Manager;
using Models.Model.Backend;
bool stop = false;
do
{
string? input = Console.ReadLine();
if (string.Equals(input, "stop"))
{
Commands.StopServer();
}
else if (string.Equals(input, "q"))
{
stop = true;
}
else if (string.Equals(input, "clear"))
{
Console.Clear();
}
else if (string.Equals(input, "p"))
{
Commands.GetProgress();
}
else if (string.Equals(input, "v"))
{
Commands.Vacuum();
}
else if (string.Equals(input, "r"))
{
Commands.ReIndex();
}
else if (string.Equals(input, "R"))
{
Console.WriteLine("Variable name.");
string? variable = Console.ReadLine();
Console.WriteLine("Variable value.");
string? value = Console.ReadLine();
if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value))
{
Console.WriteLine("Please enter a value.");
return;
}
if (variable == RuntimeVariable.ScannerTimeout.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value);
}
else if (variable == RuntimeVariable.ContentFilter.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value);
}
else if (variable == RuntimeVariable.DbDiscarded.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value);
}
else if (variable == RuntimeVariable.DbContent.ToString())
{
Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value);
}
}
else if (string.Equals(input, "lR"))
{
Commands.PrintRuntimeVariables();
}
else if (string.Equals(input, "help"))
{
Commands.GetHelp();
}
else
{
Commands.GetHelp();
}
} while (!stop);

BIN
Models/BackupDB/ScannerResume.db
View File

Binary file not shown.

39
Models/Experimental/CustomPing.cs Normal file
View File

@ -0,0 +1,39 @@
using System.Diagnostics;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using Models.Model.Backend;
namespace Models.Experimental;
public class CustomPing
{
public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout)
{
SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8}));
using Socket rawSocket = RawSocket.GetRawSocket(socketConfig);
int ipHeaderLength = 20;
try
{
rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint);
byte[] numArray = new byte[136];
long timestamp = Stopwatch.GetTimestamp();
// TODO: WTF ???
EndPoint lol = socketConfig.EndPoint;
while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout)
{
int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol);
IPStatus status;
if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status))
return status;
}
}
catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut)
{
}
return IPStatus.TimedOut;
}
}

59
Models/Experimental/MessageConstant.cs Normal file
View File

@ -0,0 +1,59 @@
using System.Net.NetworkInformation;
namespace Models.Experimental;
public struct MessageConstant
{
public static IPStatus MapV4TypeToIpStatus(int type, int code)
{
IPStatus ipStatus1;
switch ((IcmpV4MessageType) type)
{
case IcmpV4MessageType.EchoReply:
ipStatus1 = IPStatus.Success;
break;
case IcmpV4MessageType.DestinationUnreachable:
IPStatus ipStatus2;
switch ((byte) code)
{
case 0:
ipStatus2 = IPStatus.DestinationNetworkUnreachable;
break;
case 1:
ipStatus2 = IPStatus.DestinationHostUnreachable;
break;
case 2:
ipStatus2 = IPStatus.DestinationProtocolUnreachable;
break;
case 3:
ipStatus2 = IPStatus.DestinationPortUnreachable;
break;
default:
ipStatus2 = IPStatus.DestinationUnreachable;
break;
}
ipStatus1 = ipStatus2;
break;
case IcmpV4MessageType.SourceQuench:
ipStatus1 = IPStatus.SourceQuench;
break;
case IcmpV4MessageType.TimeExceeded:
ipStatus1 = IPStatus.TtlExpired;
break;
case IcmpV4MessageType.ParameterProblemBadIpHeader:
ipStatus1 = IPStatus.BadHeader;
break;
default:
ipStatus1 = IPStatus.Unknown;
break;
}
return ipStatus1;
}
}
internal enum IcmpV4MessageType : byte
{
EchoReply = 0,
DestinationUnreachable = 3,
SourceQuench = 4,
TimeExceeded = 11, // 0x0B
ParameterProblemBadIpHeader = 12, // 0x0C
}

91
Models/Experimental/RawSocket.cs Normal file
View File

@ -0,0 +1,91 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Models.Model.Backend;
namespace Models.Experimental;
public class RawSocket
{
public static unsafe Socket GetRawSocket(SocketConfig socketConfig)
{
Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType);
rawSocket.ReceiveTimeout = socketConfig.Timeout;
rawSocket.SendTimeout = socketConfig.Timeout;
rawSocket.Connect(socketConfig.EndPoint);
int num = 1;
rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan<byte>((void*) &num, 4));
return rawSocket;
}
public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply)
{
byte num = (byte) (receiveBuffer[0] & 15U);
ipHeaderLength = 4 * num;
int start = ipHeaderLength;
int srcOffset = ipHeaderLength + 8;
IcmpHeader icmpHeader = Unsafe.ReadUnaligned<IcmpHeader>(ref MemoryMarshal.GetReference<byte>(receiveBuffer.AsSpan<byte>(start)));
byte[] numArray = new byte[bytesReceived - srcOffset];
Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length);
reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code);
return true;
}
public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader)
{
int length = sizeof (IcmpHeader);
byte[] sendMessageBuffer = new byte[length];
new Span<byte>((void*) &icmpHeader, length).CopyTo(new Span<byte>(sendMessageBuffer, 0, length));
ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan<byte>(0));
sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8);
sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue);
return sendMessageBuffer;
}
private static ushort ComputeBufferChecksum(ReadOnlySpan<byte> buffer)
{
uint num1 = 0;
for (int index = 0; index < buffer.Length; index += 2)
{
ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0));
num1 += num2;
}
while (num1 >> 16 != 0U)
num1 = (num1 & ushort.MaxValue) + (num1 >> 16);
return (ushort) ~num1;
}
}
public class SocketConfig
{
public EndPoint EndPoint;
public readonly int Timeout;
public readonly CustomProtocolType ProtocolType;
public readonly byte[] SendBuffer;
public SocketConfig(
EndPoint endPoint,
int timeout,
CustomProtocolType protocolType,
byte[] sendBuffer)
{
EndPoint = endPoint;
Timeout = timeout;
ProtocolType = protocolType;
SendBuffer = sendBuffer;
}
}
public struct IcmpHeader
{
public byte Type;
public byte Code;
public ushort HeaderChecksum;
public ushort Identifier;
public ushort SequenceNumber;
}

359
Models/Handler/DbHandler.cs
View File

@ -1,5 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Text;
using Microsoft.Data.Sqlite;
using Models.Helper;
using Models.Model.Backend;
@ -13,8 +15,10 @@ public class DbHandler
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
private readonly ConcurrentQueue<Discarded> _discardedQueue;
private readonly ConcurrentQueue<ScannerResumeObject> _resumeQueue;
private readonly ConcurrentQueue<FilterQueueItem> _preFilteredQueue;
private readonly string _unfilteredConnectionString;
private readonly string _preFilteredConnectionString;
private readonly string _filteredConnectionString;
private readonly string _resumeConnectionString;
private readonly string _compressedConnectionString;
@ -25,15 +29,20 @@ public class DbHandler
" INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" +
" VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)";
private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" +
" VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)";
private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" +
" INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," +
" Description1, Description2, Url1, Url2, ServerType1, ServerType2," +
" INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," +
" Url1, Url2, ServerType1, ServerType2," +
" RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," +
" CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," +
" PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," +
" ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " +
" @title1, @title2, @description1, @description2, @url1, @url2, " +
" @url1, @url2, " +
" (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " +
" (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " +
" @robotsTXT1, @robotsTXT2," +
@ -68,13 +77,13 @@ public class DbHandler
private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" +
" VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)";
" INSERT INTO Discarded (PackedData)" +
" VALUES (@packedData)";
private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
" INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" +
" VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);";
" INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" +
" VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);";
private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" +
" PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" +
@ -86,11 +95,16 @@ public class DbHandler
private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;";
private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;";
private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;";
private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;";
private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;";
private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;";
private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;";
private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;";
private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;";
private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;";
private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;";
private const string ReIndexDatabasesStatement = "REINDEX;";
private const string VacuumDatabasesStatement = "VACUUM;";
@ -111,12 +125,15 @@ public class DbHandler
public DbHandler(ConcurrentQueue<Filtered> filteredQueue,
ConcurrentQueue<Discarded> discardedQueue,
ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue,
ConcurrentQueue<ScannerResumeObject> resumeQueue, string basePath)
ConcurrentQueue<ScannerResumeObject> resumeQueue,
ConcurrentQueue<FilterQueueItem> preFilteredQueue,
string basePath)
{
_filteredQueue = filteredQueue;
_discardedQueue = discardedQueue;
_unfilteredQueue = unfilteredQueue;
_resumeQueue = resumeQueue;
_preFilteredQueue = preFilteredQueue;
SetContentWaitTime(100);
SetDiscardedWaitTime(10);
@ -127,6 +144,7 @@ public class DbHandler
_filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db";
_resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db";
_compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db";
_preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db";
}
public void SetContentWaitTime(int waitTime)
@ -188,6 +206,24 @@ public class DbHandler
Console.WriteLine("Filtered DbHandler stopped.");
}
public void PrefilteredDbHandler()
{
Console.WriteLine("PreFiltered Db handler started.");
while (!_stop)
{
if (_preFilteredQueue.IsEmpty)
{
Thread.Sleep(4);
continue;
}
_preFilteredQueue.TryDequeue(out FilterQueueItem queueItem);
InsertPrefiltered(queueItem);
}
}
public void ResumeDbHandler()
{
@ -202,12 +238,9 @@ public class DbHandler
continue;
}
_resumeQueue.TryDequeue(out ScannerResumeObject? queueItem);
_resumeQueue.TryDequeue(out ScannerResumeObject queueItem);
if (queueItem is not null)
{
InsertResumeObject(queueItem);
}
InsertResumeObject(queueItem);
}
Console.WriteLine("Resume DbHandler stopped.");
@ -232,7 +265,7 @@ public class DbHandler
Thread f = new (DiscardedDbHandler!);
f.Start(discardedDbHandlerSetting);
Thread.Sleep(5000);
Thread.Sleep(150);
}
return waitHandles;
@ -247,7 +280,16 @@ public class DbHandler
int i = 0;
while (!_stop)
SqliteConnection connection = new(connectionString);
connection.Open();
SqliteCommand command = new(InsertIntoDiscarded, connection);
StringBuilder stringBuilder = new();
bool running = true;
while (!_stop && running)
{
if (_discardedQueue.IsEmpty || _pause)
{
@ -255,31 +297,46 @@ public class DbHandler
_paused = true;
continue;
}
if (i >= 50_000_000 && !_compressing)
if (_stop && _discardedQueue.IsEmpty)
{
_compressing = true;
i = 0;
InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString));
int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models");
CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}");
DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId);
_compressing = false;
running = false;
}
_discardedQueue.TryDequeue(out Discarded queueItem);
if (i == 2048)
{
command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
_ = command.ExecuteNonQuery();
// Re-use the command object.
command.Parameters.Clear();
stringBuilder.Clear();
i = 0;
}
InsertDiscarded(queueItem, connectionString);
if (_discardedQueue.TryDequeue(out Discarded queueItem))
{
stringBuilder.Append(queueItem.Ip.PackIp());
stringBuilder.Append(':');
stringBuilder.Append(queueItem.ResponseCode);
stringBuilder.Append(',');
}
i++;
}
if (stringBuilder.Length != 0)
{
command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString()));
_ = command.ExecuteNonQuery();
}
connection.Close();
connection.Dispose();
command.Dispose();
discardedDbHandlerSetting.Handle!.Set();
Console.WriteLine("Discarded DbHandler stopped.");
@ -328,23 +385,6 @@ public class DbHandler
connection.Close();
}
private static void InsertDiscarded(Discarded discarded, string dbConnectionString)
{
using SqliteConnection connection = new(dbConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoDiscarded, connection);
command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1);
command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3);
command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4);
command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode);
_ = command.ExecuteNonQuery();
connection.Close();
}
private void InsertFiltered(Filtered filtered)
{
using SqliteConnection connection = new(_filteredConnectionString);
@ -427,10 +467,6 @@ public class DbHandler
command.Parameters.AddWithValue("@port2", filtered.Port2);
command.Parameters.AddWithValue("@url1", filtered.Url1);
command.Parameters.AddWithValue("@url2", filtered.Url2);
command.Parameters.AddWithValue("@title1", filtered.Title1);
command.Parameters.AddWithValue("@title2", filtered.Title2);
command.Parameters.AddWithValue("@description1", filtered.Description1);
command.Parameters.AddWithValue("@description2", filtered.Description2);
command.Parameters.AddWithValue("@serverType1", filtered.ServerType1);
command.Parameters.AddWithValue("@serverType2", filtered.ServerType2);
command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1);
@ -461,19 +497,31 @@ public class DbHandler
{
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(InsertIntoResume, connection);
SqliteCommand command;
if (resumeObject.Operation == Operations.Insert)
{
command = new(InsertIntoResume, connection);
command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
}
else
{
command = new(UpdateResumeStatement, connection);
}
command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber);
command.Parameters.AddWithValue("@startRange", resumeObject.StartRange);
command.Parameters.AddWithValue("@endRange", resumeObject.EndRange);
command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte);
command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte);
command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte);
command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte);
command.Parameters.AddWithValue("@paused", resumeObject.Paused);
command.Parameters.AddWithValue("@completed", resumeObject.Completed);
_ = command.ExecuteNonQuery();
connection.Close();
command.Dispose();
}
private void InsertCompressedDatabase(int threadNumber, long rows)
@ -489,6 +537,25 @@ public class DbHandler
_ = command.ExecuteNonQuery();
connection.Close();
}
private void InsertPrefiltered(FilterQueueItem filterQueueItem)
{
using SqliteConnection connection = new(_preFilteredConnectionString);
connection.Open();
using SqliteCommand command = new(InsertPreFilteredStatement, connection);
command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1);
command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2);
command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3);
command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4);
command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode);
command.Parameters.AddWithValue("@filtered", 0);
_ = command.ExecuteNonQuery();
connection.Close();
}
private void UpdateUnfiltered(Unfiltered unfiltered)
{
@ -557,79 +624,42 @@ public class DbHandler
return ids;
}
public long GetFilteredIndexes()
public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem)
{
long rowId = 0;
using SqliteConnection connection = new(_filteredConnectionString);
using SqliteConnection connection = new(_preFilteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredIdsStatement, connection);
SqliteCommand command = new(ReadPreFilteredStatement, connection);
using SqliteDataReader reader = command.ExecuteReader();
filterQueueItem = new();
Ip ip = new();
long id = 0;
if (!reader.HasRows)
{
return 0;
return false;
}
while (reader.Read())
{
rowId = reader.GetInt64(0);
ip.Ip1 = reader.GetInt32(0);
ip.Ip2 = reader.GetInt32(1);
ip.Ip3 = reader.GetInt32(2);
ip.Ip4 = reader.GetInt32(3);
filterQueueItem.ResponseCode = reader.GetInt32(4);
id = reader.GetInt64(5);
}
return rowId;
}
public long GetDiscardedIndexes()
{
long rowId = 0;
filterQueueItem.Ip = ip;
command = new(UpdatePreFilteredStatement, connection);
command.Parameters.AddWithValue("@id", id);
SqliteConnection connection;
SqliteCommand command;
SqliteDataReader reader;
for (int i = 0; i < _discardedConnectionStrings.Count; i++)
{
connection = new(_discardedConnectionStrings[i]);
connection.Open();
command = new(ReadDiscardedSeqIdsStatement, connection);
reader = command.ExecuteReader();
if (!reader.HasRows)
{
return rowId;
}
while (reader.Read())
{
rowId += reader.GetInt64(0);
}
connection.Close();
}
connection = new(_compressedConnectionString);
connection.Open();
command = new(ReadCompressedDbRowsStatement, connection);
reader = command.ExecuteReader();
if (!reader.HasRows)
{
return rowId;
}
while (reader.Read())
{
rowId += reader.GetInt64(0);
}
connection.Close();
connection.Dispose();
command.ExecuteNonQuery();
command.Dispose();
reader.Dispose();
return rowId;
return true;
}
private static long GetDiscardedIndexesForSpecificDb(string connectionString)
@ -706,36 +736,6 @@ public class DbHandler
return true;
}
public List<SearchResult?> GetSearchResults()
{
lock (_readFilteredLock)
{
using SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
using SqliteCommand command = new(ReadFilteredStatement, connection);
using SqliteDataReader reader = command.ExecuteReader();
if (!reader.HasRows)
{
return [];
}
List<SearchResult?> results = [];
while (reader.Read())
{
SearchResult result = new();
result.Title = reader.GetString(0);
result.Url = reader.GetString(1);
results.Add(result);
}
return results;
}
}
public ScannerResumeObject? GetResumeObject(int threadNumber)
{
lock (_readAndDeleteResumeLock)
@ -743,7 +743,7 @@ public class DbHandler
using SqliteConnection connection = new(_resumeConnectionString);
connection.Open();
using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection);
using SqliteCommand command = new(ReadResumeStatement, connection);
command.Parameters.AddWithValue("@threadNumber", threadNumber);
using SqliteDataReader reader = command.ExecuteReader();
@ -764,81 +764,20 @@ public class DbHandler
resumeObject.SecondByte = reader.GetInt32(4);
resumeObject.ThirdByte = reader.GetInt32(5);
resumeObject.FourthByte = reader.GetInt32(6);
resumeObject.Paused = reader.GetBoolean(7);
resumeObject.Completed = reader.GetBoolean(8);
}
return resumeObject;
}
}
public void ReIndex()
{
_pause = true;
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
if (!_paused)
{
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
SqliteCommand command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(ReIndexDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection.Dispose();
command.Dispose();
_pause = false;
_paused = false;
}
public void Vacuum()
{
_pause = true;
Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open.
if (!_paused)
{
Thread.Sleep(5000); // Just for safety.
}
SqliteConnection connection = new(_filteredConnectionString);
connection.Open();
SqliteCommand command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection = new(_unfilteredConnectionString);
connection.Open();
command = new(VacuumDatabasesStatement, connection);
_ = command.ExecuteNonQuery();
connection.Close();
connection.Dispose();
command.Dispose();
_pause = false;
_paused = false;
}
private (string, string) CreateDiscardedDb(int threadNumber)
{
string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db";
string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))";
_discardedConnectionStrings.Add(databaseName);

30
Models/Helper/CompressionHelper.cs
View File

@ -1,4 +1,5 @@
using System.IO.Compression;
using System.Text;
namespace Models.Helper;
@ -8,7 +9,7 @@ public static class CompressionHelper
{
using FileStream originalFileStream = new(sourceFile, FileMode.Open);
using FileStream compressedFileStream = File.Create($"{targetFile}.gz");
using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest);
using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize);
originalFileStream.CopyTo(compressor);
}
@ -19,4 +20,31 @@ public static class CompressionHelper
using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress);
decompressor.CopyTo(decompressedFileStream);
}
public static string CompressString(string text)
{
byte[] byteArray = Encoding.UTF8.GetBytes(text);
using MemoryStream memoryStream = new();
using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize))
{
compressionStream.Write(byteArray, 0, byteArray.Length);
}
return Convert.ToBase64String(memoryStream.ToArray());
}
public static string DecompressString(string compressedText)
{
byte[] byteArray = Convert.FromBase64String(compressedText);
using MemoryStream memoryStream = new(byteArray);
using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress);
using StreamReader reader = new(decompressionStream);
return reader.ReadToEnd();
}
}

6
Models/Model/Backend/CustomProtocolType.cs Normal file
View File

@ -0,0 +1,6 @@
namespace Models.Model.Backend;
public enum CustomProtocolType
{
Icmp = 1
}

10
Models/Model/Backend/DatabaseSizes.cs
View File

@ -1,10 +0,0 @@
namespace Models.Model.Backend;
public struct DatabaseSizes
{
public double DiscardedDbSize { get; set; }
public double FilteredDbSize { get; set; }
public double MyDbSize { get; set; }
}

7
Models/Model/Backend/FilterQueueItem.cs Normal file
View File

@ -0,0 +1,7 @@
namespace Models.Model.Backend;
public struct FilterQueueItem
{
public Ip Ip { get; set; }
public int ResponseCode { get; set; }
}

4
Models/Model/Backend/Filtered.cs
View File

@ -3,10 +3,6 @@ namespace Models.Model.Backend;
public class Filtered
{
public Ip Ip { get; set; }
public string Title1 { get; set; } = "";
public string Title2 { get; set; } = "";
public string Description1 { get; set; } = "";
public string Description2 { get; set; } = "";
public string Url1 { get; set; } = "";
public string Url2 { get; set; } = "";
public int Port1 { get; set; }

5
Models/Model/Backend/IP.cs
View File

@ -14,4 +14,9 @@ public struct Ip
{
return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}";
}
public uint PackIp()
{
return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4;
}
}

9
Models/Model/Backend/RuntimeVariable.cs
View File

@ -1,9 +0,0 @@
namespace Models.Model.Backend;
public enum RuntimeVariable
{
DbContent,
DbDiscarded,
ContentFilter,
ScannerTimeout
}

3
Models/Model/Backend/ScannerResumeObject.cs
View File

@ -9,4 +9,7 @@ public class ScannerResumeObject
public int ThirdByte { get; set; }
public int FourthByte { get; set; }
public int ThreadNumber { get; set; }
public Operations Operation { get; set; }
public bool Paused { get; set; }
public bool Completed { get; set; }
}

8
Models/Model/Backend/SizeUnits.cs
View File

@ -1,8 +0,0 @@
namespace Models.Model.Backend;
public enum SizeUnits
{
Byte,
KB,
MB,
GB,
}

11
Models/Model/External/CommunicationCommand.cs vendored
View File

@ -1,11 +0,0 @@
namespace Models.Model.External;
public enum CommunicationCommand
{
GetScanningProgress,
GetSearches,
StopScanning,
DbReindex,
DbVacuum,
ChangeRuntimeVariable,
}

17
Models/Model/External/CommunicationObject.cs vendored
View File

@ -1,17 +0,0 @@
namespace Models.Model.External;
//[MessagePackObject]
public class CommunicationObject
{
//[Key(0)]
public CommunicationCommand Command { get; set; }
//[Key(1)]
public string? SearchTerm { get; set; } = "";
//[Key(2)]
public string? Variable { get; set; } = "";
//[Key(3)]
public string? VariableValue { get; set; } = "";
}

8
Models/Model/External/CommunicationResult.cs vendored
View File

@ -1,8 +0,0 @@
namespace Models.Model.External;
public class CommunicationResult
{
public List<SearchResult?>? Result { get; set; }
public ScanningStatus? Status { get; set; }
}

18
Models/Model/External/ScanningStatus.cs vendored
View File

@ -1,18 +0,0 @@
namespace Models.Model.External;
public struct ScanningStatus
{
public float PercentageOfIpv4Scanned { get; set; }
public long TotalFiltered { get; set; }
public long AmountOfIpv4Left { get; set; }
public long TotalDiscarded { get; set; }
public double DiscardedDbSize { get; set; }
public double FilteredDbSize { get; set; }
public double MyDbSize { get; set; }
}

6
Models/Model/External/SearchResults.cs vendored
View File

@ -1,6 +0,0 @@
namespace Models.Model.External;
public class SearchResults
{
public List<SearchResult?>? Results { get; set; }
}

5
Models/Models.csproj
View File

@ -1,12 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.10" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.1" />
</ItemGroup>
</Project>

89
Proxy/Program.cs
View File

@ -1,89 +0,0 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using AspNetCoreRateLimit;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.Extensions.Caching.Memory;
using Models.Model.External;
using NetMQ;
using NetMQ.Sockets;
const string myAllowSpecificOrigins = "_myAllowSpecificOrigins";
WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args);
builder.Services.ConfigureHttpJsonOptions(options =>
{
options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default);
});
builder.Services.AddCors(options =>
{
options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader());
});
builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5));
WebApplication app = builder.Build();
app.UseForwardedHeaders(new()
{
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
});
app.UseCors(myAllowSpecificOrigins);
RouteGroupBuilder progressApi = app.MapGroup("/progress");
progressApi.MapGet("/", (IMemoryCache memoryCache) =>
{
const string cacheKey = "progress_status";
if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus))
{
return scanningStatus;
}
CommunicationObject communicationObject = new()
{
Command = CommunicationCommand.GetScanningProgress
};
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(bytes);
byte[] msg = client.ReceiveFrameBytes();
client.Close();
scanningStatus = JsonSerializer.Deserialize<ScanningStatus>(msg);
memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5));
return scanningStatus;
});
RouteGroupBuilder searchApi = app.MapGroup("/search");
searchApi.MapGet("/{term}", (string term) =>
{
CommunicationObject communicationObject = new();
communicationObject.Command = CommunicationCommand.GetSearches;
communicationObject.SearchTerm = term;
byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject);
using RequestSocket client = new();
client.Connect("tcp://127.0.0.1:5556");
client.SendFrame(bytes);
string msg = client.ReceiveFrameString();
client.Close();
return JsonSerializer.Deserialize<SearchResults?>(msg);
});
app.Run();
[JsonSerializable(typeof(ScanningStatus))]
//[JsonSerializable(typeof(SearchResults))]
[JsonSerializable(typeof(CommunicationObject))]
internal partial class AppJsonSerializerContext : JsonSerializerContext
{
}

15
Proxy/Properties/launchSettings.json
View File

@ -1,15 +0,0 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "",
"applicationUrl": "http://localhost:5224",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}

20
Proxy/Proxy.csproj
View File

@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<PublishAot>false</PublishAot>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AspNetCoreRateLimit" Version="5.0.0" />
<PackageReference Include="NetMQ" Version="4.0.1.13" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Models\Models.csproj" />
</ItemGroup>
</Project>

11
Proxy/Proxy.http
View File

@ -1,11 +0,0 @@
@Proxy_HostAddress = http://localhost:5224
GET {{Proxy_HostAddress}}/progress/
Accept: application/json
###
GET {{Proxy_HostAddress}}/search/asd
Accept: application/json
###

8
Proxy/appsettings.Development.json
View File

@ -1,8 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
}
}

9
Proxy/appsettings.json
View File

@ -1,9 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Warning",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*"
}

6
README.md
View File

@ -2,4 +2,8 @@
Rasmus Search Engine
This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol)
This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture.
I'm just trying to minimize memory usage while maximizing performance.

16
RSE.sln
View File

@ -4,9 +4,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend.
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -22,13 +20,9 @@ Global
{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU
{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU
{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU
{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU
{55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU
{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

26
RSE.sln.DotSettings.user
View File

@ -2,14 +2,38 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000002pdb1Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F51b607df472a454cb6ed940749bbfdfd6000_003Fde_003F2c578873_003F02000002pdb1Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000009pdb7Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fbbcfe225942e4131bc589e82ae4b92ab9800_003Fc0_003Fa20d693d_003F02000009pdb7Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Cpdb6Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fb53c196a821648e4ae3b142a6ae58d7b9400_003Fa8_003F21a43479_003F0200000Cpdb6Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A0200000Dpdb9Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003Fe0d70616f09e43d786491f7daf762067ce00_003Fde_003F0a28485b_003F0200000Dpdb9Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003A02000011pdb3Low_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003FILViewer_003F7c3ed02c2ce44598b7f304f8ac45e58f8600_003F6d_003F99b875d1_003F02000011pdb3Low_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueueSegment_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4f72364deba8e6142d1f42386f1fd282d2e8c6bca41d389c28da4c3835d90_003FConcurrentQueueSegment_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb049a159646b52d2dd6ced21de315f79dff86421243e94ffd4f29c6f7e4df25_003FConcurrentQueue_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AConcurrentQueue_00601_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F85_003Fa8fb60ce_003FConcurrentQueue_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADirectoryInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fe4ec446cfe0489bc3ef68a45c6766d183e999ebdc657e94fb1ad059de2bb9_003FDirectoryInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADnsEndPoint_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F3d_003F64a36162_003FDnsEndPoint_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbf9021a960b74107a7e141aa06bc9d8a0a53c929178c2fb95b1597be8af8dc_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AFileSystemEnumerator_002EUnix_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F233863917bb42f133182fb4926e94ef8139c6f704da0c4574a8de3209f4761_003FFileSystemEnumerator_002EUnix_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpResponseMessage_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F85e97f467d698c9e98eae9e3a1b39d58541173e57992d8f7111eabdd3db3526_003FHttpResponseMessage_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIcmpV6MessageType_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F4e_003Fec7b627f_003FIcmpV6MessageType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddressParser_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5a18f227dadd72bd8268cdb333cd70aa19d8663c3610d541cda4fd0199acbf4_003FIPAddressParser_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F7aa62fd0fbc144a6818ff7b2fd2626dc34800_003F48_003Fb056cfcf_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPAddress_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fdcb058a821641cccb63e4a61914bd75ed5d336dda19353b41994ef1159c85bec_003FIPAddress_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIPStatus_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F6079bfa8f3c0474bb21ade75ce5d010318000_003F15_003F57afed32_003FIPStatus_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APingReply_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F76f05f4da452fadb1ab24cdb83dccb74b6e6484519e28acc6ce2c02c7aabac24_003FPingReply_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003F44_003F939f4a86_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APing_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3080b18e3637ea741b5b65abd6aee06e41494a82a58b3e2ed87d4ddb5cc62_003FPing_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4682fe5857f946a98dcb2cd6f0a403983f200_003F3d_003Fca9070b2_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARateLimitRule_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8fbca8b1bca27d45830c443b2c773d979015ea216430366f285514a39fc0b9_003FRateLimitRule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARawSocketPermissions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0a3c9a6c6f0343119978ec009640fbbb18000_003Fc7_003Fb8dcfca9_003FRawSocketPermissions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc90597e9198b448fad8f1fd970196b198c600_003Ffc_003F32090d76_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASocket_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fed603888eeb1f16770cbbbb8321a5999df9d8962d9b5cb4d5de621123659_003FSocket_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteCommand_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F7af32a60b614a4736554243e7b8aba5c9a167efe6e7254e6648651482183_003FSqliteCommand_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Ff74c15be4ebfb9eebdf1d29c607aa58a3dd7cfccafcbf075c7887256d97ccb40_003FSqliteDataReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteDataRecord_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fb6b6fc545335beb5092e3c9821a4d1f34b5ef02f42bcfb7c872755f5d8f19_003FSqliteDataRecord_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteException_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F154220569126135ad5d7314bf2bc694d3cf7c95840d481d44f0336f4f1f8e9c_003FSqliteException_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASqliteParameterCollection_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F56cf675b4777645c714ae85e12bde2163da8ec62d2a23f8b35ef357547a9_003FSqliteParameterCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStartupExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ce5d581dd9cc0e4cdfd914e797ba2da05e894767d76b86f0515ef5226bac_003FStartupExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AStreamReader_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fb8417950f7e54f048304227d9faf80e9d1be00_003F53_003F95e505fc_003FStreamReader_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002ESearching_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F49ee52518952e16b89adee3d6c9346ae6c74be268730f0497eb14b34b49d56c_003FString_002ESearching_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskToAsyncResult_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F66952b4c92d2c944c42ecf9237964f8d12a6feb1734b428a866a643c391da59_003FTaskToAsyncResult_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATCPClient_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F48f66bd9377db6244f6f84da3534394e9416923c69d34598df3ea5864e75d_003FTCPClient_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AThread_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F693e634d7742afaf486acd69d84fe2a9e1ee1b11ba84f29cd1d67668d20dd59_003FThread_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003F_002E_002E_003F_002Econfig_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F9f559e1a197aae3f0e896a08ca2436e5665aa18fbf73266415afd59de3b943_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String></wpf:ResourceDictionary>

BIN
Type Dependencies Diagram for Communication and other elements.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.7 MiB

7
global.json Normal file
View File

@ -0,0 +1,7 @@
{
"sdk": {
"version": "9.0.0",
"rollForward": "latestMajor",
"allowPrerelease": true
}
}