diff --git a/.gitignore b/.gitignore index 30c8ce1..a13d7d8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ **/**/bin/* **/obj/* **/**/obj/* +*.gz # User-specific stuff .idea/**/workspace.xml diff --git a/Analyze/Analyze.csproj b/Analyze/Analyze.csproj new file mode 100644 index 0000000..4f9b42c --- /dev/null +++ b/Analyze/Analyze.csproj @@ -0,0 +1,14 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + diff --git a/Analyze/Program.cs b/Analyze/Program.cs new file mode 100644 index 0000000..e5dff12 --- /dev/null +++ b/Analyze/Program.cs @@ -0,0 +1,3 @@ +// See https://aka.ms/new-console-template for more information + +Console.WriteLine("Hello, World!"); \ No newline at end of file diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj index 95c5c1e..96b89cb 100644 --- a/Backend/Backend.csproj +++ b/Backend/Backend.csproj @@ -2,22 +2,15 @@ Exe - net8.0 + net9.0 enable enable - - - false + false + true - - - - - - diff --git a/Backend/Handler/Communication.cs b/Backend/Handler/Communication.cs deleted file mode 100644 index f0aef9c..0000000 --- a/Backend/Handler/Communication.cs +++ /dev/null @@ -1,201 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Text.Json; -using Backend.Helper; -using Models.Handler; -using Models.Model.Backend; -using Models.Model.External; -using NetMQ; -using NetMQ.Sockets; - -namespace Backend.Handler; - -public class Communication -{ - private readonly DbHandler _dbHandler; - private readonly ThreadHandler _threadHandler; - private readonly IpScanner _ipScanner; - private readonly ContentFilter _contentFilter; - private bool _isRunning = true; - private string _basePath; - - public Communication(DbHandler dbHandler, ThreadHandler threadHandler, IpScanner ipScanner, ContentFilter contentFilter, string basePath) - { - _dbHandler = dbHandler; - _threadHandler = threadHandler; - _ipScanner = ipScanner; - _contentFilter = contentFilter; - _basePath = basePath; - } - - public WaitHandle[] Start() - { - WaitHandle[] waitHandles = new WaitHandle[1]; - EventWaitHandle handle = new(false, EventResetMode.ManualReset); - waitHandles[0] = handle; - - Thread thread = new(Server!); - thread.Start(handle); - - return waitHandles; - } - - private void Server(object obj) - { - using ResponseSocket rep = new(); - - //rep.Options.IPv4Only = true; - - rep.Bind("tcp://127.0.0.1:5556"); - - while (_isRunning) - { - byte[] message = rep.ReceiveFrameBytes(); - - CommunicationObject? communicationObject = JsonSerializer.Deserialize(message); - - //rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success")); - - if (communicationObject is null) - { - continue; - } - - OnServerOnReceiveReady(communicationObject, rep); - } - - ((EventWaitHandle) obj).Set(); - } - - [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")] - [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.Serialize(TValue, JsonSerializerOptions)")] - private void OnServerOnReceiveReady(CommunicationObject communicationObject, ResponseSocket rep) - { - switch (communicationObject.Command) - { - case CommunicationCommand.GetScanningProgress: - { - DatabaseSizes databaseSizes = FilesystemHelper.GetDatabaseSizes(_basePath); - - long discardedIndexes = _dbHandler.GetDiscardedIndexes(); - - ScanningStatus status = new(); - // 4294967296 is all Ipv4 addresses. - - if (discardedIndexes != 0) - { - status.PercentageOfIpv4Scanned = (float)discardedIndexes / 4294967296 * 100; - } - else - { - // This is a workaround for the frontend not understanding a 0f as an actual float, so we use a very small float. - status.PercentageOfIpv4Scanned = 0.0000000001f; - } - - status.AmountOfIpv4Left = 4294967296 - discardedIndexes; - status.TotalFiltered = _dbHandler.GetFilteredIndexes(); - status.TotalDiscarded = discardedIndexes; - status.MyDbSize = databaseSizes.MyDbSize; - status.FilteredDbSize = databaseSizes.FilteredDbSize; - status.DiscardedDbSize = databaseSizes.DiscardedDbSize; - - byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(status); - - rep.SendFrame(serializedResult); - - break; - } - - case CommunicationCommand.DbReindex: - { - _dbHandler.ReIndex(); - - SendStringResponse(rep, "All Dbs have been reindexed."); - break; - } - - case CommunicationCommand.DbVacuum: - { - _dbHandler.Vacuum(); - - SendStringResponse(rep, "All Dbs have been vacuumed."); - break; - } - - case CommunicationCommand.GetSearches: - { - if (!string.IsNullOrWhiteSpace(communicationObject.SearchTerm)) - { - SendSearchResponse(rep, communicationObject.SearchTerm); - } - - break; - } - - case CommunicationCommand.StopScanning: - { - _isRunning = false; - break; - } - - case CommunicationCommand.ChangeRuntimeVariable: - { - if (string.IsNullOrWhiteSpace(communicationObject.VariableValue)) break; - - if (communicationObject.Variable == RuntimeVariable.DbContent.ToString()) - { - int value = int.Parse(communicationObject.VariableValue); - _dbHandler.SetContentWaitTime(value); - } - - if (communicationObject.Variable == RuntimeVariable.DbDiscarded.ToString()) - { - int value = int.Parse(communicationObject.VariableValue); - _dbHandler.SetDiscardedWaitTime(value); - } - - if (communicationObject.Variable == RuntimeVariable.ScannerTimeout.ToString()) - { - int value = int.Parse(communicationObject.VariableValue); - _ipScanner.SetTimeout(value); - } - - if (communicationObject.Variable == RuntimeVariable.ContentFilter.ToString()) - { - int value = int.Parse(communicationObject.VariableValue); - _contentFilter.SetTimeout(value); - } - - rep.SendFrame(JsonSerializer.SerializeToUtf8Bytes("Success")); - - break; - } - } - } - - private static void SendStringResponse(ResponseSocket rep, string response) - { - byte[] serializedResult = JsonSerializer.SerializeToUtf8Bytes(response); - - rep.SendFrame(serializedResult); - } - - private static void SendSearchResponse(ResponseSocket rep, string searchTerm) - { - //SearchResults result = SearchHelper.Search(communicationObject.SearchTerm!, _dbHandler); - SearchResults result = new() - { - Results = [] - }; - SearchResult lol = new() - { - Url = "Remember to use an actual search tearm. Like 'dotnet 9.0'", - Title = "Remember to use an actual search tearm. Like 'dotnet 9.0'", - }; - - result.Results.Add(lol); - - string serializedResult = JsonSerializer.Serialize(result); - - rep.SendFrame(serializedResult); - } -} \ No newline at end of file diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index b6930b6..2531f4b 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -6,27 +6,44 @@ using Models.Model.Backend; namespace Backend.Handler; +public class Content +{ + public int Port1 { get; set; } + public int Port2 { get; set; } + public Ip Ip { get; set; } +} + +public class ContentThread +{ + public int ThreadId { get; set; } + public EventWaitHandle? EventWaitHandle { get; set; } +} + public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _contentQueue = new(); private readonly DbHandler _dbHandler; + private readonly ThreadHandler _threadHandler; private readonly string _getDomainPort80; private readonly string _getDomainPort443; private bool _stop; private int _timeOut; private readonly string _basePath; - - public ContentFilter(ConcurrentQueue queue, ConcurrentQueue unfilteredQueue, DbHandler dbHandler, string basePath) + + public ContentFilter(ConcurrentQueue queue, ConcurrentQueue unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler) { _queue = queue; _dbHandler = dbHandler; _basePath = basePath; + _threadHandler = threadHandler; _unfilteredQueue = unfilteredQueue; _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; + SetTimeout(3000); } @@ -52,11 +69,23 @@ public class ContentFilter while (!_stop) { List indexes = _dbHandler.GetUnfilteredIndexes(); + + if (indexes.Count == 0) + { + _stop = true; + _threadHandler.Stop(); + break; + } for (int i = 0; i < indexes.Count; i++) { if (_stop) break; + if (_contentQueue.Count >= 500) + { + Thread.Sleep(1000); + } + Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]); if (unfiltered.Filtered) continue; @@ -77,13 +106,13 @@ public class ContentFilter { continue; } + + Content content = new(); + content.Ip = ip; + content.Port1 = unfiltered.Port1; + content.Port2 = unfiltered.Port2; - Filtered filtered = GetSiteData(ip); - - filtered.Port1 = unfiltered.Port1; - filtered.Port2 = unfiltered.Port2; - - _queue.Enqueue(filtered); + _contentQueue.Enqueue(content); } Thread.Sleep(_timeOut); @@ -91,18 +120,65 @@ public class ContentFilter ((EventWaitHandle) obj).Set(); } - - private Filtered GetSiteData(Ip ip) + + public WaitHandle[] StartFilterThread(int threads) { - StartProcess(ip, 80); - StartProcess(ip, 443); + WaitHandle[] waitHandle = new WaitHandle[threads]; + + for (int i = 0; i < threads; i++) + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + ContentThread contentThread = new(); + contentThread.ThreadId = i; + contentThread.EventWaitHandle = handle; + waitHandle[i] = handle; + + Thread thread = new(FilterThread!); + thread.Start(contentThread); + + Thread.Sleep(8); + } + + return waitHandle; + } + + private void FilterThread(object obj) + { + Console.WriteLine("Filter Thread started."); + ContentThread thread = (ContentThread) obj; + + while (!_stop) + { + if (_contentQueue.IsEmpty) + { + Thread.Sleep(1000); + } + + _contentQueue.TryDequeue(out Content? content); + + if (content is null) + { + continue; + } + + Filtered filtered = GetSiteData(content.Ip, thread.ThreadId); + + filtered.Port1 = content.Port1; + filtered.Port2 = content.Port2; + + _queue.Enqueue(filtered); + } + + thread.EventWaitHandle!.Set(); + } + + private Filtered GetSiteData(Ip ip, int threadId) + { + StartProcess(ip, 80, threadId); + StartProcess(ip, 443, threadId); string url1 = ""; string url2 = ""; - string title1 = ""; - string title2 = ""; - string description1 = ""; - string description2 = ""; bool robotsTxt1 = false; bool robotsTxt2 = false; string serverType1 = ""; @@ -128,7 +204,7 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt"); + using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt"); while (streamReader.Peek() != -1) { @@ -161,37 +237,6 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - if (ports[i] == 80) - { - if (string.IsNullOrWhiteSpace(url1)) continue; - - try - { - (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult(); - title1 = temp.Item1; - description1 = temp.Item2; - } - catch - { - // - } - } - else - { - if (string.IsNullOrWhiteSpace(url2)) continue; - - try - { - (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult(); - title2 = temp.Item1; - description2 = temp.Item2; - } - catch - { - // - } - } - if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); } if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); } } @@ -201,10 +246,6 @@ public class ContentFilter Ip = ip, Url1 = url1, Url2 = url2, - Title1 = title1, - Title2 = title2, - Description1 = description1, - Description2 = description2, ServerType1 = serverType1, ServerType2 = serverType2, RobotsTXT1 = robotsTxt1, @@ -230,7 +271,7 @@ public class ContentFilter return siteData; } - private void StartProcess(Ip ip, int port) + private void StartProcess(Ip ip, int port, int threadId) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; @@ -238,7 +279,7 @@ public class ContentFilter proc.StartInfo = new() { FileName = "/bin/bash", - Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt", + Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, diff --git a/Backend/Handler/IpFilterHandler.cs b/Backend/Handler/IpFilterHandler.cs new file mode 100644 index 0000000..fe46813 --- /dev/null +++ b/Backend/Handler/IpFilterHandler.cs @@ -0,0 +1,227 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using Backend.Helper; +using Models.Handler; +using Models.Model.Backend; + +namespace Backend.Handler; + +public class IpFilterHandler +{ + private readonly ConcurrentQueue _discardedQueue; + private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _preFilteredQueue; + private DbHandler _dbHandler; + private ThreadHandler _threadHandler; + private bool _stop; + private bool _fillerStop; + private bool _stopAutoscaledThreads; + private int _timeout; + + public IpFilterHandler(ConcurrentQueue discardedQueue, + ConcurrentQueue unfilteredQueue, + ConcurrentQueue preFilteredQueue, + DbHandler dbHandler, ThreadHandler threadHandler) + { + _discardedQueue = discardedQueue; + _unfilteredQueue = unfilteredQueue; + _preFilteredQueue = preFilteredQueue; + _dbHandler = dbHandler; + _threadHandler = threadHandler; + + _timeout = 16; + } + + public List Start(int threadCount) + { + WaitHandle[] waitHandle = new WaitHandle[64]; + + int counter = 0; + + List waitHandles = []; + + for (int i = 0; i < threadCount; i++) + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + + if (counter < 64) + { + waitHandle[counter] = handle; + counter++; + + Thread f = new (Filter!); + f.Start(handle); + + Console.WriteLine($"Filter thread ({i}) started"); + Thread.Sleep(16); + + continue; + } + + counter = 0; + + waitHandles.Add(waitHandle); + + waitHandle = new WaitHandle[64]; + } + + return waitHandles; + } + + public void AutoScaler() + { + int i = 0; + int j = 0; + + while (!_stop) + { + if (_preFilteredQueue.Count >= 2000) + { + if (i == 10) + { + _stopAutoscaledThreads = false; + Console.WriteLine("Autoscaler started"); + + while (!_stopAutoscaledThreads) + { + if (_preFilteredQueue.Count <= 2000) + { + if (j == 1000) + { + _stopAutoscaledThreads = true; + } + + j++; + + Thread.Sleep(128); + } + else + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + Thread f = new (Filter_AutoScaler!); + f.Start(handle); + Thread.Sleep(16); + } + } + } + + i++; + } + else + { + i = 0; + j = 0; + } + + Thread.Sleep(128); + } + } + + private void Filter(object obj) + { + int counter = 0; + while (!_stop) + { + if (_preFilteredQueue.IsEmpty && _fillerStop) + { + if (counter == 100) + { + _threadHandler.Stop(); + _stop = true; + } + + counter++; + Thread.Sleep(128); + } + + _preFilteredQueue.TryDequeue(out FilterQueueItem item); + + (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443); + + if (ports is { Item1: 0, Item2: 0 }) + { + _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode)); + continue; + } + + _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports)); + } + + ((EventWaitHandle) obj).Set(); + } + + public void FillFilterQueue() + { + Console.WriteLine("Fill FilterQueue started."); + while (!_stop) + { + if (_preFilteredQueue.Count > 500) continue; + + if (_dbHandler.GetPreFilterQueueItem(out FilterQueueItem item)) + { + _preFilteredQueue.Enqueue(item); + } + else + { + _fillerStop = true; + } + } + } + + private void Filter_AutoScaler(object obj) + { + while (!_stopAutoscaledThreads) + { + if (_preFilteredQueue.IsEmpty) + { + Thread.Sleep(_timeout); + continue; + } + + _preFilteredQueue.TryDequeue(out FilterQueueItem item); + + (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443); + + if (ports is { Item1: 0, Item2: 0 }) + { + _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode)); + continue; + } + + _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports)); + } + + ((EventWaitHandle) obj).Set(); + } + + private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) + { + return new() + { + Ip = ip, + ResponseCode = responseCode + }; + } + + private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) + { + Unfiltered unfiltered = new() + { + Ip = ip, + Port1 = ports.Item1, + Port2 = ports.Item2, + Filtered = false + }; + + return new() + { + Unfiltered = unfiltered, + Operations = Operations.Insert + }; + } + + public void Stop() + { + _stop = true; + } +} \ No newline at end of file diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index cd4f360..8afa19c 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -1,7 +1,11 @@ +using System.Buffers.Binary; using System.Collections.Concurrent; using System.Net; using System.Net.NetworkInformation; +using System.Numerics; +using System.Runtime.InteropServices; using Backend.Helper; +using Models.Experimental; using Models.Handler; using Models.Model.Backend; @@ -18,27 +22,22 @@ public class ScanSettings public class IpScanner { private readonly ConcurrentQueue _discardedQueue; - private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _preFilteredQueue; private readonly ConcurrentQueue _resumeQueue; private readonly DbHandler _dbHandler; private bool _stop; - private int _timeout; + private readonly int _timeout; - public IpScanner(ConcurrentQueue unfilteredQueue, ConcurrentQueue discardedQueue, - ConcurrentQueue resumeQueue, DbHandler dbHandler - ) + public IpScanner(ConcurrentQueue discardedQueue, + ConcurrentQueue resumeQueue, DbHandler dbHandler, + ConcurrentQueue preFilteredQueue) { _dbHandler = dbHandler; + _preFilteredQueue = preFilteredQueue; _discardedQueue = discardedQueue; - _unfilteredQueue = unfilteredQueue; _resumeQueue = resumeQueue; - - SetTimeout(64); - } - - public void SetTimeout(int milliseconds) - { - _timeout = milliseconds; + + _timeout = 32; } public List Start(int threads) @@ -76,7 +75,7 @@ public class IpScanner f.Start(scanSettings); Console.WriteLine($"Scanner thread ({i}) started"); - Thread.Sleep(100); + Thread.Sleep(128); continue; } @@ -108,16 +107,26 @@ public class IpScanner if (resumeNow is not null) { + if (resumeNow.Completed) + { + return; + } + scanSettings.Start = resumeNow.FirstByte; scanSettings.End = resumeNow.EndRange; secondByte = resumeNow.SecondByte; thirdByte = resumeNow.ThirdByte; fourthByte = resumeNow.FourthByte; } + else + { + CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, scanSettings.Start, secondByte, thirdByte, fourthByte, false, false, Operations.Insert); + } // Empty buffer so we use the lowest abstracted ping.Send() method. byte[] buf = []; using Ping ping = new(); + int x = 0; for (int i = scanSettings.Start; i < scanSettings.End; i++) { @@ -136,19 +145,34 @@ public class IpScanner if (i == 192 && k == 2) continue; if (i == 192 && j == 88 && k == 99) continue; - if (_discardedQueue.Count >= 2000) + if (_discardedQueue.Count >= 20_000) { - Thread.Sleep(500); + Thread.Sleep(1000); + } + + if (_preFilteredQueue.Count >= 20_000) + { + Thread.Sleep(1000); } for (int l = fourthByte; l < 256; l++) { + if (x == 75_000) + { + CreateResumeObject(scanSettings.ThreadNumber, scanSettings.Start, scanSettings.End, i, j, k, l, false, false, Operations.Update); + x = 0; + } + + x++; + if (_stop) { resumeObject.FourthByte = l; break; } - + + + Ip ip = new() { Ip1 = i, @@ -156,19 +180,29 @@ public class IpScanner Ip3 = k, Ip4 = l }; - + IPStatus responseCode = IPStatus.Unknown; try { // Sometimes, if the pinger gets a Destination Unreachable Communication administratively prohibited response, the pinger will throw an exception. // https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol?useskin=vector#Control_messages - _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address); - if (address is not null) + //_ = IPAddress.TryParse(ip.ToString(), out IPAddress? address); + + responseCode = CustomPing.SendIcmpEchoRequestOverRawSocket(Parse(ip.ToString()), _timeout); + + /*if (l % 2 == 0) { - responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status; - Thread.Sleep(_timeout); + responseCode = IPStatus.Success; } + else + { + responseCode = IPStatus.TimedOut; + }*/ + + //responseCode = IPStatus.TimedOut; + + //Thread.Sleep(0); } catch { @@ -181,15 +215,7 @@ public class IpScanner continue; } - (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443); - - if (ports is { Item1: 0, Item2: 0 }) - { - _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); - continue; - } - - _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports)); + _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode)); } if (_stop) @@ -211,14 +237,42 @@ public class IpScanner resumeObject.FirstByte = i; break; } - - Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})"); } + Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) stopped."); + + if (_stop) + { + resumeObject.Paused = true; + } + else + { + resumeObject.Completed = true; + } + + resumeObject.Operation = Operations.Update; + _resumeQueue.Enqueue(resumeObject); scanSettings.Handle!.Set(); } + + private void CreateResumeObject(int threadNumber, int startRange, int endRange, int firstByte, int secondByte, int thirdByte, int fourthByte, bool paused, bool completed, Operations operation) + { + ScannerResumeObject resumeObject = new(); + resumeObject.ThreadNumber = threadNumber; + resumeObject.StartRange = startRange; + resumeObject.EndRange = endRange; + resumeObject.FirstByte = firstByte; + resumeObject.SecondByte = secondByte; + resumeObject.ThirdByte = thirdByte; + resumeObject.FourthByte = fourthByte; + resumeObject.Paused = paused; + resumeObject.Completed = completed; + resumeObject.Operation = operation; + + _resumeQueue.Enqueue(resumeObject); + } private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) { @@ -229,25 +283,114 @@ public class IpScanner }; } - private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) + private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode) { - Unfiltered unfiltered = new() + FilterQueueItem filterQueueItem = new() { Ip = ip, - Port1 = ports.Item1, - Port2 = ports.Item2, - Filtered = false + ResponseCode = responseCode }; - return new() - { - Unfiltered = unfiltered, - Operations = Operations.Insert - }; + return filterQueueItem; } public void Stop() { _stop = true; } + + private static unsafe IPAddress Parse(ReadOnlySpan ipSpan) + { + int length = ipSpan.Length; + long nonCanonical; + fixed (char* name = &MemoryMarshal.GetReference(ipSpan)) + nonCanonical = ParseNonCanonical(name, 0, ref length, true); + + return new IPAddress(BitOperations.RotateRight((uint)nonCanonical & 16711935U, 8) + BitOperations.RotateLeft((uint)nonCanonical & 4278255360U, 8)); + } + + private static unsafe long ParseNonCanonical(char* name, int start, ref int end, bool notImplicitFile) + { + long* numPtr = stackalloc long[4]; + long num1 = 0; + bool flag = false; + int index1 = 0; + int index2; + for (index2 = start; index2 < end; ++index2) + { + char ch = name[index2]; + num1 = 0L; + int num2 = 10; + if (ch == '0') + { + num2 = 8; + ++index2; + flag = true; + if (index2 < end) + { + switch (name[index2]) + { + case 'X': + case 'x': + num2 = 16; + ++index2; + flag = false; + break; + } + } + } + for (; index2 < end; ++index2) + { + char c = name[index2]; + int num3; + if ((num2 == 10 || num2 == 16) && char.IsAsciiDigit(c)) + num3 = (int) c - 48; + else if (num2 == 8 && '0' <= c && c <= '7') + num3 = (int) c - 48; + else if (num2 == 16 && 'a' <= c && c <= 'f') + num3 = (int) c + 10 - 97; + else if (num2 == 16 && 'A' <= c && c <= 'F') + num3 = (int) c + 10 - 65; + else + break; + num1 = num1 * (long) num2 + (long) num3; + if (num1 > (long) uint.MaxValue) + return -1; + flag = true; + } + if (index2 < end && name[index2] == '.') + { + if (index1 >= 3 || !flag || num1 > (long) byte.MaxValue) + return -1; + numPtr[index1] = num1; + ++index1; + flag = false; + } + else + break; + } + if (!flag) + return -1; + if (index2 < end) + { + char ch; + if ((ch = name[index2]) != '/' && ch != '\\' && (!notImplicitFile || ch != ':' && ch != '?' && ch != '#')) + return -1; + end = index2; + } + numPtr[index1] = num1; + switch (index1) + { + case 0: + return numPtr[0] > (long) uint.MaxValue ? -1L : numPtr[0]; + case 1: + return numPtr[1] > 16777215L ? -1L : numPtr[0] << 24 | numPtr[1] & 16777215L; + case 2: + return numPtr[2] > (long) ushort.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | numPtr[2] & (long) ushort.MaxValue; + case 3: + return numPtr[3] > (long) byte.MaxValue ? -1L : numPtr[0] << 24 | (numPtr[1] & (long) byte.MaxValue) << 16 | (numPtr[2] & (long) byte.MaxValue) << 8 | numPtr[3] & (long) byte.MaxValue; + default: + return -1; + } + } } \ No newline at end of file diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 4cebb5a..e819f51 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -7,57 +7,108 @@ namespace Backend.Handler; public class ThreadHandler { private readonly DbHandler _dbHandler; - private readonly Communication _communication; private readonly IpScanner _ipScanner; private readonly ContentFilter _contentFilter; + private readonly IpFilterHandler _ipFilterHandler; - private bool _communicationStopped; private bool _ipScannerStopped; private bool _contentFilterStopped; + private bool _ipFilterStopped; + + private bool _stage1 = true; + private bool _stage2; + private bool _stage3; + + ConcurrentQueue filteredQueue = new(); + ConcurrentQueue discardedQueue = new(); + ConcurrentQueue unfilteredQueue = new(); + ConcurrentQueue scannerResumeQueue = new(); + ConcurrentQueue preFilteredQueue = new(); public ThreadHandler(string path) { - ConcurrentQueue filteredQueue = new(); - ConcurrentQueue discardedQueue = new(); - ConcurrentQueue unfilteredQueue = new(); - ConcurrentQueue scannerResumeQueue = new(); - - _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path); - _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler); - _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path); - _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path); + _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, preFilteredQueue, path); + _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue); + _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path, this); + _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue, _dbHandler, this); } public void Start() { Thread scanner = new(StartScanner); + Thread ipFilter = new(StartIpFilter); Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); Thread discarded = new(StartDiscardedDbHandler); Thread filtered = new(StartFilteredDbHandler); Thread resume = new(StartResumeDbHandler); - Thread communication = new(StartCommunicationHandler); - - scanner.Start(); - indexer.Start(); - database.Start(); - discarded.Start(); - filtered.Start(); - resume.Start(); - communication.Start(); - - scanner.Join(); - indexer.Join(); - database.Join(); - discarded.Join(); - filtered.Join(); - resume.Join(); - communication.Join(); + Thread ipFilterAutoScaler = new(StartIpFilterAutoScaler); + Thread contentFilterThread = new(StartContentFilterThread); + Thread prefilterDb = new(StartPreFilterDbHandler); + Thread fillIpFilterQueue = new(StartFillIpFilterQueue); + //Thread check = new(CheckQueue); + + if (_stage1) + { + discarded.Start(); // de-queues from discardedQueue + prefilterDb.Start(); // de-queues from preFilteredQueue + scanner.Start(); // en-queues to discardedQueue and preFilteredQueue + resume.Start(); // de-queues from resumeQueue + + scanner.Join(); + Stop(); + + discarded.Join(); + prefilterDb.Join(); + resume.Join(); + } + + if (_stage2) + { + database.Start(); // de-queues from unfilteredQueue + discarded.Start(); // de-queues from discardedQueue + ipFilter.Start(); // en-queues to discardedQueue and unfilteredQueue + ipFilterAutoScaler.Start(); // de-queues from preFilteredQueue, en-queues to discardedQueue and unfilteredQueue + fillIpFilterQueue.Start(); // reads from preFiltered database, en-queues to preFilteredQueue + + database.Join(); + discarded.Join(); + ipFilter.Join(); + ipFilterAutoScaler.Join(); + fillIpFilterQueue.Join(); + } + + if (_stage3) + { + filtered.Start(); // de-queues from filteredQueue + database.Start(); // de-queues from unfilteredQueue + indexer.Start(); // en-queues to unfilteredQueue and contentQueue + contentFilterThread.Start(); // de-queues from contentQueue, en-queues to filteredQueue + + contentFilterThread.Join(); + filtered.Join(); + database.Join(); + indexer.Join(); + } + } + + private void CheckQueue() + { + while (true) + { + Console.Clear(); + Console.WriteLine($"filteredQueue.Count: {filteredQueue.Count}"); + Console.WriteLine($"discardedQueue.Count: {discardedQueue.Count}"); + Console.WriteLine($"unfilteredQueue.Count: {unfilteredQueue.Count}"); + Console.WriteLine($"scannerResumeQueue.Count: {scannerResumeQueue.Count}"); + Console.WriteLine($"preFilteredQueue.Count: {preFilteredQueue.Count}"); + Thread.Sleep(5); + } } private void StartScanner() { - Thread.Sleep(5000); // Let the database handler instantiate and warm up first. + Thread.Sleep(15000); // Let the database handler instantiate and warm up first. List wait = _ipScanner.Start(256); @@ -67,8 +118,6 @@ public class ThreadHandler } Console.WriteLine("Scanner finished"); - - _ipScannerStopped = true; } private void StartContentFilter() @@ -83,6 +132,39 @@ public class ThreadHandler _contentFilterStopped = true; } + + private void StartContentFilterThread() + { + WaitHandle[] wait = _contentFilter.StartFilterThread(64); + + WaitHandle.WaitAll(wait); + } + + private void StartIpFilterAutoScaler() + { + _ipFilterHandler.AutoScaler(); + } + + private void StartFillIpFilterQueue() + { + _ipFilterHandler.FillFilterQueue(); + } + + private void StartIpFilter() + { + Thread.Sleep(1000); + + List wait = _ipFilterHandler.Start(256); + + for (int i = 0; i < wait.Count; i++) + { + WaitHandle.WaitAll(wait[i]); + } + + Console.WriteLine("Ip filter finished"); + + _ipFilterStopped = true; + } private void StartDbHandler() { @@ -93,6 +175,11 @@ public class ThreadHandler { _dbHandler.FilteredDbHandler(); } + + private void StartPreFilterDbHandler() + { + _dbHandler.PrefilteredDbHandler(); + } private void StartResumeDbHandler() { @@ -101,42 +188,26 @@ public class ThreadHandler private void StartDiscardedDbHandler() { - WaitHandle[] wait = _dbHandler.Start(5); + WaitHandle[] wait = _dbHandler.Start(4); WaitHandle.WaitAll(wait); Console.WriteLine("Discarded DbHandler finished"); } - private void StartCommunicationHandler() - { - WaitHandle[] wait = _communication.Start(); - - WaitHandle.WaitAll(wait); - - Console.WriteLine("Communicator finished"); - - _communicationStopped = true; - - Stop(); - } - - private void Stop() + public void Stop() { + Console.WriteLine("Stopping..."); _ipScanner.Stop(); _contentFilter.Stop(); + _ipFilterHandler.Stop(); + Console.WriteLine("Stopping Extra..."); - bool stopping = true; + Thread.Sleep(30_000); - while (stopping) - { - if (_communicationStopped && _ipScannerStopped && _contentFilterStopped) - { - _dbHandler.Stop(); - stopping = false; - } - - Thread.Sleep(3000); - } + Console.WriteLine("Stopping Super Extra..."); + _dbHandler.Stop(); + + Console.WriteLine("Stopped."); } } \ No newline at end of file diff --git a/Backend/Helper/FilesystemHelper.cs b/Backend/Helper/FilesystemHelper.cs deleted file mode 100644 index b74cd9b..0000000 --- a/Backend/Helper/FilesystemHelper.cs +++ /dev/null @@ -1,43 +0,0 @@ -using Models.Model.Backend; - -namespace Backend.Helper; - -public static class FilesystemHelper -{ - public static DatabaseSizes GetDatabaseSizes(string basePath) - { - DatabaseSizes databaseSizes = new(); - - FileInfo fileInfo = new($"{basePath}/Models/mydb.db"); - databaseSizes.MyDbSize = fileInfo.Length.ToSize(SizeUnits.KB); - - databaseSizes.DiscardedDbSize = GetDiscardedDbSizes(basePath).ToSize(SizeUnits.KB); - - fileInfo = new($"{basePath}/Models/Filtered.db"); - databaseSizes.FilteredDbSize = fileInfo.Length.ToSize(SizeUnits.KB); - - return databaseSizes; - } - - private static long GetDiscardedDbSizes(string basePath) - { - string folder = $"{basePath}/Models"; - string[] files = Directory.GetFiles(folder, "*.db"); - - long size = 0; - - for (int i = 0; i < files.Length; i++) - { - if (!files[i].Contains("Discarded")) continue; - FileInfo fileInfo = new(files[i]); - size += fileInfo.Length; - } - - return size; - } - - private static double ToSize(this long value, SizeUnits unit) - { - return double.Parse((value / Math.Pow(1024, (long)unit)).ToString("0.00")); - } -} \ No newline at end of file diff --git a/Backend/Helper/FilterHelper.cs b/Backend/Helper/FilterHelper.cs index 1fa04c2..c314b86 100644 --- a/Backend/Helper/FilterHelper.cs +++ b/Backend/Helper/FilterHelper.cs @@ -2,12 +2,10 @@ using System.Text.RegularExpressions; namespace Backend.Helper; -public static partial class FilterHelper +public static class FilterHelper { // https://stackoverflow.com/a/56116499 private const string DomainPattern = @"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$"; - private const string TitlePattern = "(.*)"; - private const string DescriptionPattern = " GetTitleAndDescription(string url, int port) - { - using HttpClient client = new(); - - if (port == 80) - { - try - { - client.BaseAddress = new($"http://{url}"); - } - catch - { - // - } - } - else - { - try - { - client.BaseAddress = new($"https://{url}"); - } - catch - { - // - } - } - - client.DefaultRequestHeaders.Accept.Clear(); - client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgentHeader); - client.Timeout = TimeSpan.FromSeconds(30); - - HttpResponseMessage? response; - - try - { - response = await client.GetAsync("/"); - } - catch - { - return ("", ""); - } - - if (!response.IsSuccessStatusCode) - { - return ("", ""); - } - - string html = await response.Content.ReadAsStringAsync(); - - int firstIndex = 0; - int lastIndex = 0; - - if (html.Contains(StartHeadTag) && html.Contains(EndHeadTag)) - { - firstIndex = html.IndexOf(StartHeadTag, StringComparison.Ordinal); - lastIndex = html.IndexOf(EndHeadTag, StringComparison.Ordinal); - } - - string head = html.AsSpan().Slice(firstIndex, lastIndex).ToString(); - html = ""; - - string title = ""; - string description = ""; - - Regex titleRegex = TitleRegEx(); - Match titleMatch = titleRegex.Match(head); - - if (titleMatch.Success) - { - title = titleMatch.Groups[1].Value; - } - - Regex descriptionRegex = DexcriptionRegEx(); - Match descriptionMatch = descriptionRegex.Match(head); - - if (descriptionMatch.Success) - { - description = descriptionMatch.Groups[1].Value; - } - - return (title, description); - } public static async Task HasRobotsTxt(string url, int port) { @@ -139,9 +51,4 @@ public static partial class HttpClientHelper return response is not null && response.IsSuccessStatusCode; } - - [GeneratedRegex(TitlePattern)] - private static partial Regex TitleRegEx(); - [GeneratedRegex(DescriptionPattern)] - private static partial Regex DexcriptionRegEx(); } \ No newline at end of file diff --git a/Backend/Helper/SearchHelper.cs b/Backend/Helper/SearchHelper.cs deleted file mode 100644 index d1f4b23..0000000 --- a/Backend/Helper/SearchHelper.cs +++ /dev/null @@ -1,44 +0,0 @@ -using FuzzySharp; -using Models.Handler; -using Models.Model.External; - -namespace Backend.Helper; - -public static class SearchHelper -{ - public static SearchResults Search(string searchText, DbHandler dbHandler) - { - if (string.IsNullOrWhiteSpace(searchText)) - { - SearchResult searchResult = new(); - searchResult.Description = "asd"; - searchResult.Title = "asd"; - searchResult.Url = "asd"; - SearchResults searchResults = new(); - searchResults.Results = - [ - searchResult - ]; - return searchResults; - } - - List temp = dbHandler.GetSearchResults(); - SearchResults searchResultsList = new(); - - for (int i = 0; i < temp.Count; i++) - { - if (temp[i] is null) continue; - - SearchResult result = new(); - - if (Fuzz.Ratio(searchText, temp[i]!.Url) <= 50 && Fuzz.Ratio(searchText, temp[i]!.Title) <= 50) continue; - - result.Url = temp[i]?.Url; - result.Title = temp[i]?.Title; - - searchResultsList.Results.Add(result); - } - - return searchResultsList; - } -} \ No newline at end of file diff --git a/Backend/Helper/TcpClientHelper.cs b/Backend/Helper/TcpClientHelper.cs index 4e01df5..eb4b249 100644 --- a/Backend/Helper/TcpClientHelper.cs +++ b/Backend/Helper/TcpClientHelper.cs @@ -1,20 +1,24 @@ using System.Net; using System.Net.Sockets; +using Models.Model.Backend; namespace Backend.Helper; public static class TcpClientHelper { - public static (int, int) CheckPort(string ip, params int[] ports) + public static (int, int) CheckPort(Ip ip, params int[] ports) { // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached, // and it would just return a result.error, for example. for (int i = 0; i < ports.Length; i++) { + using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.SendTimeout = 250; + try { - using TcpClient client = new(); - client.Connect(ip, ports[i]); - // If the connection is successful, update the result array with the port number + socket.Connect(new IPEndPoint(IPAddress.Parse(ip.ToString()), ports[i])); + socket.Close(); + // If the connection is not successful, update the ports array with 0. } catch { diff --git a/Manager/Commands.cs b/Manager/Commands.cs deleted file mode 100644 index 3c77e2e..0000000 --- a/Manager/Commands.cs +++ /dev/null @@ -1,122 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Text.Json; -using Models.Model.Backend; -using Models.Model.External; -using NetMQ; -using NetMQ.Sockets; - -namespace Manager; - -public static class Commands -{ - public static void GetProgress() - { - Console.WriteLine("Getting progress ..."); - - CommunicationObject communicationObject = new(); - communicationObject.Command = CommunicationCommand.GetScanningProgress; - - ScanningStatus temp = GetProgress(communicationObject); - - Console.WriteLine($"Total filtered: {temp.TotalFiltered:n0}"); - Console.WriteLine($"Total discarded: {temp.TotalDiscarded:n0}"); - Console.WriteLine($"Total percentage scanned: {temp.PercentageOfIpv4Scanned}"); - Console.WriteLine($"Total Ips left: {temp.AmountOfIpv4Left:n0}"); - Console.WriteLine($"Filtered DB size: {temp.FilteredDbSize} Kb"); - Console.WriteLine($"Discarded DB size: {temp.DiscardedDbSize} Kb"); - Console.WriteLine($"Mydb DB size: {temp.MyDbSize} Kb"); - } - - public static void StopServer() - { - CommunicationObject communicationObject = new(); - communicationObject.Command = CommunicationCommand.StopScanning; - - Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); - } - - public static void Vacuum() - { - CommunicationObject communicationObject = new(); - communicationObject.Command = CommunicationCommand.DbVacuum; - - Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); - } - - public static void ReIndex() - { - CommunicationObject communicationObject = new(); - communicationObject.Command = CommunicationCommand.DbReindex; - - Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); - } - - public static void SetRuntimeVariable(RuntimeVariable runtimeVariable, string value) - { - CommunicationObject communicationObject = new() - { - Command = CommunicationCommand.ChangeRuntimeVariable, - Variable = runtimeVariable.ToString(), - VariableValue = value - }; - - Console.WriteLine(SendAndRecieveStringMessage(communicationObject)); - } - - public static void GetHelp() - { - Console.WriteLine("Available commands:"); - Console.WriteLine(" stop - stops the server"); - Console.WriteLine(" clear - clears the console"); - Console.WriteLine(" q - quits the program"); - Console.WriteLine(" p - print the progress information of the scanner"); - Console.WriteLine(" g - manual garbage collect on the server"); - Console.WriteLine(" r - manual reindex the databases"); - Console.WriteLine(" v - manual vacuum the databases"); - Console.WriteLine(" R - change runtime variable"); - Console.WriteLine(" lR - list runtime variables"); - Console.WriteLine(" help - shows this help"); - Console.WriteLine(); - } - - public static void PrintRuntimeVariables() - { - Console.WriteLine("Runtime variables:"); - Console.WriteLine($"{RuntimeVariable.ScannerTimeout.ToString()} - Sets the timeout in milliseconds for the scanner"); - Console.WriteLine($"{RuntimeVariable.ContentFilter.ToString()} - Sets the timeout in milliseconds for the content filter"); - Console.WriteLine($"{RuntimeVariable.DbContent.ToString()} - Sets the wait time in milliseconds for the content database if the queue is empty"); - Console.WriteLine($"{RuntimeVariable.DbDiscarded.ToString()} - Sets the wait time in milliseconds for the discarded database if the queue is empty"); - } - - [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] - [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] - private static string? SendAndRecieveStringMessage(CommunicationObject communicationObject) - { - //byte[] bytes = MessagePackSerializer.Serialize(communicationObject, ContractlessStandardResolver.Options); - - byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject); - - using RequestSocket client = new(); - client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(lol); - byte[] msg = client.ReceiveFrameBytes(); - client.Close(); - - return JsonSerializer.Deserialize(msg); - } - - [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] - [RequiresDynamicCode("Calls System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(TValue, JsonSerializerOptions)")] - private static ScanningStatus GetProgress(CommunicationObject communicationObject) - { - byte[] lol = JsonSerializer.SerializeToUtf8Bytes(communicationObject); - - using RequestSocket client = new(); - client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(lol); - byte[] msg = client.ReceiveFrameBytes(); - client.Close(); - - return JsonSerializer.Deserialize(msg); - } -} \ No newline at end of file diff --git a/Manager/Manager.csproj b/Manager/Manager.csproj deleted file mode 100644 index 6bed32a..0000000 --- a/Manager/Manager.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - Exe - net8.0 - enable - enable - - - - - - - - - - - - - diff --git a/Manager/Program.cs b/Manager/Program.cs deleted file mode 100644 index 787506a..0000000 --- a/Manager/Program.cs +++ /dev/null @@ -1,90 +0,0 @@ -using Manager; -using Models.Model.Backend; - -bool stop = false; - -do -{ - string? input = Console.ReadLine(); - - if (string.Equals(input, "stop")) - { - Commands.StopServer(); - } - - else if (string.Equals(input, "q")) - { - stop = true; - } - - else if (string.Equals(input, "clear")) - { - Console.Clear(); - } - - else if (string.Equals(input, "p")) - { - Commands.GetProgress(); - } - - else if (string.Equals(input, "v")) - { - Commands.Vacuum(); - } - - else if (string.Equals(input, "r")) - { - Commands.ReIndex(); - } - - else if (string.Equals(input, "R")) - { - Console.WriteLine("Variable name."); - string? variable = Console.ReadLine(); - - Console.WriteLine("Variable value."); - string? value = Console.ReadLine(); - - if (string.IsNullOrWhiteSpace(variable) || string.IsNullOrWhiteSpace(value)) - { - Console.WriteLine("Please enter a value."); - return; - } - - if (variable == RuntimeVariable.ScannerTimeout.ToString()) - { - Commands.SetRuntimeVariable(RuntimeVariable.ScannerTimeout, value); - } - - else if (variable == RuntimeVariable.ContentFilter.ToString()) - { - Commands.SetRuntimeVariable(RuntimeVariable.ContentFilter, value); - } - - else if (variable == RuntimeVariable.DbDiscarded.ToString()) - { - Commands.SetRuntimeVariable(RuntimeVariable.DbDiscarded, value); - } - - else if (variable == RuntimeVariable.DbContent.ToString()) - { - Commands.SetRuntimeVariable(RuntimeVariable.DbContent, value); - } - } - - else if (string.Equals(input, "lR")) - { - Commands.PrintRuntimeVariables(); - } - - else if (string.Equals(input, "help")) - { - Commands.GetHelp(); - } - - else - { - Commands.GetHelp(); - } - -} while (!stop); \ No newline at end of file diff --git a/Models/BackupDB/ScannerResume.db b/Models/BackupDB/ScannerResume.db index a44725c..9063c7f 100644 Binary files a/Models/BackupDB/ScannerResume.db and b/Models/BackupDB/ScannerResume.db differ diff --git a/Models/Experimental/CustomPing.cs b/Models/Experimental/CustomPing.cs new file mode 100644 index 0000000..2188dc6 --- /dev/null +++ b/Models/Experimental/CustomPing.cs @@ -0,0 +1,39 @@ +using System.Diagnostics; +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using Models.Model.Backend; +namespace Models.Experimental; +public class CustomPing +{ + public static IPStatus SendIcmpEchoRequestOverRawSocket(IPAddress address, int timeout) + { + SocketConfig socketConfig = new SocketConfig(new IPEndPoint(address, 0), timeout, (CustomProtocolType) 1, RawSocket.CreateSendMessageBuffer(new() {Type = 8})); + using Socket rawSocket = RawSocket.GetRawSocket(socketConfig); + int ipHeaderLength = 20; + + try + { + rawSocket.SendTo(socketConfig.SendBuffer, 0, socketConfig.SendBuffer.Length, SocketFlags.None, socketConfig.EndPoint); + byte[] numArray = new byte[136]; + long timestamp = Stopwatch.GetTimestamp(); + + // TODO: WTF ??? + EndPoint lol = socketConfig.EndPoint; + + while (Stopwatch.GetElapsedTime(timestamp).TotalMilliseconds < timeout) + { + int from = rawSocket.ReceiveFrom(numArray, SocketFlags.None, ref lol); + + IPStatus status; + if (from - ipHeaderLength >= 8 && RawSocket.TryGetPingReply(numArray, from, ref ipHeaderLength, out status)) + return status; + } + } + catch (SocketException ex) when (ex.SocketErrorCode == SocketError.TimedOut) + { + } + + return IPStatus.TimedOut; + } +} \ No newline at end of file diff --git a/Models/Experimental/MessageConstant.cs b/Models/Experimental/MessageConstant.cs new file mode 100644 index 0000000..5800e4c --- /dev/null +++ b/Models/Experimental/MessageConstant.cs @@ -0,0 +1,59 @@ +using System.Net.NetworkInformation; +namespace Models.Experimental; +public struct MessageConstant +{ + public static IPStatus MapV4TypeToIpStatus(int type, int code) + { + IPStatus ipStatus1; + switch ((IcmpV4MessageType) type) + { + case IcmpV4MessageType.EchoReply: + ipStatus1 = IPStatus.Success; + break; + case IcmpV4MessageType.DestinationUnreachable: + IPStatus ipStatus2; + switch ((byte) code) + { + case 0: + ipStatus2 = IPStatus.DestinationNetworkUnreachable; + break; + case 1: + ipStatus2 = IPStatus.DestinationHostUnreachable; + break; + case 2: + ipStatus2 = IPStatus.DestinationProtocolUnreachable; + break; + case 3: + ipStatus2 = IPStatus.DestinationPortUnreachable; + break; + default: + ipStatus2 = IPStatus.DestinationUnreachable; + break; + } + ipStatus1 = ipStatus2; + break; + case IcmpV4MessageType.SourceQuench: + ipStatus1 = IPStatus.SourceQuench; + break; + case IcmpV4MessageType.TimeExceeded: + ipStatus1 = IPStatus.TtlExpired; + break; + case IcmpV4MessageType.ParameterProblemBadIpHeader: + ipStatus1 = IPStatus.BadHeader; + break; + default: + ipStatus1 = IPStatus.Unknown; + break; + } + return ipStatus1; + } +} + +internal enum IcmpV4MessageType : byte +{ + EchoReply = 0, + DestinationUnreachable = 3, + SourceQuench = 4, + TimeExceeded = 11, // 0x0B + ParameterProblemBadIpHeader = 12, // 0x0C +} \ No newline at end of file diff --git a/Models/Experimental/RawSocket.cs b/Models/Experimental/RawSocket.cs new file mode 100644 index 0000000..2b7e359 --- /dev/null +++ b/Models/Experimental/RawSocket.cs @@ -0,0 +1,91 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Models.Model.Backend; + +namespace Models.Experimental; + +public class RawSocket +{ + public static unsafe Socket GetRawSocket(SocketConfig socketConfig) + { + Socket rawSocket = new Socket(AddressFamily.InterNetwork, SocketType.Raw, (ProtocolType)socketConfig.ProtocolType); + rawSocket.ReceiveTimeout = socketConfig.Timeout; + rawSocket.SendTimeout = socketConfig.Timeout; + rawSocket.Connect(socketConfig.EndPoint); + int num = 1; + rawSocket.SetRawSocketOption(0, 11, new ReadOnlySpan((void*) &num, 4)); + return rawSocket; + } + + public static bool TryGetPingReply(byte[] receiveBuffer, int bytesReceived, ref int ipHeaderLength, out IPStatus reply) + { + byte num = (byte) (receiveBuffer[0] & 15U); + ipHeaderLength = 4 * num; + + int start = ipHeaderLength; + int srcOffset = ipHeaderLength + 8; + + IcmpHeader icmpHeader = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(receiveBuffer.AsSpan(start))); + byte[] numArray = new byte[bytesReceived - srcOffset]; + Buffer.BlockCopy(receiveBuffer, srcOffset, numArray, 0, numArray.Length); + reply = MessageConstant.MapV4TypeToIpStatus(icmpHeader.Type, icmpHeader.Code); + return true; + } + + public static unsafe byte[] CreateSendMessageBuffer(IcmpHeader icmpHeader) + { + int length = sizeof (IcmpHeader); + byte[] sendMessageBuffer = new byte[length]; + new Span((void*) &icmpHeader, length).CopyTo(new Span(sendMessageBuffer, 0, length)); + ushort bufferChecksum = ComputeBufferChecksum(sendMessageBuffer.AsSpan(0)); + sendMessageBuffer[2] = (byte) ((uint) bufferChecksum >> 8); + sendMessageBuffer[3] = (byte) (bufferChecksum & byte.MaxValue); + return sendMessageBuffer; + } + + private static ushort ComputeBufferChecksum(ReadOnlySpan buffer) + { + uint num1 = 0; + for (int index = 0; index < buffer.Length; index += 2) + { + ushort num2 = (ushort) ((ushort) (buffer[index] << 8 & 65280) | (index + 1 < buffer.Length ? (ushort) (buffer[index + 1] & (uint) byte.MaxValue) : 0)); + num1 += num2; + } + while (num1 >> 16 != 0U) + num1 = (num1 & ushort.MaxValue) + (num1 >> 16); + return (ushort) ~num1; + } +} + +public class SocketConfig +{ + public EndPoint EndPoint; + public readonly int Timeout; + public readonly CustomProtocolType ProtocolType; + public readonly byte[] SendBuffer; + + public SocketConfig( + EndPoint endPoint, + int timeout, + CustomProtocolType protocolType, + byte[] sendBuffer) + { + EndPoint = endPoint; + Timeout = timeout; + ProtocolType = protocolType; + SendBuffer = sendBuffer; + } +} + + +public struct IcmpHeader +{ + public byte Type; + public byte Code; + public ushort HeaderChecksum; + public ushort Identifier; + public ushort SequenceNumber; +} \ No newline at end of file diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 282eaf8..6e4d3ff 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -1,5 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; using Microsoft.Data.Sqlite; using Models.Helper; using Models.Model.Backend; @@ -13,8 +15,10 @@ public class DbHandler private readonly ConcurrentQueue _unfilteredQueue; private readonly ConcurrentQueue _discardedQueue; private readonly ConcurrentQueue _resumeQueue; + private readonly ConcurrentQueue _preFilteredQueue; private readonly string _unfilteredConnectionString; + private readonly string _preFilteredConnectionString; private readonly string _filteredConnectionString; private readonly string _resumeConnectionString; private readonly string _compressedConnectionString; @@ -25,15 +29,20 @@ public class DbHandler " INSERT INTO Unfiltered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Filtered)" + " VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, @filtered)"; + private const string InsertPreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + + " INSERT INTO PreFiltered (Ip1, Ip2, Ip3, Ip4, ResponseCode, Filtered)" + + " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode, @filtered)"; + private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" + - " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," + - " Description1, Description2, Url1, Url2, ServerType1, ServerType2," + + " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," + + " Url1, Url2, ServerType1, ServerType2," + " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," + " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," + " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," + " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " + - " @title1, @title2, @description1, @description2, @url1, @url2, " + + " @url1, @url2, " + " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " + " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " + " @robotsTXT1, @robotsTXT2," + @@ -68,13 +77,13 @@ public class DbHandler private const string InsertIntoDiscarded = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + - " INSERT INTO Discarded (Ip1, Ip2, Ip3, Ip4, ResponseCode)" + - " VALUES (@ip1, @ip2, @ip3, @ip4, @responseCode)"; + " INSERT INTO Discarded (PackedData)" + + " VALUES (@packedData)"; private const string InsertIntoResume = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + - " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte)" + - " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte);"; + " INSERT INTO Resume (ThreadNumber, StartRange, EndRange, FirstByte, SecondByte, ThirdByte, FourthByte, Paused, Completed)" + + " VALUES (@threadNumber, @startRange, @endRange, @firstByte, @secondByte, @thirdByte, @fourthByte, @paused, @completed);"; private const string InsertIntoCompressedDbConnectionString = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off;" + @@ -86,11 +95,16 @@ public class DbHandler private const string ReadFilteredIdsStatement = "SELECT Id FROM Filtered WHERE Id != 0 ORDER BY Id DESC LIMIT 1;"; private const string ReadFilteredIpStatement = "SELECT Ip1, Ip2, Ip3, Ip4 FROM Filtered WHERE Ip1 == @ip1 AND Ip2 == @ip1 AND Ip3 == @ip1 AND Ip4 == @ip1 ORDER BY Ip1 DESC LIMIT 1;"; private const string ReadDiscardedSeqIdsStatement = "SELECT seq FROM sqlite_sequence;"; - private const string ReadAndDeleteResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber; DELETE FROM RESUME WHERE ThreadNumber == @threadNumber;"; + private const string ReadResumeStatement = "SELECT * FROM Resume WHERE ThreadNumber == @threadNumber;"; private const string ReadCompressedDbRowsStatement = "SELECT Rows FROM CompressedDatabases;"; + private const string ReadPreFilteredIdsStatement = "SELECT Id FROM PreFiltered WHERE Filtered == 0;"; + private const string ReadPreFilteredStatement = "SELECT Ip1, Ip2, Ip3, Ip4, ResponseCode, Id FROM PreFiltered WHERE Filtered == 0 ORDER BY Ip1 ASC LIMIT 1;"; + private const string UpdatePreFilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE PreFiltered SET Filtered = 1 WHERE Id == @id;"; private const string UpdateUnfilteredStatement = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY; PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = off; UPDATE Unfiltered SET Filtered = 1 WHERE Id == @id;"; + private const string UpdateResumeStatement = "UPDATE Resume SET FirstByte = @firstByte, SecondByte = @secondByte, ThirdByte = @thirdByte, FourthByte = @fourthByte, Paused = @paused, Completed = @completed WHERE ThreadNumber = @threadNumber;"; + private const string ReIndexDatabasesStatement = "REINDEX;"; private const string VacuumDatabasesStatement = "VACUUM;"; @@ -111,12 +125,15 @@ public class DbHandler public DbHandler(ConcurrentQueue filteredQueue, ConcurrentQueue discardedQueue, ConcurrentQueue unfilteredQueue, - ConcurrentQueue resumeQueue, string basePath) + ConcurrentQueue resumeQueue, + ConcurrentQueue preFilteredQueue, + string basePath) { _filteredQueue = filteredQueue; _discardedQueue = discardedQueue; _unfilteredQueue = unfilteredQueue; _resumeQueue = resumeQueue; + _preFilteredQueue = preFilteredQueue; SetContentWaitTime(100); SetDiscardedWaitTime(10); @@ -127,6 +144,7 @@ public class DbHandler _filteredConnectionString = $"Data Source={basePath}/Models/Filtered.db"; _resumeConnectionString = $"Data Source={basePath}/Models/ScannerResume.db"; _compressedConnectionString = $"Data Source={basePath}/Models/CompressedDatabases.db"; + _preFilteredConnectionString = $"Data Source={basePath}/Models/PreFiltered.db"; } public void SetContentWaitTime(int waitTime) @@ -188,6 +206,24 @@ public class DbHandler Console.WriteLine("Filtered DbHandler stopped."); } + + public void PrefilteredDbHandler() + { + Console.WriteLine("PreFiltered Db handler started."); + + while (!_stop) + { + if (_preFilteredQueue.IsEmpty) + { + Thread.Sleep(4); + continue; + } + + _preFilteredQueue.TryDequeue(out FilterQueueItem queueItem); + + InsertPrefiltered(queueItem); + } + } public void ResumeDbHandler() { @@ -202,12 +238,9 @@ public class DbHandler continue; } - _resumeQueue.TryDequeue(out ScannerResumeObject? queueItem); + _resumeQueue.TryDequeue(out ScannerResumeObject queueItem); - if (queueItem is not null) - { - InsertResumeObject(queueItem); - } + InsertResumeObject(queueItem); } Console.WriteLine("Resume DbHandler stopped."); @@ -232,7 +265,7 @@ public class DbHandler Thread f = new (DiscardedDbHandler!); f.Start(discardedDbHandlerSetting); - Thread.Sleep(5000); + Thread.Sleep(150); } return waitHandles; @@ -247,7 +280,16 @@ public class DbHandler int i = 0; - while (!_stop) + SqliteConnection connection = new(connectionString); + connection.Open(); + + SqliteCommand command = new(InsertIntoDiscarded, connection); + + StringBuilder stringBuilder = new(); + + bool running = true; + + while (!_stop && running) { if (_discardedQueue.IsEmpty || _pause) { @@ -255,31 +297,46 @@ public class DbHandler _paused = true; continue; } - - if (i >= 50_000_000 && !_compressing) + + if (_stop && _discardedQueue.IsEmpty) { - _compressing = true; - - i = 0; - - InsertCompressedDatabase(discardedDbHandlerSetting.ThreadId, GetDiscardedIndexesForSpecificDb(connectionString)); - - int compressedDatabases = GetDatabasesHelper.GetTotalCompressedDatabases($"{_basePath}/Models"); - - CompressionHelper.CompressFile(absolutePath, $"{absolutePath}_{compressedDatabases}"); - - DropAndCreateDiscarded(discardedDbHandlerSetting.ThreadId); - - _compressing = false; + running = false; } - _discardedQueue.TryDequeue(out Discarded queueItem); + if (i == 2048) + { + command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString())); + _ = command.ExecuteNonQuery(); + + // Re-use the command object. + command.Parameters.Clear(); + + stringBuilder.Clear(); + + i = 0; + } - InsertDiscarded(queueItem, connectionString); + if (_discardedQueue.TryDequeue(out Discarded queueItem)) + { + stringBuilder.Append(queueItem.Ip.PackIp()); + stringBuilder.Append(':'); + stringBuilder.Append(queueItem.ResponseCode); + stringBuilder.Append(','); + } i++; } + if (stringBuilder.Length != 0) + { + command.Parameters.AddWithValue("@packedData", CompressionHelper.CompressString(stringBuilder.ToString())); + _ = command.ExecuteNonQuery(); + } + + connection.Close(); + connection.Dispose(); + command.Dispose(); + discardedDbHandlerSetting.Handle!.Set(); Console.WriteLine("Discarded DbHandler stopped."); @@ -328,23 +385,6 @@ public class DbHandler connection.Close(); } - private static void InsertDiscarded(Discarded discarded, string dbConnectionString) - { - using SqliteConnection connection = new(dbConnectionString); - connection.Open(); - - using SqliteCommand command = new(InsertIntoDiscarded, connection); - - command.Parameters.AddWithValue("@ip1", discarded.Ip.Ip1); - command.Parameters.AddWithValue("@ip2", discarded.Ip.Ip2); - command.Parameters.AddWithValue("@ip3", discarded.Ip.Ip3); - command.Parameters.AddWithValue("@ip4", discarded.Ip.Ip4); - command.Parameters.AddWithValue("@responseCode", discarded.ResponseCode); - _ = command.ExecuteNonQuery(); - - connection.Close(); - } - private void InsertFiltered(Filtered filtered) { using SqliteConnection connection = new(_filteredConnectionString); @@ -427,10 +467,6 @@ public class DbHandler command.Parameters.AddWithValue("@port2", filtered.Port2); command.Parameters.AddWithValue("@url1", filtered.Url1); command.Parameters.AddWithValue("@url2", filtered.Url2); - command.Parameters.AddWithValue("@title1", filtered.Title1); - command.Parameters.AddWithValue("@title2", filtered.Title2); - command.Parameters.AddWithValue("@description1", filtered.Description1); - command.Parameters.AddWithValue("@description2", filtered.Description2); command.Parameters.AddWithValue("@serverType1", filtered.ServerType1); command.Parameters.AddWithValue("@serverType2", filtered.ServerType2); command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1); @@ -461,19 +497,31 @@ public class DbHandler { using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); - - using SqliteCommand command = new(InsertIntoResume, connection); + SqliteCommand command; + + if (resumeObject.Operation == Operations.Insert) + { + command = new(InsertIntoResume, connection); + command.Parameters.AddWithValue("@startRange", resumeObject.StartRange); + command.Parameters.AddWithValue("@endRange", resumeObject.EndRange); + } + else + { + command = new(UpdateResumeStatement, connection); + } command.Parameters.AddWithValue("@threadNumber", resumeObject.ThreadNumber); - command.Parameters.AddWithValue("@startRange", resumeObject.StartRange); - command.Parameters.AddWithValue("@endRange", resumeObject.EndRange); command.Parameters.AddWithValue("@firstByte", resumeObject.FirstByte); command.Parameters.AddWithValue("@secondByte", resumeObject.SecondByte); command.Parameters.AddWithValue("@thirdByte", resumeObject.ThirdByte); command.Parameters.AddWithValue("@fourthByte", resumeObject.FourthByte); + command.Parameters.AddWithValue("@paused", resumeObject.Paused); + command.Parameters.AddWithValue("@completed", resumeObject.Completed); _ = command.ExecuteNonQuery(); connection.Close(); + + command.Dispose(); } private void InsertCompressedDatabase(int threadNumber, long rows) @@ -489,6 +537,25 @@ public class DbHandler _ = command.ExecuteNonQuery(); connection.Close(); } + + private void InsertPrefiltered(FilterQueueItem filterQueueItem) + { + using SqliteConnection connection = new(_preFilteredConnectionString); + connection.Open(); + + using SqliteCommand command = new(InsertPreFilteredStatement, connection); + + command.Parameters.AddWithValue("@ip1", filterQueueItem.Ip.Ip1); + command.Parameters.AddWithValue("@ip2", filterQueueItem.Ip.Ip2); + command.Parameters.AddWithValue("@ip3", filterQueueItem.Ip.Ip3); + command.Parameters.AddWithValue("@ip4", filterQueueItem.Ip.Ip4); + command.Parameters.AddWithValue("@responseCode", filterQueueItem.ResponseCode); + command.Parameters.AddWithValue("@filtered", 0); + + _ = command.ExecuteNonQuery(); + + connection.Close(); + } private void UpdateUnfiltered(Unfiltered unfiltered) { @@ -557,79 +624,42 @@ public class DbHandler return ids; } - public long GetFilteredIndexes() + public bool GetPreFilterQueueItem(out FilterQueueItem filterQueueItem) { - long rowId = 0; - - using SqliteConnection connection = new(_filteredConnectionString); + using SqliteConnection connection = new(_preFilteredConnectionString); connection.Open(); - using SqliteCommand command = new(ReadFilteredIdsStatement, connection); + SqliteCommand command = new(ReadPreFilteredStatement, connection); using SqliteDataReader reader = command.ExecuteReader(); + filterQueueItem = new(); + Ip ip = new(); + long id = 0; + if (!reader.HasRows) { - return 0; + return false; } while (reader.Read()) { - rowId = reader.GetInt64(0); + ip.Ip1 = reader.GetInt32(0); + ip.Ip2 = reader.GetInt32(1); + ip.Ip3 = reader.GetInt32(2); + ip.Ip4 = reader.GetInt32(3); + filterQueueItem.ResponseCode = reader.GetInt32(4); + id = reader.GetInt64(5); } - return rowId; - } - - public long GetDiscardedIndexes() - { - long rowId = 0; + filterQueueItem.Ip = ip; + + command = new(UpdatePreFilteredStatement, connection); + command.Parameters.AddWithValue("@id", id); - SqliteConnection connection; - SqliteCommand command; - SqliteDataReader reader; - - for (int i = 0; i < _discardedConnectionStrings.Count; i++) - { - connection = new(_discardedConnectionStrings[i]); - connection.Open(); - - command = new(ReadDiscardedSeqIdsStatement, connection); - reader = command.ExecuteReader(); - - if (!reader.HasRows) - { - return rowId; - } - - while (reader.Read()) - { - rowId += reader.GetInt64(0); - } - - connection.Close(); - } - - connection = new(_compressedConnectionString); - connection.Open(); - command = new(ReadCompressedDbRowsStatement, connection); - reader = command.ExecuteReader(); - - if (!reader.HasRows) - { - return rowId; - } - - while (reader.Read()) - { - rowId += reader.GetInt64(0); - } - - connection.Close(); - connection.Dispose(); + command.ExecuteNonQuery(); command.Dispose(); - reader.Dispose(); - return rowId; + return true; } private static long GetDiscardedIndexesForSpecificDb(string connectionString) @@ -706,36 +736,6 @@ public class DbHandler return true; } - public List GetSearchResults() - { - lock (_readFilteredLock) - { - using SqliteConnection connection = new(_filteredConnectionString); - connection.Open(); - - using SqliteCommand command = new(ReadFilteredStatement, connection); - using SqliteDataReader reader = command.ExecuteReader(); - - if (!reader.HasRows) - { - return []; - } - - List results = []; - - while (reader.Read()) - { - SearchResult result = new(); - result.Title = reader.GetString(0); - result.Url = reader.GetString(1); - - results.Add(result); - } - - return results; - } - } - public ScannerResumeObject? GetResumeObject(int threadNumber) { lock (_readAndDeleteResumeLock) @@ -743,7 +743,7 @@ public class DbHandler using SqliteConnection connection = new(_resumeConnectionString); connection.Open(); - using SqliteCommand command = new(ReadAndDeleteResumeStatement, connection); + using SqliteCommand command = new(ReadResumeStatement, connection); command.Parameters.AddWithValue("@threadNumber", threadNumber); using SqliteDataReader reader = command.ExecuteReader(); @@ -764,81 +764,20 @@ public class DbHandler resumeObject.SecondByte = reader.GetInt32(4); resumeObject.ThirdByte = reader.GetInt32(5); resumeObject.FourthByte = reader.GetInt32(6); + resumeObject.Paused = reader.GetBoolean(7); + resumeObject.Completed = reader.GetBoolean(8); } return resumeObject; } } - - public void ReIndex() - { - _pause = true; - Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open. - - if (!_paused) - { - Thread.Sleep(5000); // Just for safety. - } - - SqliteConnection connection = new(_filteredConnectionString); - connection.Open(); - - SqliteCommand command = new(ReIndexDatabasesStatement, connection); - _ = command.ExecuteNonQuery(); - connection.Close(); - - connection = new(_unfilteredConnectionString); - connection.Open(); - - command = new(ReIndexDatabasesStatement, connection); - _ = command.ExecuteNonQuery(); - connection.Close(); - - connection.Dispose(); - command.Dispose(); - - _pause = false; - _paused = false; - } - - public void Vacuum() - { - _pause = true; - - Thread.Sleep(5000); // Wait for 5 secs before doing anything with the db So we're sure that no db is open. - - if (!_paused) - { - Thread.Sleep(5000); // Just for safety. - } - - SqliteConnection connection = new(_filteredConnectionString); - connection.Open(); - - SqliteCommand command = new(VacuumDatabasesStatement, connection); - _ = command.ExecuteNonQuery(); - connection.Close(); - - connection = new(_unfilteredConnectionString); - connection.Open(); - - command = new(VacuumDatabasesStatement, connection); - _ = command.ExecuteNonQuery(); - connection.Close(); - - connection.Dispose(); - command.Dispose(); - - _pause = false; - _paused = false; - } private (string, string) CreateDiscardedDb(int threadNumber) { string absolutePath = $"{_basePath}/Models/Discarded{threadNumber}.db"; string databaseName = $"Data Source={_basePath}/Models/Discarded{threadNumber}.db"; - const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, Ip1 INTEGER NOT NULL, Ip2 INTEGER NOT NULL, Ip3 INTEGER NOT NULL, Ip4 INTEGER NOT NULL, ResponseCode INTEGER NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))"; + const string createStatement = "CREATE TABLE IF NOT EXISTS Discarded (Id INTEGER NOT NULL, PackedData TEXT NOT NULL, PRIMARY KEY(Id AUTOINCREMENT))"; _discardedConnectionStrings.Add(databaseName); diff --git a/Models/Helper/CompressionHelper.cs b/Models/Helper/CompressionHelper.cs index 319b9ff..88eada1 100644 --- a/Models/Helper/CompressionHelper.cs +++ b/Models/Helper/CompressionHelper.cs @@ -1,4 +1,5 @@ using System.IO.Compression; +using System.Text; namespace Models.Helper; @@ -8,7 +9,7 @@ public static class CompressionHelper { using FileStream originalFileStream = new(sourceFile, FileMode.Open); using FileStream compressedFileStream = File.Create($"{targetFile}.gz"); - using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest); + using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize); originalFileStream.CopyTo(compressor); } @@ -19,4 +20,31 @@ public static class CompressionHelper using GZipStream decompressor = new(compressedFileStream, CompressionMode.Decompress); decompressor.CopyTo(decompressedFileStream); } + + public static string CompressString(string text) + { + byte[] byteArray = Encoding.UTF8.GetBytes(text); + + using MemoryStream memoryStream = new(); + + using (BrotliStream compressionStream = new(memoryStream, CompressionLevel.SmallestSize)) + { + compressionStream.Write(byteArray, 0, byteArray.Length); + } + + return Convert.ToBase64String(memoryStream.ToArray()); + } + + public static string DecompressString(string compressedText) + { + byte[] byteArray = Convert.FromBase64String(compressedText); + + using MemoryStream memoryStream = new(byteArray); + + using BrotliStream decompressionStream = new(memoryStream, CompressionMode.Decompress); + + using StreamReader reader = new(decompressionStream); + + return reader.ReadToEnd(); + } } \ No newline at end of file diff --git a/Models/Model/Backend/CustomProtocolType.cs b/Models/Model/Backend/CustomProtocolType.cs new file mode 100644 index 0000000..9bb0e42 --- /dev/null +++ b/Models/Model/Backend/CustomProtocolType.cs @@ -0,0 +1,6 @@ +namespace Models.Model.Backend; + +public enum CustomProtocolType +{ + Icmp = 1 +} \ No newline at end of file diff --git a/Models/Model/Backend/DatabaseSizes.cs b/Models/Model/Backend/DatabaseSizes.cs deleted file mode 100644 index ba4c06b..0000000 --- a/Models/Model/Backend/DatabaseSizes.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Models.Model.Backend; - -public struct DatabaseSizes -{ - public double DiscardedDbSize { get; set; } - - public double FilteredDbSize { get; set; } - - public double MyDbSize { get; set; } -} \ No newline at end of file diff --git a/Models/Model/Backend/FilterQueueItem.cs b/Models/Model/Backend/FilterQueueItem.cs new file mode 100644 index 0000000..eef0da3 --- /dev/null +++ b/Models/Model/Backend/FilterQueueItem.cs @@ -0,0 +1,7 @@ +namespace Models.Model.Backend; + +public struct FilterQueueItem +{ + public Ip Ip { get; set; } + public int ResponseCode { get; set; } +} \ No newline at end of file diff --git a/Models/Model/Backend/Filtered.cs b/Models/Model/Backend/Filtered.cs index 5499d3e..0e9a4c6 100644 --- a/Models/Model/Backend/Filtered.cs +++ b/Models/Model/Backend/Filtered.cs @@ -3,10 +3,6 @@ namespace Models.Model.Backend; public class Filtered { public Ip Ip { get; set; } - public string Title1 { get; set; } = ""; - public string Title2 { get; set; } = ""; - public string Description1 { get; set; } = ""; - public string Description2 { get; set; } = ""; public string Url1 { get; set; } = ""; public string Url2 { get; set; } = ""; public int Port1 { get; set; } diff --git a/Models/Model/Backend/IP.cs b/Models/Model/Backend/IP.cs index 3b835e1..bf0d88a 100644 --- a/Models/Model/Backend/IP.cs +++ b/Models/Model/Backend/IP.cs @@ -14,4 +14,9 @@ public struct Ip { return $"{Ip1}.{Ip2}.{Ip3}.{Ip4}"; } + + public uint PackIp() + { + return (uint)(Ip1 << 24) | (uint)(Ip2 << 16) | (uint)(Ip3 << 8) | (uint)Ip4; + } } \ No newline at end of file diff --git a/Models/Model/Backend/RuntimeVariable.cs b/Models/Model/Backend/RuntimeVariable.cs deleted file mode 100644 index 536a86f..0000000 --- a/Models/Model/Backend/RuntimeVariable.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Models.Model.Backend; - -public enum RuntimeVariable -{ - DbContent, - DbDiscarded, - ContentFilter, - ScannerTimeout -} \ No newline at end of file diff --git a/Models/Model/Backend/ScannerResumeObject.cs b/Models/Model/Backend/ScannerResumeObject.cs index 0799b6e..e29c57d 100644 --- a/Models/Model/Backend/ScannerResumeObject.cs +++ b/Models/Model/Backend/ScannerResumeObject.cs @@ -9,4 +9,7 @@ public class ScannerResumeObject public int ThirdByte { get; set; } public int FourthByte { get; set; } public int ThreadNumber { get; set; } + public Operations Operation { get; set; } + public bool Paused { get; set; } + public bool Completed { get; set; } } \ No newline at end of file diff --git a/Models/Model/Backend/SizeUnits.cs b/Models/Model/Backend/SizeUnits.cs deleted file mode 100644 index 2aa7a90..0000000 --- a/Models/Model/Backend/SizeUnits.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Models.Model.Backend; -public enum SizeUnits -{ - Byte, - KB, - MB, - GB, -} diff --git a/Models/Model/External/CommunicationCommand.cs b/Models/Model/External/CommunicationCommand.cs deleted file mode 100644 index cc314a1..0000000 --- a/Models/Model/External/CommunicationCommand.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Models.Model.External; - -public enum CommunicationCommand -{ - GetScanningProgress, - GetSearches, - StopScanning, - DbReindex, - DbVacuum, - ChangeRuntimeVariable, -} \ No newline at end of file diff --git a/Models/Model/External/CommunicationObject.cs b/Models/Model/External/CommunicationObject.cs deleted file mode 100644 index 9fa6609..0000000 --- a/Models/Model/External/CommunicationObject.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace Models.Model.External; - -//[MessagePackObject] -public class CommunicationObject -{ - //[Key(0)] - public CommunicationCommand Command { get; set; } - - //[Key(1)] - public string? SearchTerm { get; set; } = ""; - - //[Key(2)] - public string? Variable { get; set; } = ""; - - //[Key(3)] - public string? VariableValue { get; set; } = ""; -} \ No newline at end of file diff --git a/Models/Model/External/CommunicationResult.cs b/Models/Model/External/CommunicationResult.cs deleted file mode 100644 index 95808d9..0000000 --- a/Models/Model/External/CommunicationResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Models.Model.External; - -public class CommunicationResult -{ - public List? Result { get; set; } - - public ScanningStatus? Status { get; set; } -} \ No newline at end of file diff --git a/Models/Model/External/ScanningStatus.cs b/Models/Model/External/ScanningStatus.cs deleted file mode 100644 index 29a09e4..0000000 --- a/Models/Model/External/ScanningStatus.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Models.Model.External; - -public struct ScanningStatus -{ - public float PercentageOfIpv4Scanned { get; set; } - - public long TotalFiltered { get; set; } - - public long AmountOfIpv4Left { get; set; } - - public long TotalDiscarded { get; set; } - - public double DiscardedDbSize { get; set; } - - public double FilteredDbSize { get; set; } - - public double MyDbSize { get; set; } -} \ No newline at end of file diff --git a/Models/Model/External/SearchResults.cs b/Models/Model/External/SearchResults.cs deleted file mode 100644 index 3abf452..0000000 --- a/Models/Model/External/SearchResults.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Models.Model.External; - -public class SearchResults -{ - public List? Results { get; set; } -} \ No newline at end of file diff --git a/Models/Models.csproj b/Models/Models.csproj index 3a65374..a4e0a76 100644 --- a/Models/Models.csproj +++ b/Models/Models.csproj @@ -1,12 +1,13 @@  - net8.0 + net9.0 enable enable + true - + diff --git a/Proxy/Program.cs b/Proxy/Program.cs deleted file mode 100644 index 8bf0147..0000000 --- a/Proxy/Program.cs +++ /dev/null @@ -1,89 +0,0 @@ -using System.Text.Json; -using System.Text.Json.Serialization; -using AspNetCoreRateLimit; -using Microsoft.AspNetCore.HttpOverrides; -using Microsoft.Extensions.Caching.Memory; -using Models.Model.External; -using NetMQ; -using NetMQ.Sockets; -const string myAllowSpecificOrigins = "_myAllowSpecificOrigins"; - -WebApplicationBuilder builder = WebApplication.CreateSlimBuilder(args); - -builder.Services.ConfigureHttpJsonOptions(options => -{ - options.SerializerOptions.TypeInfoResolverChain.Insert(0, AppJsonSerializerContext.Default); -}); - -builder.Services.AddCors(options => -{ - options.AddPolicy(name: myAllowSpecificOrigins, x => x.AllowAnyOrigin().AllowAnyMethod().AllowAnyHeader()); -}); - -builder.Services.AddMemoryCache(options => options.ExpirationScanFrequency = TimeSpan.FromSeconds(5)); - -WebApplication app = builder.Build(); - -app.UseForwardedHeaders(new() -{ - ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto -}); - -app.UseCors(myAllowSpecificOrigins); - -RouteGroupBuilder progressApi = app.MapGroup("/progress"); -progressApi.MapGet("/", (IMemoryCache memoryCache) => -{ - const string cacheKey = "progress_status"; - if (memoryCache.TryGetValue(cacheKey, out ScanningStatus scanningStatus)) - { - return scanningStatus; - } - - CommunicationObject communicationObject = new() - { - Command = CommunicationCommand.GetScanningProgress - }; - - byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject); - - using RequestSocket client = new(); - client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(bytes); - byte[] msg = client.ReceiveFrameBytes(); - client.Close(); - - scanningStatus = JsonSerializer.Deserialize(msg); - - memoryCache.Set(cacheKey, scanningStatus, DateTimeOffset.Now.AddSeconds(5)); - - return scanningStatus; -}); - - -RouteGroupBuilder searchApi = app.MapGroup("/search"); -searchApi.MapGet("/{term}", (string term) => -{ - CommunicationObject communicationObject = new(); - communicationObject.Command = CommunicationCommand.GetSearches; - communicationObject.SearchTerm = term; - - byte[] bytes = JsonSerializer.SerializeToUtf8Bytes(communicationObject); - - using RequestSocket client = new(); - client.Connect("tcp://127.0.0.1:5556"); - client.SendFrame(bytes); - string msg = client.ReceiveFrameString(); - client.Close(); - - return JsonSerializer.Deserialize(msg); -}); - -app.Run(); - -[JsonSerializable(typeof(ScanningStatus))] -//[JsonSerializable(typeof(SearchResults))] -[JsonSerializable(typeof(CommunicationObject))] -internal partial class AppJsonSerializerContext : JsonSerializerContext -{ -} \ No newline at end of file diff --git a/Proxy/Properties/launchSettings.json b/Proxy/Properties/launchSettings.json deleted file mode 100644 index 408ef90..0000000 --- a/Proxy/Properties/launchSettings.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "http://json.schemastore.org/launchsettings.json", - "profiles": { - "http": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": true, - "launchUrl": "", - "applicationUrl": "http://localhost:5224", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/Proxy/Proxy.csproj b/Proxy/Proxy.csproj deleted file mode 100644 index 9601591..0000000 --- a/Proxy/Proxy.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - net8.0 - enable - enable - true - false - - - - - - - - - - - - diff --git a/Proxy/Proxy.http b/Proxy/Proxy.http deleted file mode 100644 index d6bdc09..0000000 --- a/Proxy/Proxy.http +++ /dev/null @@ -1,11 +0,0 @@ -@Proxy_HostAddress = http://localhost:5224 - -GET {{Proxy_HostAddress}}/progress/ -Accept: application/json - -### - -GET {{Proxy_HostAddress}}/search/asd -Accept: application/json - -### diff --git a/Proxy/appsettings.Development.json b/Proxy/appsettings.Development.json deleted file mode 100644 index 0c208ae..0000000 --- a/Proxy/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/Proxy/appsettings.json b/Proxy/appsettings.json deleted file mode 100644 index 51c11b2..0000000 --- a/Proxy/appsettings.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*" -} diff --git a/README.md b/README.md index 35fb6fd..f090365 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,8 @@ Rasmus Search Engine -This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol) \ No newline at end of file +This is just a hobby project I'm working on at the moment. (Rasmus is my name BTW lol) + +This is also an "Anything goes" type of project. I'm not really aiming for correctness or any other type of paradigm or architecture. + +I'm just trying to minimize memory usage while maximizing performance. diff --git a/RSE.sln b/RSE.sln index 7851f8c..cbe0482 100644 --- a/RSE.sln +++ b/RSE.sln @@ -4,9 +4,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Backend", "Backend\Backend. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Models", "Models\Models.csproj", "{3B0DFF2F-334A-4039-9510-EB4DDB2C5100}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager.csproj", "{B8F0548D-356C-48B4-909B-D6CC317E3772}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -22,13 +20,9 @@ Global {3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Debug|Any CPU.Build.0 = Debug|Any CPU {3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.ActiveCfg = Release|Any CPU {3B0DFF2F-334A-4039-9510-EB4DDB2C5100}.Release|Any CPU.Build.0 = Release|Any CPU - {B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B8F0548D-356C-48B4-909B-D6CC317E3772}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B8F0548D-356C-48B4-909B-D6CC317E3772}.Release|Any CPU.Build.0 = Release|Any CPU - {55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU - {55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU - {55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index a46370d..f09b7a2 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -2,14 +2,38 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded - ForceIncluded \ No newline at end of file + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded \ No newline at end of file diff --git a/Type Dependencies Diagram for Communication and other elements.png b/Type Dependencies Diagram for Communication and other elements.png new file mode 100644 index 0000000..3c0f490 Binary files /dev/null and b/Type Dependencies Diagram for Communication and other elements.png differ diff --git a/global.json b/global.json new file mode 100644 index 0000000..f4fd385 --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "9.0.0", + "rollForward": "latestMajor", + "allowPrerelease": true + } +} \ No newline at end of file