diff --git a/Analyze/Analyze.csproj b/Analyze/Analyze.csproj new file mode 100644 index 0000000..2f601cf --- /dev/null +++ b/Analyze/Analyze.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + diff --git a/Analyze/Program.cs b/Analyze/Program.cs new file mode 100644 index 0000000..e5dff12 --- /dev/null +++ b/Analyze/Program.cs @@ -0,0 +1,3 @@ +// See https://aka.ms/new-console-template for more information + +Console.WriteLine("Hello, World!"); \ No newline at end of file diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index b6930b6..368e458 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -99,10 +99,6 @@ public class ContentFilter string url1 = ""; string url2 = ""; - string title1 = ""; - string title2 = ""; - string description1 = ""; - string description2 = ""; bool robotsTxt1 = false; bool robotsTxt2 = false; string serverType1 = ""; @@ -161,37 +157,6 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - if (ports[i] == 80) - { - if (string.IsNullOrWhiteSpace(url1)) continue; - - try - { - (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 80).GetAwaiter().GetResult(); - title1 = temp.Item1; - description1 = temp.Item2; - } - catch - { - // - } - } - else - { - if (string.IsNullOrWhiteSpace(url2)) continue; - - try - { - (string, string) temp = HttpClientHelper.GetTitleAndDescription(url1, 443).GetAwaiter().GetResult(); - title2 = temp.Item1; - description2 = temp.Item2; - } - catch - { - // - } - } - if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); } if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); } } @@ -201,10 +166,6 @@ public class ContentFilter Ip = ip, Url1 = url1, Url2 = url2, - Title1 = title1, - Title2 = title2, - Description1 = description1, - Description2 = description2, ServerType1 = serverType1, ServerType2 = serverType2, RobotsTXT1 = robotsTxt1, diff --git a/Backend/Handler/IpFilterHandler.cs b/Backend/Handler/IpFilterHandler.cs new file mode 100644 index 0000000..7462ff7 --- /dev/null +++ b/Backend/Handler/IpFilterHandler.cs @@ -0,0 +1,118 @@ +using System.Collections.Concurrent; +using Backend.Helper; +using Models.Model.Backend; + +namespace Backend.Handler; + +public class IpFilterHandler +{ + private readonly ConcurrentQueue _discardedQueue; + private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _preFilteredQueue; + private bool _stop; + private int _timeout; + + public IpFilterHandler(ConcurrentQueue discardedQueue, + ConcurrentQueue unfilteredQueue, + ConcurrentQueue filteredQueue) + { + _discardedQueue = discardedQueue; + _unfilteredQueue = unfilteredQueue; + _preFilteredQueue = filteredQueue; + + _timeout = 16; + } + + public List Start(int threadCount) + { + WaitHandle[] waitHandle = new WaitHandle[64]; + + int counter = 0; + + List waitHandles = []; + + for (int i = 0; i < threadCount; i++) + { + EventWaitHandle handle = new(false, EventResetMode.ManualReset); + + if (counter < 64) + { + waitHandle[counter] = handle; + counter++; + + Thread f = new (Filter!); + f.Start(handle); + + Console.WriteLine($"Filter thread ({i}) started"); + Thread.Sleep(25); + + continue; + } + + counter = 0; + + waitHandles.Add(waitHandle); + + waitHandle = new WaitHandle[64]; + } + + return waitHandles; + } + + private void Filter(object obj) + { + while (!_stop) + { + if (_preFilteredQueue.IsEmpty) + { + Thread.Sleep(_timeout); + continue; + } + + _preFilteredQueue.TryDequeue(out FilterQueueItem item); + + (int, int) ports = TcpClientHelper.CheckPort(item.Ip, 80, 443); + + if (ports is { Item1: 0, Item2: 0 }) + { + _discardedQueue.Enqueue(CreateDiscardedQueueItem(item.Ip, item.ResponseCode)); + continue; + } + + _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(item.Ip, ports)); + } + + ((EventWaitHandle) obj).Set(); + } + + private static Discarded CreateDiscardedQueueItem(Ip ip, int responseCode) + { + return new() + { + Ip = ip, + ResponseCode = responseCode + }; + } + + private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) + { + Unfiltered unfiltered = new() + { + Ip = ip, + Port1 = ports.Item1, + Port2 = ports.Item2, + Filtered = false + }; + + return new() + { + Unfiltered = unfiltered, + Operations = Operations.Insert + }; + } + + public void Stop() + { + _stop = true; + } +} \ No newline at end of file diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index cd4f360..2971008 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -18,22 +18,22 @@ public class ScanSettings public class IpScanner { private readonly ConcurrentQueue _discardedQueue; - private readonly ConcurrentQueue _unfilteredQueue; + private readonly ConcurrentQueue _preFilteredQueue; private readonly ConcurrentQueue _resumeQueue; private readonly DbHandler _dbHandler; private bool _stop; private int _timeout; - public IpScanner(ConcurrentQueue unfilteredQueue, ConcurrentQueue discardedQueue, - ConcurrentQueue resumeQueue, DbHandler dbHandler - ) + public IpScanner(ConcurrentQueue discardedQueue, + ConcurrentQueue resumeQueue, DbHandler dbHandler, + ConcurrentQueue preFilteredQueue) { _dbHandler = dbHandler; + _preFilteredQueue = preFilteredQueue; _discardedQueue = discardedQueue; - _unfilteredQueue = unfilteredQueue; _resumeQueue = resumeQueue; - SetTimeout(64); + SetTimeout(128); } public void SetTimeout(int milliseconds) @@ -76,7 +76,7 @@ public class IpScanner f.Start(scanSettings); Console.WriteLine($"Scanner thread ({i}) started"); - Thread.Sleep(100); + Thread.Sleep(50); continue; } @@ -138,7 +138,12 @@ public class IpScanner if (_discardedQueue.Count >= 2000) { - Thread.Sleep(500); + Thread.Sleep(1000); + } + + if (_preFilteredQueue.Count >= 2000) + { + Thread.Sleep(1000); } for (int l = fourthByte; l < 256; l++) @@ -156,7 +161,7 @@ public class IpScanner Ip3 = k, Ip4 = l }; - + IPStatus responseCode = IPStatus.Unknown; try @@ -166,8 +171,8 @@ public class IpScanner _ = IPAddress.TryParse(ip.ToString(), out IPAddress? address); if (address is not null) { - responseCode = IPStatus.TimedOut; //ping.Send(address, _timeout, buf, null).Status; - Thread.Sleep(_timeout); + responseCode = /*IPStatus.TimedOut;*/ ping.Send(address, _timeout, buf, null).Status; + //Thread.Sleep(16); } } catch @@ -181,15 +186,7 @@ public class IpScanner continue; } - (int, int) ports = TcpClientHelper.CheckPort(ip.ToString(), 80, 443); - - if (ports is { Item1: 0, Item2: 0 }) - { - _discardedQueue.Enqueue(CreateDiscardedQueueItem(ip, (int)responseCode)); - continue; - } - - _unfilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, ports)); + _preFilteredQueue.Enqueue(CreateUnfilteredQueueItem(ip, (int)responseCode)); } if (_stop) @@ -229,21 +226,15 @@ public class IpScanner }; } - private static UnfilteredQueueItem CreateUnfilteredQueueItem(Ip ip, (int, int) ports) + private static FilterQueueItem CreateUnfilteredQueueItem(Ip ip, int responseCode) { - Unfiltered unfiltered = new() + FilterQueueItem filterQueueItem = new() { Ip = ip, - Port1 = ports.Item1, - Port2 = ports.Item2, - Filtered = false + ResponseCode = responseCode }; - return new() - { - Unfiltered = unfiltered, - Operations = Operations.Insert - }; + return filterQueueItem; } public void Stop() diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 4cebb5a..bafcdac 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -10,10 +10,12 @@ public class ThreadHandler private readonly Communication _communication; private readonly IpScanner _ipScanner; private readonly ContentFilter _contentFilter; + private readonly IpFilterHandler _ipFilterHandler; private bool _communicationStopped; private bool _ipScannerStopped; private bool _contentFilterStopped; + private bool _ipFilterStopped; public ThreadHandler(string path) { @@ -21,16 +23,19 @@ public class ThreadHandler ConcurrentQueue discardedQueue = new(); ConcurrentQueue unfilteredQueue = new(); ConcurrentQueue scannerResumeQueue = new(); + ConcurrentQueue preFilteredQueue = new(); _dbHandler = new(filteredQueue, discardedQueue, unfilteredQueue, scannerResumeQueue, path); - _ipScanner = new(unfilteredQueue, discardedQueue, scannerResumeQueue, _dbHandler); + _ipScanner = new(discardedQueue, scannerResumeQueue, _dbHandler, preFilteredQueue); _contentFilter = new(filteredQueue, unfilteredQueue, _dbHandler, path); _communication = new(_dbHandler, this, _ipScanner, _contentFilter, path); + _ipFilterHandler = new(discardedQueue, unfilteredQueue, preFilteredQueue); } public void Start() { Thread scanner = new(StartScanner); + Thread ipFilter = new(StartIpFilter); Thread indexer = new(StartContentFilter); Thread database = new(StartDbHandler); Thread discarded = new(StartDiscardedDbHandler); @@ -39,6 +44,7 @@ public class ThreadHandler Thread communication = new(StartCommunicationHandler); scanner.Start(); + ipFilter.Start(); indexer.Start(); database.Start(); discarded.Start(); @@ -47,6 +53,7 @@ public class ThreadHandler communication.Start(); scanner.Join(); + ipFilter.Join(); indexer.Join(); database.Join(); discarded.Join(); @@ -57,7 +64,7 @@ public class ThreadHandler private void StartScanner() { - Thread.Sleep(5000); // Let the database handler instantiate and warm up first. + Thread.Sleep(15000); // Let the database handler instantiate and warm up first. List wait = _ipScanner.Start(256); @@ -83,6 +90,22 @@ public class ThreadHandler _contentFilterStopped = true; } + + private void StartIpFilter() + { + Thread.Sleep(1000); + + List wait = _ipFilterHandler.Start(256); + + for (int i = 0; i < wait.Count; i++) + { + WaitHandle.WaitAll(wait[i]); + } + + Console.WriteLine("Ip filter finished"); + + _ipFilterStopped = true; + } private void StartDbHandler() { @@ -101,7 +124,7 @@ public class ThreadHandler private void StartDiscardedDbHandler() { - WaitHandle[] wait = _dbHandler.Start(5); + WaitHandle[] wait = _dbHandler.Start(4); WaitHandle.WaitAll(wait); @@ -130,7 +153,7 @@ public class ThreadHandler while (stopping) { - if (_communicationStopped && _ipScannerStopped && _contentFilterStopped) + if (_communicationStopped && _ipScannerStopped && _contentFilterStopped && _ipFilterStopped) { _dbHandler.Stop(); stopping = false; diff --git a/Backend/Helper/TcpClientHelper.cs b/Backend/Helper/TcpClientHelper.cs index 4e01df5..a448cfd 100644 --- a/Backend/Helper/TcpClientHelper.cs +++ b/Backend/Helper/TcpClientHelper.cs @@ -1,20 +1,23 @@ -using System.Net; using System.Net.Sockets; +using Models.Model.Backend; namespace Backend.Helper; public static class TcpClientHelper { - public static (int, int) CheckPort(string ip, params int[] ports) + public static (int, int) CheckPort(Ip ip, params int[] ports) { // This would be way cleaner if the TcpClient didn't throw an exception if the destination couldn't be reached, // and it would just return a result.error, for example. for (int i = 0; i < ports.Length; i++) { + using Socket socket = new(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket.SendTimeout = 250; + try { - using TcpClient client = new(); - client.Connect(ip, ports[i]); - // If the connection is successful, update the result array with the port number + socket.Connect(ip.ToString(), ports[i]); + socket.Close(); + // If the connection is not successful, update the ports array with 0. } catch { diff --git a/Backend/Scripts/443Header.txt b/Backend/Scripts/443Header.txt new file mode 100644 index 0000000..a663ab6 --- /dev/null +++ b/Backend/Scripts/443Header.txt @@ -0,0 +1,9 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 188.0.5.111:443... +* Connected to 188.0.5.111 (188.0.5.111) port 443 +* ALPN: curl offers h2,http/1.1 +} [5 bytes data] +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +} [512 bytes data] + 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 \ No newline at end of file diff --git a/Backend/Scripts/80Header.txt b/Backend/Scripts/80Header.txt new file mode 100644 index 0000000..00cc4a2 --- /dev/null +++ b/Backend/Scripts/80Header.txt @@ -0,0 +1,7 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Trying 188.0.5.111:80... + 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0* Connection timed out after 10002 milliseconds + 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 +* closing connection #0 +curl: (28) Connection timed out after 10002 milliseconds diff --git a/Models/BackupDB/Filtered.db b/Models/BackupDB/Filtered.db index a1172a1..ccb8ab2 100644 Binary files a/Models/BackupDB/Filtered.db and b/Models/BackupDB/Filtered.db differ diff --git a/Models/Handler/DbHandler.cs b/Models/Handler/DbHandler.cs index 282eaf8..5b8136d 100644 --- a/Models/Handler/DbHandler.cs +++ b/Models/Handler/DbHandler.cs @@ -27,13 +27,13 @@ public class DbHandler private const string InsertIntoFiltered = "PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;" + " PRAGMA journal_mode = MEMORY; PRAGMA foreign_keys = on;" + - " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2, Title1, Title2," + - " Description1, Description2, Url1, Url2, ServerType1, ServerType2," + + " INSERT INTO Filtered (Ip1, Ip2, Ip3, Ip4, Port1, Port2," + + " Url1, Url2, ServerType1, ServerType2," + " RobotsTXT1, RobotsTXT2, HttpVersion1, HttpVersion2, CertificateIssuerCountry," + " CertificateOrganizationName, IpV6, TlsVersion, CipherSuite, KeyExchangeAlgorithm," + " PublicKeyType1, PublicKeyType2, PublicKeyType3, AcceptEncoding1, AcceptEncoding2," + " ALPN, Connection1, Connection2) VALUES (@ip1, @ip2, @ip3, @ip4, @port1, @port2, " + - " @title1, @title2, @description1, @description2, @url1, @url2, " + + " @url1, @url2, " + " (SELECT ServerId FROM ServerType WHERE Type = @serverType1), " + " (SELECT ServerId FROM ServerType WHERE Type = @serverType2), " + " @robotsTXT1, @robotsTXT2," + @@ -256,7 +256,7 @@ public class DbHandler continue; } - if (i >= 50_000_000 && !_compressing) + if (i >= 500_000 && !_compressing) { _compressing = true; @@ -427,10 +427,6 @@ public class DbHandler command.Parameters.AddWithValue("@port2", filtered.Port2); command.Parameters.AddWithValue("@url1", filtered.Url1); command.Parameters.AddWithValue("@url2", filtered.Url2); - command.Parameters.AddWithValue("@title1", filtered.Title1); - command.Parameters.AddWithValue("@title2", filtered.Title2); - command.Parameters.AddWithValue("@description1", filtered.Description1); - command.Parameters.AddWithValue("@description2", filtered.Description2); command.Parameters.AddWithValue("@serverType1", filtered.ServerType1); command.Parameters.AddWithValue("@serverType2", filtered.ServerType2); command.Parameters.AddWithValue("@robotsTXT1", filtered.RobotsTXT1); diff --git a/Models/Helper/CompressionHelper.cs b/Models/Helper/CompressionHelper.cs index 319b9ff..64b8a5c 100644 --- a/Models/Helper/CompressionHelper.cs +++ b/Models/Helper/CompressionHelper.cs @@ -8,7 +8,7 @@ public static class CompressionHelper { using FileStream originalFileStream = new(sourceFile, FileMode.Open); using FileStream compressedFileStream = File.Create($"{targetFile}.gz"); - using GZipStream compressor = new(compressedFileStream, CompressionLevel.Fastest); + using GZipStream compressor = new(compressedFileStream, CompressionLevel.SmallestSize); originalFileStream.CopyTo(compressor); } diff --git a/Models/Model/Backend/FilterQueueItem.cs b/Models/Model/Backend/FilterQueueItem.cs new file mode 100644 index 0000000..e7b72c2 --- /dev/null +++ b/Models/Model/Backend/FilterQueueItem.cs @@ -0,0 +1,7 @@ +namespace Models.Model.Backend; + +public struct FilterQueueItem +{ + public Ip Ip { get; init; } + public int ResponseCode { get; init; } +} \ No newline at end of file diff --git a/Models/Model/Backend/Filtered.cs b/Models/Model/Backend/Filtered.cs index 5499d3e..0e9a4c6 100644 --- a/Models/Model/Backend/Filtered.cs +++ b/Models/Model/Backend/Filtered.cs @@ -3,10 +3,6 @@ namespace Models.Model.Backend; public class Filtered { public Ip Ip { get; set; } - public string Title1 { get; set; } = ""; - public string Title2 { get; set; } = ""; - public string Description1 { get; set; } = ""; - public string Description2 { get; set; } = ""; public string Url1 { get; set; } = ""; public string Url2 { get; set; } = ""; public int Port1 { get; set; } diff --git a/RSE.sln b/RSE.sln index 7851f8c..42c9599 100644 --- a/RSE.sln +++ b/RSE.sln @@ -8,6 +8,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Manager", "Manager\Manager. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proxy", "Proxy\Proxy.csproj", "{55208481-5203-4B25-A20D-4EF644F76773}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Analyze", "Analyze\Analyze.csproj", "{7B0C666E-DC4F-4008-9933-08AF5FAB0099}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,9 @@ Global {55208481-5203-4B25-A20D-4EF644F76773}.Debug|Any CPU.Build.0 = Debug|Any CPU {55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.ActiveCfg = Release|Any CPU {55208481-5203-4B25-A20D-4EF644F76773}.Release|Any CPU.Build.0 = Release|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7B0C666E-DC4F-4008-9933-08AF5FAB0099}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index a46370d..e7d85bb 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -2,14 +2,22 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded \ No newline at end of file diff --git a/Type Dependencies Diagram for Communication and other elements.png b/Type Dependencies Diagram for Communication and other elements.png new file mode 100644 index 0000000..3c0f490 Binary files /dev/null and b/Type Dependencies Diagram for Communication and other elements.png differ