using System.Collections.Concurrent; using System.Diagnostics; using Backend.Helper; using Models.Handler; using Models.Model.Backend; namespace Backend.Handler; public class Content { public int Port1 { get; set; } public int Port2 { get; set; } public Ip Ip { get; set; } } public class ContentThread { public int ThreadId { get; set; } public EventWaitHandle? EventWaitHandle { get; set; } } public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly ConcurrentQueue _unfilteredQueue; private readonly ConcurrentQueue _contentQueue = new(); private readonly DbHandler _dbHandler; private readonly ThreadHandler _threadHandler; private readonly string _getDomainPort80; private readonly string _getDomainPort443; private bool _stop; private int _timeOut; private readonly string _basePath; public ContentFilter(ConcurrentQueue queue, ConcurrentQueue unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler) { _queue = queue; _dbHandler = dbHandler; _basePath = basePath; _threadHandler = threadHandler; _unfilteredQueue = unfilteredQueue; _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; SetTimeout(3000); } public void SetTimeout(int timeOut) { _timeOut = timeOut; } public WaitHandle[] Start() { WaitHandle[] waitHandles = new WaitHandle[1]; EventWaitHandle handle = new(false, EventResetMode.ManualReset); waitHandles[0] = handle; Thread f = new (Filter!); f.Start(handle); return waitHandles; } private void Filter(object obj) { while (!_stop) { List indexes = _dbHandler.GetUnfilteredIndexes(); if (indexes.Count == 0) { _stop = true; _threadHandler.Stop(); break; } for (int i = 0; i < indexes.Count; i++) { if (_stop) break; if (_contentQueue.Count >= 500) { Thread.Sleep(1000); } Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]); if (unfiltered.Filtered) continue; Ip ip = unfiltered.Ip; unfiltered.Filtered = true; UnfilteredQueueItem superUnfilteredObject = new() { Unfiltered = unfiltered, Operations = Operations.Update }; _unfilteredQueue.Enqueue(superUnfilteredObject); if (_dbHandler.FilteredIpExists(unfiltered.Ip)) { continue; } Content content = new(); content.Ip = ip; content.Port1 = unfiltered.Port1; content.Port2 = unfiltered.Port2; _contentQueue.Enqueue(content); } Thread.Sleep(_timeOut); } ((EventWaitHandle) obj).Set(); } public WaitHandle[] StartFilterThread(int threads) { WaitHandle[] waitHandle = new WaitHandle[threads]; for (int i = 0; i < threads; i++) { EventWaitHandle handle = new(false, EventResetMode.ManualReset); ContentThread contentThread = new(); contentThread.ThreadId = i; contentThread.EventWaitHandle = handle; waitHandle[i] = handle; Thread thread = new(FilterThread!); thread.Start(contentThread); Thread.Sleep(8); } return waitHandle; } private void FilterThread(object obj) { Console.WriteLine("Filter Thread started."); ContentThread thread = (ContentThread) obj; while (!_stop) { if (_contentQueue.IsEmpty) { Thread.Sleep(1000); } _contentQueue.TryDequeue(out Content? content); if (content is null) { continue; } Filtered filtered = GetSiteData(content.Ip, thread.ThreadId); filtered.Port1 = content.Port1; filtered.Port2 = content.Port2; _queue.Enqueue(filtered); } thread.EventWaitHandle!.Set(); } private Filtered GetSiteData(Ip ip, int threadId) { StartProcess(ip, 80, threadId); StartProcess(ip, 443, threadId); string url1 = ""; string url2 = ""; bool robotsTxt1 = false; bool robotsTxt2 = false; string serverType1 = ""; string serverType2 = ""; string httpVersion1 = ""; string httpVersion2 = ""; string alpn = ""; string certificateIssuerCountry = ""; string certificateOrganizationName = ""; string ipV6 = ""; string tlsVersion = ""; string cipherSuite = ""; string keyExchangeAlgorithm = ""; string publicKeyType1 = ""; string publicKeyType2 = ""; string publicKeyType3 = ""; string acceptEncoding1 = ""; string acceptEncoding2 = ""; string connection1 = ""; string connection2 = ""; int[] ports = [80, 443]; for (int i = 0; i < ports.Length; i++) { using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt"); while (streamReader.Peek() != -1) { string? line = streamReader.ReadLine(); if (string.IsNullOrWhiteSpace(line)) continue; if (ports[i] == 80 && string.IsNullOrWhiteSpace(url1)) { FilterHelper.GetDomain(line, out url1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(url2)) { FilterHelper.GetDomain(line, out url2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(serverType1)) { FilterHelper.GetServerType(line, out serverType1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(serverType2)) { FilterHelper.GetServerType(line, out serverType2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(httpVersion1)) { FilterHelper.GetHttpVersion(line, out httpVersion1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(httpVersion2)) { FilterHelper.GetHttpVersion(line, out httpVersion2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(alpn)) { FilterHelper.GetALPN(line, out alpn); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateIssuerCountry)) { FilterHelper.GetCertificateIssuerCountry(line, out certificateIssuerCountry); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateOrganizationName)) { FilterHelper.GetCertificateOrganizationName(line, out certificateOrganizationName); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(ipV6)) { FilterHelper.GetIpV6(line, out ipV6); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(tlsVersion)) { FilterHelper.GetTlsVersion(line, out tlsVersion); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(connection1)) { FilterHelper.GetConnection(line, out connection1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(connection2)) { FilterHelper.GetConnection(line, out connection2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(acceptEncoding1)) { FilterHelper.GetEncoding(line, out acceptEncoding1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(acceptEncoding2)) { FilterHelper.GetEncoding(line, out acceptEncoding2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType1)) { FilterHelper.GetPublicKeyType(line, out publicKeyType1, "Certificate level 0: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType2)) { FilterHelper.GetPublicKeyType(line, out publicKeyType2, "Certificate level 1: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType3)) { FilterHelper.GetPublicKeyType(line, out publicKeyType3, "Certificate level 2: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(cipherSuite)) { FilterHelper.GetCipherSuite(line, out cipherSuite); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(keyExchangeAlgorithm)) { FilterHelper.GetKeyExchangeAlgorithm(line, out keyExchangeAlgorithm); } } } for (int i = 0; i < ports.Length; i++) { if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); } if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); } } Filtered siteData = new() { Ip = ip, Url1 = url1, Url2 = url2, ServerType1 = serverType1, ServerType2 = serverType2, RobotsTXT1 = robotsTxt1, RobotsTXT2 = robotsTxt2, HttpVersion1 = httpVersion1, HttpVersion2 = httpVersion2, ALPN = alpn, CertificateIssuerCountry = certificateIssuerCountry, CertificateOrganizationName = certificateOrganizationName, IpV6 = ipV6, TlsVersion = tlsVersion, CipherSuite = cipherSuite, KeyExchangeAlgorithm = keyExchangeAlgorithm, PublicKeyType1 = publicKeyType1, PublicKeyType2 = publicKeyType2, PublicKeyType3 = publicKeyType3, AcceptEncoding1 = acceptEncoding1, AcceptEncoding2 = acceptEncoding2, Connection1 = connection1, Connection2 = connection2, }; return siteData; } private void StartProcess(Ip ip, int port, int threadId) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; Process proc = new(); proc.StartInfo = new() { FileName = "/bin/bash", Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, CreateNoWindow = true }; try { proc.Start(); proc.WaitForExit(); } catch (Exception e) { Console.WriteLine(e); } finally { proc.Close(); proc.Dispose(); } } public void Stop() { _stop = true; } }