using System.Collections.Concurrent; using System.Diagnostics; using Backend.Helper; using Models.Handler; using Models.Model.Backend; namespace Backend.Handler; public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly ConcurrentQueue _unfilteredQueue; private readonly DbHandler _dbHandler; private readonly string _getDomainPort80; private readonly string _getDomainPort443; private bool _stop; private int _timeOut; private readonly string _basePath; public ContentFilter(ConcurrentQueue queue, ConcurrentQueue unfilteredQueue, DbHandler dbHandler, string basePath) { _queue = queue; _dbHandler = dbHandler; _basePath = basePath; _unfilteredQueue = unfilteredQueue; _getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh"; _getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh"; SetTimeout(3000); } public void SetTimeout(int timeOut) { _timeOut = timeOut; } public WaitHandle[] Start() { WaitHandle[] waitHandles = new WaitHandle[1]; EventWaitHandle handle = new(false, EventResetMode.ManualReset); waitHandles[0] = handle; Thread f = new (Filter!); f.Start(handle); return waitHandles; } private void Filter(object obj) { while (!_stop) { List indexes = _dbHandler.GetUnfilteredIndexes(); for (int i = 0; i < indexes.Count; i++) { if (_stop) break; Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]); if (unfiltered.Filtered) continue; Ip ip = unfiltered.Ip; unfiltered.Filtered = true; UnfilteredQueueItem superUnfilteredObject = new() { Unfiltered = unfiltered, Operations = Operations.Update }; _unfilteredQueue.Enqueue(superUnfilteredObject); if (_dbHandler.FilteredIpExists(unfiltered.Ip)) { continue; } Filtered filtered = GetSiteData(ip); filtered.Port1 = unfiltered.Port1; filtered.Port2 = unfiltered.Port2; _queue.Enqueue(filtered); } Thread.Sleep(_timeOut); } ((EventWaitHandle) obj).Set(); } private Filtered GetSiteData(Ip ip) { StartProcess(ip, 80); StartProcess(ip, 443); string url1 = ""; string url2 = ""; string title1 = ""; string title2 = ""; string description1 = ""; string description2 = ""; bool robotsTxt1 = false; bool robotsTxt2 = false; string serverType1 = ""; string serverType2 = ""; string httpVersion1 = ""; string httpVersion2 = ""; string alpn = ""; string certificateIssuerCountry = ""; string certificateOrganizationName = ""; string ipV6 = ""; string tlsVersion = ""; string cipherSuite = ""; string keyExchangeAlgorithm = ""; string publicKeyType1 = ""; string publicKeyType2 = ""; string publicKeyType3 = ""; string acceptEncoding1 = ""; string acceptEncoding2 = ""; string connection1 = ""; string connection2 = ""; int[] ports = [80, 443]; for (int i = 0; i < ports.Length; i++) { using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header.txt"); while (streamReader.Peek() != -1) { string? line = streamReader.ReadLine(); if (string.IsNullOrWhiteSpace(line)) continue; if (ports[i] == 80 && string.IsNullOrWhiteSpace(url1)) { FilterHelper.GetDomain(line, out url1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(url2)) { FilterHelper.GetDomain(line, out url2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(serverType1)) { FilterHelper.GetServerType(line, out serverType1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(serverType2)) { FilterHelper.GetServerType(line, out serverType2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(httpVersion1)) { FilterHelper.GetHttpVersion(line, out httpVersion1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(httpVersion2)) { FilterHelper.GetHttpVersion(line, out httpVersion2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(alpn)) { FilterHelper.GetALPN(line, out alpn); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateIssuerCountry)) { FilterHelper.GetCertificateIssuerCountry(line, out certificateIssuerCountry); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateOrganizationName)) { FilterHelper.GetCertificateOrganizationName(line, out certificateOrganizationName); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(ipV6)) { FilterHelper.GetIpV6(line, out ipV6); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(tlsVersion)) { FilterHelper.GetTlsVersion(line, out tlsVersion); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(connection1)) { FilterHelper.GetConnection(line, out connection1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(connection2)) { FilterHelper.GetConnection(line, out connection2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(acceptEncoding1)) { FilterHelper.GetEncoding(line, out acceptEncoding1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(acceptEncoding2)) { FilterHelper.GetEncoding(line, out acceptEncoding2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType1)) { FilterHelper.GetPublicKeyType(line, out publicKeyType1, "Certificate level 0: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType2)) { FilterHelper.GetPublicKeyType(line, out publicKeyType2, "Certificate level 1: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType3)) { FilterHelper.GetPublicKeyType(line, out publicKeyType3, "Certificate level 2: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(cipherSuite)) { FilterHelper.GetCipherSuite(line, out cipherSuite); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(keyExchangeAlgorithm)) { FilterHelper.GetKeyExchangeAlgorithm(line, out keyExchangeAlgorithm); } } } for (int i = 0; i < ports.Length; i++) { string? html = ""; if (ports[i] == 80) { if (string.IsNullOrWhiteSpace(url1)) continue; try { html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult(); } catch (Exception e) { Console.WriteLine(e); } } else { if (string.IsNullOrWhiteSpace(url2)) continue; try { html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult(); } catch (Exception e) { Console.WriteLine(e); } } if (string.IsNullOrWhiteSpace(html)) continue; if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); } if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); } if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); } } Filtered siteData = new() { Ip = ip, Url1 = url1, Url2 = url2, Title1 = title1, Title2 = title2, Description1 = description1, Description2 = description2, ServerType1 = serverType1, ServerType2 = serverType2, RobotsTXT1 = robotsTxt1, RobotsTXT2 = robotsTxt2, HttpVersion1 = httpVersion1, HttpVersion2 = httpVersion2, ALPN = alpn, CertificateIssuerCountry = certificateIssuerCountry, CertificateOrganizationName = certificateOrganizationName, IpV6 = ipV6, TlsVersion = tlsVersion, CipherSuite = cipherSuite, KeyExchangeAlgorithm = keyExchangeAlgorithm, PublicKeyType1 = publicKeyType1, PublicKeyType2 = publicKeyType2, PublicKeyType3 = publicKeyType3, AcceptEncoding1 = acceptEncoding1, AcceptEncoding2 = acceptEncoding2, Connection1 = connection1, Connection2 = connection2, }; return siteData; } private void StartProcess(Ip ip, int port) { string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443; Process proc = new(); proc.StartInfo = new() { FileName = "/bin/bash", Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header.txt", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, CreateNoWindow = true }; try { proc.Start(); proc.WaitForExit(); } catch (Exception e) { Console.WriteLine(e); } finally { proc.Close(); proc.Dispose(); } } public void Stop() { _stop = true; } }