using System.Collections.Concurrent; using System.Diagnostics; using System.Text; using Backend.Helper; using Models.Handler; using Models.Model.Backend; namespace Backend.Handler; public class ContentFilter { private readonly ConcurrentQueue _queue; private readonly DbHandler _dbHandler; private const string GetDomainPort80 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort80.sh"; private const string GetDomainPort443 = "/home/skingging/Documents/Projects/CSharp/RSE/Backend/Scripts/GetDomainNamePort443.sh"; private bool _stop; public ContentFilter(ConcurrentQueue queue, DbHandler dbHandler) { _queue = queue; _dbHandler = dbHandler; } public WaitHandle[] Start() { WaitHandle[] waitHandles = new WaitHandle[1]; EventWaitHandle handle = new(false, EventResetMode.ManualReset); waitHandles[0] = handle; Thread f = new (Filter!); f.Start(handle); return waitHandles; } private void Filter(object obj) { long indexes = DbHandler.GetUnfilteredIndexes(); for (long i = 0; i < indexes; i++) { if (_stop) break; Unfiltered? unfiltered = DbHandler.ReadUnfilteredWithId(i); if (unfiltered is null || unfiltered.Filtered == 1) continue; unfiltered.Filtered = 1; QueueItem superUnfilteredObject = new() { Unfiltered = unfiltered, Operations = Operations.Update }; _queue.Enqueue(superUnfilteredObject); if (_dbHandler.GetFilteredIp(unfiltered.Ip)) { continue; } Filtered filtered = GetSiteData(unfiltered.Ip); filtered.Port1 = unfiltered.Port1; filtered.Port2 = unfiltered.Port2; QueueItem superFilteredObject = new() { Filtered = filtered, Operations = Operations.Insert }; _queue.Enqueue(superFilteredObject); } ((EventWaitHandle) obj).Set(); } private static Filtered GetSiteData(string ip) { StartProcess(ip, 80); StartProcess(ip, 443); string url1 = ""; string url2 = ""; string title1 = ""; string title2 = ""; string description1 = ""; string description2 = ""; bool robotsTxt1 = false; bool robotsTxt2 = false; string serverType1 = ""; string serverType2 = ""; string httpVersion1 = ""; string httpVersion2 = ""; string alpn = ""; string certificateIssuerCountry = ""; string certificateOrganizationName = ""; string ipV6 = ""; string tlsVersion = ""; string cipherSuite = ""; string keyExchangeAlgorithm = ""; string publicKeyType1 = ""; string publicKeyType2 = ""; string publicKeyType3 = ""; string acceptEncoding1 = ""; string acceptEncoding2 = ""; string connection1 = ""; string connection2 = ""; int[] ports = [80, 443]; for (int i = 0; i < ports.Length; i++) { using StreamReader streamReader = new($"{ports[i]}Header.txt"); while (streamReader.Peek() != -1) { string? line = streamReader.ReadLine(); if (string.IsNullOrWhiteSpace(line)) continue; if (ports[i] == 80 && string.IsNullOrWhiteSpace(url1)) { FilterHelper.GetDomain(line, out url1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(url2)) { FilterHelper.GetDomain(line, out url2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(serverType1)) { FilterHelper.GetServerType(line, out serverType1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(serverType2)) { FilterHelper.GetServerType(line, out serverType2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(httpVersion1)) { FilterHelper.GetHttpVersion(line, out httpVersion1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(httpVersion2)) { FilterHelper.GetHttpVersion(line, out httpVersion2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(alpn)) { FilterHelper.GetALPN(line, out alpn); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateIssuerCountry)) { FilterHelper.GetCertificateIssuerCountry(line, out certificateIssuerCountry); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateOrganizationName)) { FilterHelper.GetCertificateOrganizationName(line, out certificateOrganizationName); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(ipV6)) { FilterHelper.GetIpV6(line, out ipV6); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(tlsVersion)) { FilterHelper.GetTlsVersion(line, out tlsVersion); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(connection1)) { FilterHelper.GetConnection(line, out connection1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(connection2)) { FilterHelper.GetConnection(line, out connection2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(acceptEncoding1)) { FilterHelper.GetEncoding(line, out acceptEncoding1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(acceptEncoding2)) { FilterHelper.GetEncoding(line, out acceptEncoding2); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType1)) { FilterHelper.GetPublicKeyType(line, out publicKeyType1, "Certificate level 0: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType2)) { FilterHelper.GetPublicKeyType(line, out publicKeyType2, "Certificate level 1: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType3)) { FilterHelper.GetPublicKeyType(line, out publicKeyType3, "Certificate level 2: Public key type "); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(cipherSuite)) { FilterHelper.GetCipherSuite(line, out cipherSuite); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(keyExchangeAlgorithm)) { FilterHelper.GetKeyExchangeAlgorithm(line, out keyExchangeAlgorithm); } } } for (int i = 0; i < ports.Length; i++) { string? html; if (ports[i] == 80) { if (string.IsNullOrWhiteSpace(url1)) continue; html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result; } else { if (string.IsNullOrWhiteSpace(url2)) continue; html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result; } if (string.IsNullOrWhiteSpace(html)) continue; if (ports[i] == 80 && string.IsNullOrWhiteSpace(title1)) { FilterHelper.GetTitle(html, out title1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); } if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; } if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; } } Filtered siteData = new() { Ip = ip, Url1 = url1, Url2 = url2, Title1 = title1, Title2 = title2, Description1 = description1, Description2 = description2, ServerType1 = serverType1, ServerType2 = serverType2, RobotsTXT1 = robotsTxt1, RobotsTXT2 = robotsTxt2, HttpVersion1 = httpVersion1, HttpVersion2 = httpVersion2, ALPN = alpn, CertificateIssuerCountry = certificateIssuerCountry, CertificateOrganizationName = certificateOrganizationName, IpV6 = ipV6, TlsVersion = tlsVersion, CipherSuite = cipherSuite, KeyExchangeAlgorithm = keyExchangeAlgorithm, PublicKeyType1 = publicKeyType1, PublicKeyType2 = publicKeyType2, PublicKeyType3 = publicKeyType3, AcceptEncoding1 = acceptEncoding1, AcceptEncoding2 = acceptEncoding2, Connection1 = connection1, Connection2 = connection2, }; return siteData; } private static void StartProcess(string ip, int port) { string fileName = port == 80 ? GetDomainPort80 : GetDomainPort443; Process proc = new(); proc.StartInfo = new() { FileName = fileName, Arguments = $"{ip}", UseShellExecute = false, RedirectStandardOutput = false, RedirectStandardError = false, RedirectStandardInput = false, CreateNoWindow = true }; try { proc.Start(); proc.WaitForExit(); } catch (Exception e) { Console.WriteLine(e); } finally { proc.Close(); proc.Dispose(); } } public void Stop() { _stop = true; } }