309 lines
11 KiB
C#
309 lines
11 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using Backend.Helper;
|
|
using Models.Handler;
|
|
using Models.Model.Backend;
|
|
|
|
namespace Backend.Handler;
|
|
|
|
public class Content
|
|
{
|
|
public int Port1 { get; set; }
|
|
public int Port2 { get; set; }
|
|
public Ip Ip { get; set; }
|
|
}
|
|
|
|
public class ContentThread
|
|
{
|
|
public int ThreadId { get; set; }
|
|
public EventWaitHandle? EventWaitHandle { get; set; }
|
|
}
|
|
|
|
public class ContentFilter
|
|
{
|
|
private readonly ConcurrentQueue<Filtered> _queue;
|
|
private readonly ConcurrentQueue<UnfilteredQueueItem> _unfilteredQueue;
|
|
private readonly ConcurrentQueue<Content?> _contentQueue = new();
|
|
private readonly DbHandler _dbHandler;
|
|
private readonly ThreadHandler _threadHandler;
|
|
private readonly string _getDomainPort80;
|
|
private readonly string _getDomainPort443;
|
|
private bool _stop;
|
|
private int _timeOut;
|
|
private readonly string _basePath;
|
|
|
|
public ContentFilter(ConcurrentQueue<Filtered> queue, ConcurrentQueue<UnfilteredQueueItem> unfilteredQueue, DbHandler dbHandler, string basePath, ThreadHandler threadHandler)
|
|
{
|
|
_queue = queue;
|
|
_dbHandler = dbHandler;
|
|
_basePath = basePath;
|
|
_threadHandler = threadHandler;
|
|
_unfilteredQueue = unfilteredQueue;
|
|
|
|
_getDomainPort80 = $"{basePath}/Backend/Scripts/GetDomainNamePort80.sh";
|
|
_getDomainPort443 = $"{basePath}/Backend/Scripts/GetDomainNamePort443.sh";
|
|
|
|
|
|
SetTimeout(3000);
|
|
}
|
|
|
|
public void SetTimeout(int timeOut)
|
|
{
|
|
_timeOut = timeOut;
|
|
}
|
|
|
|
public WaitHandle[] Start()
|
|
{
|
|
WaitHandle[] waitHandles = new WaitHandle[1];
|
|
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
|
|
waitHandles[0] = handle;
|
|
|
|
Thread f = new (Filter!);
|
|
f.Start(handle);
|
|
|
|
return waitHandles;
|
|
}
|
|
|
|
private void Filter(object obj)
|
|
{
|
|
while (!_stop)
|
|
{
|
|
List<long> indexes = _dbHandler.GetUnfilteredIndexes();
|
|
|
|
if (indexes.Count == 0)
|
|
{
|
|
_stop = true;
|
|
_threadHandler.Stop();
|
|
break;
|
|
}
|
|
|
|
for (int i = 0; i < indexes.Count; i++)
|
|
{
|
|
if (_stop) break;
|
|
|
|
if (_contentQueue.Count >= 500)
|
|
{
|
|
Thread.Sleep(1000);
|
|
}
|
|
|
|
Unfiltered unfiltered = _dbHandler.ReadUnfilteredWithId(indexes[i]);
|
|
|
|
if (unfiltered.Filtered) continue;
|
|
|
|
Ip ip = unfiltered.Ip;
|
|
|
|
unfiltered.Filtered = true;
|
|
|
|
UnfilteredQueueItem superUnfilteredObject = new()
|
|
{
|
|
Unfiltered = unfiltered,
|
|
Operations = Operations.Update
|
|
};
|
|
|
|
_unfilteredQueue.Enqueue(superUnfilteredObject);
|
|
|
|
if (_dbHandler.FilteredIpExists(unfiltered.Ip))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Content content = new();
|
|
content.Ip = ip;
|
|
content.Port1 = unfiltered.Port1;
|
|
content.Port2 = unfiltered.Port2;
|
|
|
|
_contentQueue.Enqueue(content);
|
|
}
|
|
|
|
Thread.Sleep(_timeOut);
|
|
}
|
|
|
|
((EventWaitHandle) obj).Set();
|
|
}
|
|
|
|
public WaitHandle[] StartFilterThread(int threads)
|
|
{
|
|
WaitHandle[] waitHandle = new WaitHandle[threads];
|
|
|
|
for (int i = 0; i < threads; i++)
|
|
{
|
|
EventWaitHandle handle = new(false, EventResetMode.ManualReset);
|
|
ContentThread contentThread = new();
|
|
contentThread.ThreadId = i;
|
|
contentThread.EventWaitHandle = handle;
|
|
waitHandle[i] = handle;
|
|
|
|
Thread thread = new(FilterThread!);
|
|
thread.Start(contentThread);
|
|
|
|
Thread.Sleep(8);
|
|
}
|
|
|
|
return waitHandle;
|
|
}
|
|
|
|
private void FilterThread(object obj)
|
|
{
|
|
Console.WriteLine("Filter Thread started.");
|
|
ContentThread thread = (ContentThread) obj;
|
|
|
|
while (!_stop)
|
|
{
|
|
if (_contentQueue.IsEmpty)
|
|
{
|
|
Thread.Sleep(1000);
|
|
}
|
|
|
|
_contentQueue.TryDequeue(out Content? content);
|
|
|
|
if (content is null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Filtered filtered = GetSiteData(content.Ip, thread.ThreadId);
|
|
|
|
filtered.Port1 = content.Port1;
|
|
filtered.Port2 = content.Port2;
|
|
|
|
_queue.Enqueue(filtered);
|
|
}
|
|
|
|
thread.EventWaitHandle!.Set();
|
|
}
|
|
|
|
private Filtered GetSiteData(Ip ip, int threadId)
|
|
{
|
|
StartProcess(ip, 80, threadId);
|
|
StartProcess(ip, 443, threadId);
|
|
|
|
string url1 = "";
|
|
string url2 = "";
|
|
bool robotsTxt1 = false;
|
|
bool robotsTxt2 = false;
|
|
string serverType1 = "";
|
|
string serverType2 = "";
|
|
string httpVersion1 = "";
|
|
string httpVersion2 = "";
|
|
string alpn = "";
|
|
string certificateIssuerCountry = "";
|
|
string certificateOrganizationName = "";
|
|
string ipV6 = "";
|
|
string tlsVersion = "";
|
|
string cipherSuite = "";
|
|
string keyExchangeAlgorithm = "";
|
|
string publicKeyType1 = "";
|
|
string publicKeyType2 = "";
|
|
string publicKeyType3 = "";
|
|
string acceptEncoding1 = "";
|
|
string acceptEncoding2 = "";
|
|
string connection1 = "";
|
|
string connection2 = "";
|
|
|
|
int[] ports = [80, 443];
|
|
|
|
for (int i = 0; i < ports.Length; i++)
|
|
{
|
|
using StreamReader streamReader = new($"{_basePath}/Backend/Scripts/{ports[i]}Header{threadId}.txt");
|
|
|
|
while (streamReader.Peek() != -1)
|
|
{
|
|
string? line = streamReader.ReadLine();
|
|
|
|
if (string.IsNullOrWhiteSpace(line)) continue;
|
|
|
|
if (ports[i] == 80 && string.IsNullOrWhiteSpace(url1)) { FilterHelper.GetDomain(line, out url1); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(url2)) { FilterHelper.GetDomain(line, out url2); }
|
|
if (ports[i] == 80 && string.IsNullOrWhiteSpace(serverType1)) { FilterHelper.GetServerType(line, out serverType1); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(serverType2)) { FilterHelper.GetServerType(line, out serverType2); }
|
|
if (ports[i] == 80 && string.IsNullOrWhiteSpace(httpVersion1)) { FilterHelper.GetHttpVersion(line, out httpVersion1); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(httpVersion2)) { FilterHelper.GetHttpVersion(line, out httpVersion2); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(alpn)) { FilterHelper.GetALPN(line, out alpn); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateIssuerCountry)) { FilterHelper.GetCertificateIssuerCountry(line, out certificateIssuerCountry); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(certificateOrganizationName)) { FilterHelper.GetCertificateOrganizationName(line, out certificateOrganizationName); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(ipV6)) { FilterHelper.GetIpV6(line, out ipV6); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(tlsVersion)) { FilterHelper.GetTlsVersion(line, out tlsVersion); }
|
|
if (ports[i] == 80 && string.IsNullOrWhiteSpace(connection1)) { FilterHelper.GetConnection(line, out connection1); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(connection2)) { FilterHelper.GetConnection(line, out connection2); }
|
|
if (ports[i] == 80 && string.IsNullOrWhiteSpace(acceptEncoding1)) { FilterHelper.GetEncoding(line, out acceptEncoding1); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(acceptEncoding2)) { FilterHelper.GetEncoding(line, out acceptEncoding2); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType1)) { FilterHelper.GetPublicKeyType(line, out publicKeyType1, "Certificate level 0: Public key type "); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType2)) { FilterHelper.GetPublicKeyType(line, out publicKeyType2, "Certificate level 1: Public key type "); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(publicKeyType3)) { FilterHelper.GetPublicKeyType(line, out publicKeyType3, "Certificate level 2: Public key type "); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(cipherSuite)) { FilterHelper.GetCipherSuite(line, out cipherSuite); }
|
|
if (ports[i] == 443 && string.IsNullOrWhiteSpace(keyExchangeAlgorithm)) { FilterHelper.GetKeyExchangeAlgorithm(line, out keyExchangeAlgorithm); }
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < ports.Length; i++)
|
|
{
|
|
if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); }
|
|
if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); }
|
|
}
|
|
|
|
Filtered siteData = new()
|
|
{
|
|
Ip = ip,
|
|
Url1 = url1,
|
|
Url2 = url2,
|
|
ServerType1 = serverType1,
|
|
ServerType2 = serverType2,
|
|
RobotsTXT1 = robotsTxt1,
|
|
RobotsTXT2 = robotsTxt2,
|
|
HttpVersion1 = httpVersion1,
|
|
HttpVersion2 = httpVersion2,
|
|
ALPN = alpn,
|
|
CertificateIssuerCountry = certificateIssuerCountry,
|
|
CertificateOrganizationName = certificateOrganizationName,
|
|
IpV6 = ipV6,
|
|
TlsVersion = tlsVersion,
|
|
CipherSuite = cipherSuite,
|
|
KeyExchangeAlgorithm = keyExchangeAlgorithm,
|
|
PublicKeyType1 = publicKeyType1,
|
|
PublicKeyType2 = publicKeyType2,
|
|
PublicKeyType3 = publicKeyType3,
|
|
AcceptEncoding1 = acceptEncoding1,
|
|
AcceptEncoding2 = acceptEncoding2,
|
|
Connection1 = connection1,
|
|
Connection2 = connection2,
|
|
};
|
|
|
|
return siteData;
|
|
}
|
|
|
|
private void StartProcess(Ip ip, int port, int threadId)
|
|
{
|
|
string fileName = port == 80 ? _getDomainPort80 : _getDomainPort443;
|
|
|
|
Process proc = new();
|
|
proc.StartInfo = new()
|
|
{
|
|
FileName = "/bin/bash",
|
|
Arguments = $"{fileName} {ip.Ip1}.{ip.Ip2}.{ip.Ip3}.{ip.Ip4} {_basePath}/Backend/Scripts/{port}Header{threadId}.txt",
|
|
UseShellExecute = false,
|
|
RedirectStandardOutput = false,
|
|
RedirectStandardError = false,
|
|
CreateNoWindow = true
|
|
};
|
|
|
|
try
|
|
{
|
|
proc.Start();
|
|
proc.WaitForExit();
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine(e);
|
|
}
|
|
finally
|
|
{
|
|
proc.Close();
|
|
proc.Dispose();
|
|
}
|
|
}
|
|
|
|
public void Stop()
|
|
{
|
|
_stop = true;
|
|
}
|
|
} |