From 5fe35192c6c3080d98327aa5b1553ed5dbfd7590 Mon Sep 17 00:00:00 2001 From: Rasmus Date: Wed, 4 Dec 2024 13:20:45 +0100 Subject: [PATCH] Misc changes such as performance tuning, added support for more than 64 threads for the scanner. --- Backend/Backend.csproj | 4 +- Backend/Handler/ContentFilter.cs | 28 +++++-- Backend/Handler/IpScanner.cs | 45 ++++++++-- Backend/Handler/ThreadHandler.cs | 17 ++-- Backend/Helper/HttpClientHelper.cs | 9 +- Models/BackupDB/Discarded.db | Bin 12288 -> 16384 bytes Models/BackupDB/Filtered.db | Bin 12288 -> 110592 bytes Models/BackupDB/mydb.db | Bin 16384 -> 24576 bytes Models/Handler/DbHandler.cs | 118 ++++++++++++++++++++++++--- Models/Model/Backend/IP.cs | 2 +- Models/Models.csproj | 1 - Models/Sql/DropAndCreateFiltered.sql | 113 ++++++++++++++++++++----- RSE.sln.DotSettings.user | 1 + 13 files changed, 281 insertions(+), 57 deletions(-) diff --git a/Backend/Backend.csproj b/Backend/Backend.csproj index 87db91a..5c26c51 100644 --- a/Backend/Backend.csproj +++ b/Backend/Backend.csproj @@ -6,8 +6,8 @@ enable enable - x64 - true + + false diff --git a/Backend/Handler/ContentFilter.cs b/Backend/Handler/ContentFilter.cs index 706202c..676bb47 100644 --- a/Backend/Handler/ContentFilter.cs +++ b/Backend/Handler/ContentFilter.cs @@ -161,19 +161,33 @@ public class ContentFilter for (int i = 0; i < ports.Length; i++) { - string? html; + string? html = ""; if (ports[i] == 80) { if (string.IsNullOrWhiteSpace(url1)) continue; - - html = Task.Run(() => HttpClientHelper.GetHtml(url1, 80).Result).Result; + + try + { + html = HttpClientHelper.GetHtml(url1, 80).GetAwaiter().GetResult(); + } + catch (Exception e) + { + Console.WriteLine(e); + } } else { if (string.IsNullOrWhiteSpace(url2)) continue; - - html = Task.Run(() => HttpClientHelper.GetHtml(url2, 443).Result).Result; + + try + { + html = HttpClientHelper.GetHtml(url2, 443).GetAwaiter().GetResult(); + } + catch (Exception e) + { + Console.WriteLine(e); + } } if (string.IsNullOrWhiteSpace(html)) continue; @@ -182,8 +196,8 @@ public class ContentFilter if (ports[i] == 443 && string.IsNullOrWhiteSpace(title2)) { FilterHelper.GetTitle(html ,out title2); } if (ports[i] == 80 && string.IsNullOrWhiteSpace(description1)) { FilterHelper.GetDescription(html, out description1); } if (ports[i] == 443 && string.IsNullOrWhiteSpace(description2)) { FilterHelper.GetDescription(html, out description2); } - if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url1, 80).Result).Result; } - if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = Task.Run(() => HttpClientHelper.HasRobotsTxt(url2, 443).Result).Result; } + if (ports[i] == 80 && !robotsTxt1) { robotsTxt1 = HttpClientHelper.HasRobotsTxt(url1, 80).GetAwaiter().GetResult(); } + if (ports[i] == 443 && !robotsTxt2) { robotsTxt2 = HttpClientHelper.HasRobotsTxt(url2, 443).GetAwaiter().GetResult(); } } Filtered siteData = new() diff --git a/Backend/Handler/IpScanner.cs b/Backend/Handler/IpScanner.cs index c4b5e51..c8d2943 100644 --- a/Backend/Handler/IpScanner.cs +++ b/Backend/Handler/IpScanner.cs @@ -41,15 +41,30 @@ public class IpScanner _timeout = milliseconds; } - public WaitHandle[] Start(int threads) + public List Start(int threads) { int threadsAmount = 0; if (threads % 2 == 0) { threadsAmount = 256 / threads; } + + WaitHandle[] waitHandle1; + WaitHandle[] waitHandle2; - WaitHandle[] waitHandles = new WaitHandle[threads]; + if (threads <= 64) + { + waitHandle1 = new WaitHandle[threads]; + waitHandle2 = new WaitHandle[threads]; + } + else + { + waitHandle1 = new WaitHandle[64]; + waitHandle2 = new WaitHandle[64]; + } + + int counter = 0; + int counter2 = 0; for (int i = 0; i < threads; i++) { @@ -62,16 +77,33 @@ public class IpScanner ThreadNumber = i, Handle = handle }; - - waitHandles[i] = handle; + + if (i < 64) + { + waitHandle1[counter] = handle; + counter++; + } + else + { + waitHandle2[counter2] = handle; + counter2++; + } Thread f = new (Scan!); f.Start(scanSettings); Console.WriteLine($"Scanner thread ({i}) started"); - Thread.Sleep(1000); + Thread.Sleep(100); } + List waitHandles = new(); + + Console.WriteLine("Waithandle 1 count = " + waitHandle1.Length); + Console.WriteLine("Waithandle 2 count = " + waitHandle2.Length); + + waitHandles.Add(waitHandle1); + waitHandles.Add(waitHandle2); + return waitHandles; } @@ -193,7 +225,8 @@ public class IpScanner resumeObject.FirstByte = i; break; } - //Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})"); + + Console.WriteLine($"Thread ({scanSettings.ThreadNumber}) is at index ({i}) out of ({scanSettings.End}). Remaining ({scanSettings.End - i})"); } _resumeQueue.Enqueue(resumeObject); diff --git a/Backend/Handler/ThreadHandler.cs b/Backend/Handler/ThreadHandler.cs index 47c9a70..784c270 100644 --- a/Backend/Handler/ThreadHandler.cs +++ b/Backend/Handler/ThreadHandler.cs @@ -59,9 +59,12 @@ public class ThreadHandler { Thread.Sleep(5000); // Let the database handler instantiate and warm up first. - WaitHandle[] wait = _ipScanner.Start(64); - - WaitHandle.WaitAll(wait); + List wait = _ipScanner.Start(128); + + for (int i = 0; i < wait.Count; i++) + { + WaitHandle.WaitAll(wait[i]); + } Console.WriteLine("Scanner finished"); @@ -70,6 +73,8 @@ public class ThreadHandler private void StartContentFilter() { + Thread.Sleep(5000); + WaitHandle[] wait = _contentFilter.Start(); WaitHandle.WaitAll(wait); @@ -83,12 +88,12 @@ public class ThreadHandler { _dbHandler.UnfilteredDbHandler(); } - + private void StartFilteredDbHandler() { _dbHandler.FilteredDbHandler(); } - + private void StartResumeDbHandler() { _dbHandler.ResumeDbHandler(); @@ -96,7 +101,7 @@ public class ThreadHandler private void StartDiscardedDbHandler() { - WaitHandle[] wait = _dbHandler.Start(2); + WaitHandle[] wait = _dbHandler.Start(4); WaitHandle.WaitAll(wait); diff --git a/Backend/Helper/HttpClientHelper.cs b/Backend/Helper/HttpClientHelper.cs index 0d8b36b..0dfe66b 100644 --- a/Backend/Helper/HttpClientHelper.cs +++ b/Backend/Helper/HttpClientHelper.cs @@ -18,9 +18,9 @@ public static class HttpClientHelper } client.DefaultRequestHeaders.Accept.Clear(); - client.Timeout = TimeSpan.FromSeconds(30); + client.Timeout = TimeSpan.FromSeconds(1); - HttpResponseMessage? response = null; + HttpResponseMessage? response; try { @@ -28,10 +28,10 @@ public static class HttpClientHelper } catch { - // + return ""; } - if (response is null || !response.IsSuccessStatusCode) + if (!response.IsSuccessStatusCode) { return ""; } @@ -53,6 +53,7 @@ public static class HttpClientHelper } client.DefaultRequestHeaders.Accept.Clear(); + client.Timeout = TimeSpan.FromSeconds(1); HttpResponseMessage? response = null; diff --git a/Models/BackupDB/Discarded.db b/Models/BackupDB/Discarded.db index 5f296eb54e16b0c555410690f45fee15a5300b27..ddddc9979f78fde1e680425326563baed6d0730b 100644 GIT binary patch delta 251 zcmZojXlP)ZAT7wkz`y{)OhC*C#2gcKjD=Yk^x~y>`MVgH_$D*(TkuWhJ-e|mnm3TY zi!obVU!So_xg;?uC$+dRCn>%pHMbxh%#An15MXf*a&-)GRS0o(@^MvA!jPYQm``o; zCq8>4pj}(RCJRGMZd77n7nhf3Y=ZDzGK-TFi&9ck;0%_@Yxrz8f8t|jWMX04EGY1v me`15!Bn1{WM*dF>{GWg#kNJhgIDj_tFfz+?!1XN>5C8zG)ky*X delta 111 zcmZo@U~EX3AT7wuz`y{)Fu*!d#~3K67ca%j-^IYh*TukZ!Pm8!Q9z21t5KVoU0hzC zu}O9ETt4y1tN2YPTk~;pD){+_DENi?_)K=;v!86pe}bKv4Q$*Z1A$2jESp&j{_q0; DB-RzB diff --git a/Models/BackupDB/Filtered.db b/Models/BackupDB/Filtered.db index 93bce66fc88ab3425ba107d6a5f3d5009b42d7ad..a1172a10b5f4a58021ac01492f6e3c0dcb57d763 100644 GIT binary patch literal 110592 zcmeI&T~8ZV8o+T6NoK%=mn6WDRMief67A#F!|k7nU&-7}eOs z9#^4ND-Ac5_Hu7~n=jGNu-~9}z1c5OsoHq~&m1$yadNR2&L0VK&UwzvoacAueX{p_ z+jcDDRjYkkbBwjrY)Vm5|1gYHDmCf-e&YRJ{rt%*+`OuI|0=PvtT>tabM?V3zw^xR zexFZfexEK)FHY@Ejor>o{yMocxtRVby*u&qM0xy|@gKF{w71%d`iFX`ey)BdTPf`b zAbn|9;@GZ;3%KmQ}t3-I?8(8I&;isy>(n_Y{`pSypeiP&+1ck~SF)JY!Ce>>iIxl}fvYfVNRQ>C9Sx#Oy=9~IS zt$A#1HjZ0u+c`O{=8{@x-N)0KzP_$pXhDWaeUjQ{#C(&wglWejLnq!Fvn>1dDOE2l z%d*egXD3#Buj`*Ua(+4Y)Z zmDW#1A1Q0Z1Pryrt;I0Ih8b3 zuWtu1Rf#Zr5PK=+GpWdB{}O zZw)$*zjBAnhl{ z^Z%$$FL@z=00IagfB*srAb`G551 zm<$m>009ILKmY**5I_I{1Q6i=|FaJufB*srAbu;KmY**5I_I{ z1Q0*~0R#}>`#<{t0tg_000IagfB*srAb;niOfB*srAb7u3Vl%<0VErvE!#nqHjRn;N^FoBVZhXL2$9Q+jve=ZW(8 zFXKOIziDr^74;AGQ2ku}3^@c4KmdV{OJIF#NlDqwBkN7)b;EY7YOU+Ef_gRIYBsI9 zW4D^sTvT3k?=NWj(vsr-;M5KqRuqX!dcJIKR!pO^`E=VfGEpF7tfnVo^}QCln32kog7OOe`@51+Unm#|TxCcQFev|(@FOJ)93PZ zRyxXh-8ys3X1#S}H;=vX3iUbn!Cg(yW)-&;4qb?feed<9BXqq?ENB}#wX0Fd{J*%P z>N_j4%yzqn4ZHr#Io#_by-ecHs+i9Nv(NoZ8nZuZQD6Ht>%(iXWhrsn!diSTxdatNqv&qX2g7xx`b)R zB10$M8?!9?^(j>^EX%Ub+h-?Md#~%CIC8Pdv^#xU)0dYOw;tpg3&o04VrH>Om{8a- zbS}X#WO=Mjs=EACS7~%!SnW=5s*B3E-E=xU)m4KmqDWMl6my6IVG3S-=<#2UK$gM& zgsR(l=}cH{$9`qkYmQZFAJ>}pcQyaaU96q@Gb0bC-Qu{W=kv1Z+UEk25E*x$s+zvBp4PqM(PR4uJ^%mv?~9uL>8HxuQgGY!rQL9>wsmy%?|t#Yc-0)<26)=EN?O4y@WsHAmA-@E$~mP zQ*Ya6{#|;~a|EHR(x+#y+i)FFA@vA2P%8UN;3cqN$ij;t?BWwWqV^xqe* zW`dDqjk5WrSvHG#bI%By!icy0^(BP2VYy7-!a_9Tt>)@Q**jMB_`_Ee+ik@sLvd)* zZmXGVp`^rPovs}n(k)%FfvHo&67BX;&Bk literal 12288 zcmeH~(Qeu>6ow5t!_CqSC;?ROPA);S?-|By~(R_5{1z z-eAwP2iUvJ=?VmTqTTEY9ZN|Z|L4cfXXga>KDVR>eiNywr+I_zvqFKrPwhJS_Y; z(cUnC4>Hi8V6?avn_b%yHW$|Wmd&e+<|?o4?o^#ob;ogq{n74nw=KBaZ?%ro_V^@E zIIVKdaZe*(dXp!0BBgWsN~KeL7((g*j7`{AX3&{|-wVd{NH~MUS2ETz3Nc?s#@u}s zU@p^)MfX76ffDl=9pXyGyxnLR=|o%!tiPp*IX`I~UxG?y^IOlVt;y-RKyop!~z?iHBG>|SV--lS+p&yN8IL2mdJp${*YIR)Xr<+k2 zg8z59@<*$ArJC1qQ|)w}3#&We=k}nMzTw>Ji?-vMJaA#VqP|=26w3!kM}@(CzDRBZ zgZVmv+ZlvDY}}UCvd4y43(b@t8)x0@0)D4C=zHaV<{$(MrHl1^x6BCG;#-Q?y36DP z%$t!hcP0l=x_J0#ZdZ~y=R diff --git a/Models/BackupDB/mydb.db b/Models/BackupDB/mydb.db index ef45e70ffa2d8792f084268584b3c401d3456d4a..1ed87f7fef7f2ef75db2e8bf39705b4a3cdef456 100644 GIT binary patch delta 515 zcmZo@U~D+RI6+#FkAZ=K4TxEQm>GywCh8cA^D*efOY!oHGcfU1Fz_AatpGtDfsKu` zxC8mcnYN1S>od0GmLw+Sq!t(EB*mAc<`%?*x$%Y=0&LDfu8twD3L%b8KCTK%81j?X z@mf#5z$?yZJo!AY1fvO%6q{_u*Dk>?&KM2032bze7aO~{q9S8U{Nx&5Nk+EG(|Cii zX#3^J#s=~a14APZ3%j_iEMsFf*kx{+IVGt@sVOj;3DsFJVI>8P$(ekz^$MYWo`IpR zI$WGeKKVIHoFT3eAqsx}Kpg7h0~8Gi^7M5Kid67+jnq(rYu4n_)MsTAca~*j$jnPg ztpIv9vm`Y>v9u%~%#Vkuj5ma_Sb|3?P? zkDCPr9`h@UaTFKwGBPW2gyy9|Jp|Ulz`z8gL1r-l6<*<& - diff --git a/Models/Sql/DropAndCreateFiltered.sql b/Models/Sql/DropAndCreateFiltered.sql index ed0d761..698098b 100644 --- a/Models/Sql/DropAndCreateFiltered.sql +++ b/Models/Sql/DropAndCreateFiltered.sql @@ -1,10 +1,68 @@ VACUUM; +CREATE TABLE IF NOT EXISTS "ServerType" ( + "ServerId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Type" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "HttpVersion" ( + "HttpId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Version" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "CertificateIssuerCountry" ( + "CertificateIssuerId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Country" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "CertificateOrganizationName" ( + "CertificateOrganizationId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Name" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "TlsVersion" ( + "TlsId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Version" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "CipherSuite" ( + "CipherId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Suite" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "KeyExchangeAlgorithm" ( + "KeyExchangeId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Algorithm" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "PublicKeyType" ( + "PublicKeyId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Type" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "AcceptEncoding" ( + "AcceptId" INTEGER PRIMARY KEY AUTOINCREMENT, + "Encoding" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "ALPN" ( + "ALPNId" INTEGER PRIMARY KEY AUTOINCREMENT, + "ALPNValue" TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS "Connection" ( + "ConnectionId" INTEGER PRIMARY KEY AUTOINCREMENT, + "ConnectionValue" TEXT NOT NULL UNIQUE +); + DROP TABLE Filtered; CREATE TABLE "Filtered" ( "Id" INTEGER NOT NULL, - "Ip" TEXT NOT NULL, + "Ip1" INTEGER NOT NULL, + "Ip2" INTEGER NOT NULL, + "Ip3" INTEGER NOT NULL, + "Ip4" INTEGER NOT NULL, "Port1" INTEGER NOT NULL, "Port2" INTEGER NOT NULL, "Title1" TEXT NOT NULL, @@ -13,25 +71,42 @@ CREATE TABLE "Filtered" ( "Description2" TEXT NOT NULL, "Url1" TEXT NOT NULL, "Url2" TEXT NOT NULL, - "ServerType1" TEXT NOT NULL, - "ServerType2" TEXT NOT NULL, + "ServerType1" INTEGER NOT NULL, + "ServerType2" INTEGER NOT NULL, "RobotsTXT1" TEXT NOT NULL, "RobotsTXT2" TEXT NOT NULL, - "HttpVersion1" TEXT NOT NULL, - "HttpVersion2" TEXT NOT NULL, - "CertificateIssuerCountry" TEXT NOT NULL, - "CertificateOrganizationName" TEXT NOT NULL, + "HttpVersion1" INTEGER NOT NULL, + "HttpVersion2" INTEGER NOT NULL, + "CertificateIssuerCountry" INTEGER NOT NULL, + "CertificateOrganizationName" INTEGER NOT NULL, "IpV6" TEXT NOT NULL, - "TlsVersion" TEXT NOT NULL, - "CipherSuite" TEXT NOT NULL, - "KeyExchangeAlgorithm" TEXT NOT NULL, - "PublicKeyType1" TEXT NOT NULL, - "PublicKeyType2" TEXT NOT NULL, - "PublicKeyType3" TEXT NOT NULL, - "AcceptEncoding1" TEXT NOT NULL, - "AcceptEncoding2" TEXT NOT NULL, - "ALPN" TEXT NOT NULL, - "Connection1" TEXT NOT NULL, - "Connection2" TEXT NOT NULL, - PRIMARY KEY("Id" AUTOINCREMENT) + "TlsVersion" INTEGER NOT NULL, + "CipherSuite" INTEGER NOT NULL, + "KeyExchangeAlgorithm" INTEGER NOT NULL, + "PublicKeyType1" INTEGER NOT NULL, + "PublicKeyType2" INTEGER NOT NULL, + "PublicKeyType3" INTEGER NOT NULL, + "AcceptEncoding1" INTEGER NOT NULL, + "AcceptEncoding2" INTEGER NOT NULL, + "ALPN" INTEGER NOT NULL, + "Connection1" INTEGER NOT NULL, + "Connection2" INTEGER NOT NULL, + PRIMARY KEY("Id" AUTOINCREMENT), + FOREIGN KEY("ALPN") REFERENCES "ALPN"("ALPNId"), + FOREIGN KEY("AcceptEncoding1") REFERENCES "AcceptEncoding"("AcceptId"), + FOREIGN KEY("AcceptEncoding2") REFERENCES "AcceptEncoding"("AcceptId"), + FOREIGN KEY("CertificateIssuerCountry") REFERENCES "CertificateIssuerCountry"("CertificateIssuerId"), + FOREIGN KEY("CertificateOrganizationName") REFERENCES "CertificateOrganizationName"("CertificateOrganizationId"), + FOREIGN KEY("CipherSuite") REFERENCES "CipherSuite"("CipherId"), + FOREIGN KEY("Connection1") REFERENCES "Connection"("ConnectionId"), + FOREIGN KEY("Connection2") REFERENCES "Connection"("ConnectionId"), + FOREIGN KEY("HttpVersion1") REFERENCES "HttpVersion"("HttpId"), + FOREIGN KEY("HttpVersion2") REFERENCES "HttpVersion"("HttpId"), + FOREIGN KEY("KeyExchangeAlgorithm") REFERENCES "KeyExchangeAlgorithm"("KeyExchangeId"), + FOREIGN KEY("PublicKeyType1") REFERENCES "PublicKeyType"("PublicKeyId"), + FOREIGN KEY("PublicKeyType2") REFERENCES "PublicKeyType"("PublicKeyId"), + FOREIGN KEY("PublicKeyType3") REFERENCES "PublicKeyType"("PublicKeyId"), + FOREIGN KEY("ServerType1") REFERENCES "ServerType"("ServerId"), + FOREIGN KEY("ServerType2") REFERENCES "ServerType"("ServerId"), + FOREIGN KEY("TlsVersion") REFERENCES "TlsVersion"("TlsId") ) \ No newline at end of file diff --git a/RSE.sln.DotSettings.user b/RSE.sln.DotSettings.user index d78a6a8..e315b01 100644 --- a/RSE.sln.DotSettings.user +++ b/RSE.sln.DotSettings.user @@ -2,4 +2,5 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded \ No newline at end of file