From ed3224c5423a7e15209bc514b2837301b16762cc Mon Sep 17 00:00:00 2001
From: Pravin Agre <agrepravin@gmail.com>
Date: Sun, 9 Jul 2017 00:41:25 +0530
Subject: [PATCH] optimization for getting status of url(using threads)

---
 modules/__init__.pyc   | Bin 0 -> 150 bytes
 modules/bcolors.pyc    | Bin 0 -> 670 bytes
 modules/getemails.pyc  | Bin 0 -> 1163 bytes
 modules/getweblinks.py |  40 +++++++++++++++++++++++++---------------
 4 files changed, 25 insertions(+), 15 deletions(-)
 create mode 100644 modules/__init__.pyc
 create mode 100644 modules/bcolors.pyc
 create mode 100644 modules/getemails.pyc

diff --git a/modules/__init__.pyc b/modules/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c8bbeb83f3819f5895d5bc5e9181ea08b64c5a9f
GIT binary patch
literal 150
zcmZSn%*)lsm>8MN00oRd+5w1*S%5?e14FO|NW@PANHCxg#r8lk{fzwFRQ-aY#Inph
zeV6>?(%jU%l4AXW%94!yy!ep(BB%TkeJ~ZGpPQdjnv+_rA0MBYmst`YuLo4c0W`w~
MB4G!zsThbE05;Mg>i_@%

literal 0
HcmV?d00001

diff --git a/modules/bcolors.pyc b/modules/bcolors.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bae219494c29f4cdf10d40eba815cde8975315c6
GIT binary patch
literal 670
zcmb_ZO;5r=5S{W-pvYAZ`~gl@6k|*bk=hECrcDZjupFA&ZGw@ugk4NH>)Ai+A8>XH
z<>Jw{`}WQ5%$v8}@=r_qR>DODho_A5BMx?fK=CUBRp1NY6JS9wF2STz!989>c#X1k
zwq-@Nf@=m2_JR-t5JhN^lF%TiR02{FlB5c>0z?H`1yTd50;vP70cilO1JQssfHZ+>
zK+eFQbecl5iqKx(-zG=aE7*0xZVJ{-nDE(NUL8rXaw@z#ruQ%_Z_w)|ofa1IIzkYn
zVG;)c*JO}nbhj3vis8r_n3gZ8H#VG^C4Y%G9{QH$N<JU>uI&!F@bNsb9Z4+Le3HcQ
z9FsS2Yv!VkW4jg?4(20!YRTqa8W`&^de2*=fgjTjCcp_;A47eWC9%HQh97UKZf4Og
ziBrz>&7QBaG?->vBjdW9rh0w=rXMcua<lh^qPRD9j-V=ky>8x7WXbfC0m=Tsg*@*=
L?yfK0U@!Ou!&HL$

literal 0
HcmV?d00001

diff --git a/modules/getemails.pyc b/modules/getemails.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e1605aec4d0d307b0256f097f907c0bb58fef732
GIT binary patch
literal 1163
zcmb_b&2AGh5FYPek~VFTkf>0RSixm4&0g9|g-~hJ9|Qy`sf37BMcdeEHr=eZ>j`S5
z<U+ZYx8fam3?v=^W}Hw22ToY_cs&2#*ml3KEPWQq`#vm&OZzS@b4o<g6p#UQ?|1=S
z0SaK`z{mw9EaQQ6;70Hi?Bhb_!#IF$Fh5g_3Un(VJ@`bysvNZw`II63L8E0jYoifI
zf_AJBIqU08m!{7GK6|wGX_;?CkkSj72|x$Xg(+}iW^4sF4wPTv9AM_am1OOR>a(im
zX$NLx=fcQelsxlbN@{=$7s?$josJg)l()=`Y|RvX=EIZ_S7AzyK0IWkm$z`a!}FaC
z;OH1@Fj@jxfpzAVX{!USup1fETt7ONRkFLf&$8y}GRy*)vghR&8=}L0s~~H45k3m8
ztP?E}=oWG>668({OEtiNm~oP3sI6uvwap_Tm{lTgNM*u9J;`Ob8Jfo=+Whx0+w7)r
zsAX6rWinQ%$}lxyt})buFwG4nd0&NRW~;HMk_o1RN!HPmB3dI)x&XR2vqhKz{!k;=
zpPFC(76M)u3aca^q`B-RS!UhO6<Tde=L-3Km?BHjLQRT7<+4q(`3rgb>ZpBi(6Z(S
zt>(V<H(1&|Q@K?fzIgdM;`d`uhDt^5O^><WT|L}M#jP~fEKBnXTe6k){yHX7me!0N
zBJu=F%r0^9P>)qyl*xxQkDIzb8B-l5F0QGkdGAD*dm3ZAoy21;Cz&!aL7wMskQq0G
z1orM~#G5BXK-9#B2*d-iChDT@tci8UbA0hs_+o{0o)fX{yaCsSJ_UfDLn#s*wi&Vg
uRLPlP0rG7v(l-_#&Gxm;jc9KW`Hky8gL^j5`<&mZ;b%dB#3OM})V-gjPx>7I

literal 0
HcmV?d00001

diff --git a/modules/getweblinks.py b/modules/getweblinks.py
index f2bf953b..df6a63f3 100644
--- a/modules/getweblinks.py
+++ b/modules/getweblinks.py
@@ -4,6 +4,24 @@
 import urllib.request 
 from modules.bcolors import bcolors
 import bs4
+import time
+import threading
+import http
+
+def link_status(web):
+    link_live = False
+    try:
+        urllib.request.urlopen(web)    
+        link_live = True
+        print(web)
+    except urllib.error.HTTPError as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    except urllib.error.URLError as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    except http.client.RemoteDisconnected as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    return 
+
 
 """Get all onion links from the website"""
 def getLinks(soup,ext,live=0):
@@ -11,9 +29,10 @@ def getLinks(soup,ext,live=0):
     extensions = []
     if ext:
         for e in ext:
-            extensions.append(e)  
+            extensions.append(e) 
     if isinstance(type(soup), type(_soup_instance)):
         websites = []
+        start_time = time.time()
         for link in soup.find_all('a'):
             web_link = link.get('href')
             if web_link != None:
@@ -31,23 +50,14 @@ def getLinks(soup,ext,live=0):
         print (bcolors.OKGREEN+'Websites Found - '+bcolors.ENDC+str(len(websites)))
         print ('-------------------------------')
         if live:
+            threads = []
             for web in websites:
-                flag=1
-                try:
-                    urllib.request.urlopen(web)    
-                except urllib.error.HTTPError as e:
-                    if e.code:
-                        print(bcolors.On_Red+web+bcolors.ENDC)
-                        flag=0  
-                except urllib.error.URLError as e:
-                    print(bcolors.On_Red+web+bcolors.ENDC)
-                    flag=0  
-
-                if flag:
-                    print(web)          
+                t = threading.Thread(target=link_status, args=(web,))
+                threads.append(t)
+                t.start()
         else:
             for web in websites:
-                print(web) 
+                print(web)
         return websites           
     else:
         raise('Method parameter is not of instance bs4.BeautifulSoup')