torbot 1.2__py3.6.egg

EGG-INFO/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 1.0
+Name: torBot
+Version: 1.2
+Summary: OSINT tool for Deep and Dark Web
+Home-page: UNKNOWN
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: GNU GPLV3.0
+Description: UNKNOWN
+Keywords: tor dark web osint
+Platform: UNKNOWN

EGG-INFO/SOURCES.txt

@@ -0,0 +1,23 @@
+setup.py
+torBot
+torBot.py
+modules/__init__.py
+modules/bcolors.py
+modules/getemails.py
+modules/getweblinks.py
+modules/go_get_urls.so
+modules/go_linker.py
+modules/info.py
+modules/pagereader.py
+modules/savefile.py
+modules/updater.py
+tests/__init__.py
+tests/test_getemails.py
+tests/test_getweblinks.py
+tests/test_pagereader.py
+tests/test_savetofile.py
+torBot.egg-info/PKG-INFO
+torBot.egg-info/SOURCES.txt
+torBot.egg-info/dependency_links.txt
+torBot.egg-info/requires.txt
+torBot.egg-info/top_level.txt

EGG-INFO/dependency_links.txt

@@ -0,0 +1 @@
+

EGG-INFO/native_libs.txt

@@ -0,0 +1 @@
+modules/go_get_urls.so

EGG-INFO/not-zip-safe

@@ -0,0 +1 @@
+

EGG-INFO/requires.txt

@@ -0,0 +1,8 @@
+PySocks>=1.6.7
+beautifulsoup4>=4.6.0
+pyinstaller>=3.3.1
+pytest>=3.4.2
+requests>=2.18.4
+requests_mock>=1.4.0
+termcolor>=1.1.0
+tldextract>=2.2.0

EGG-INFO/scripts/torBot.py

@@ -0,0 +1,188 @@
+import argparse
+import socket
+import socks
+from modules import (bcolors, getemails, pagereader, getweblinks, updater,
+                     info, go_linker, savefile)
+
+# TorBot VERSION
+__VERSION = "1.2"
+LOCALHOST = "127.0.0.1"
+PORT = 9050
+
+
+def connect(address, port):
+    """ Establishes connection to port
+
+    Assumes port is bound to localhost, if host that port is bound to changes
+    then change the port
+
+    Args:
+        address: address for port to bound to
+        port: Establishes connect to this port
+    """
+    socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, address, port)
+    socket.socket = socks.socksocket  # Monkey Patch our socket to tor socket
+
+    def getaddrinfo(*args):
+        """
+        Overloads socket function for std socket library
+        Check socket.getaddrinfo() documentation to understand parameters.
+        Simple description below:
+        argument - explanation (actual value)
+        socket.AF_INET - the type of address the socket can speak to (IPV4)
+        sock.SOCK_STREAM - creates a stream connecton rather than packets
+        6 - protocol being used is TCP
+        Last two arguments should be a tuple containing the address and port
+        """
+        return [(socket.AF_INET, socket.SOCK_STREAM, 6,
+                '', (args[0], args[1]))]
+    socket.getaddrinfo = getaddrinfo
+
+
+def header():
+    """
+    Prints out header ASCII art
+    """
+
+    b_color = bcolors.Bcolors()
+    D3DSEC = b_color.FAIL + " D3DSEC " + b_color.WHITE
+    INS1DE = b_color.FAIL + " INS1DE " + b_color.WHITE
+
+    header = r"""
+                {WHITE}
+                ######################################################
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWWMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWWMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWNXNWWWWWMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWWWX0KXXKKXWMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMMWNNKOkOOkOXWMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMMNX0kdodoxKWMMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMW0doccloONMWWMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMMMMKl;;:cxKWMMMMMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMWKOXNx;,,cONMMMMMMMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMXdxKk:',lONMMMM{D3DSEC}MMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMOo0NOdxc,kMMMM{INS1DE}MMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMMMOcONOxkx;dWMMMMMMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMMNkcdXXOkxkd:oXMMMMMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMNOoclONNX00OkOxc:lkXWMMMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMN0olld0NWNNX0O00kxkxl:;ckXMWWMMMMMMMMM
+                MMMMMMMMMMMWMMNxccd0NNNNNXNXOkOK0dodxdo:,;o0WMMMMMMMMM
+                MMMMMMMMMMMMNk:ckXNNWNXXXXNXOOOOK0oloooooc,'oKMMMMMMMM
+                MMMMMMMMMMMXc;xXNNNXKXXXNNWKOO0Ok0x:clllllc:.,OWMMMMMM
+                MMMMMMMMMMX:;0WNNX00XNNNNNNKOO0KkkOc,ccccccc:.'OWMMMMM
+                MMMMMMMMMNl,ONNN0OXNNNNNXXNKOkOK0xkl':c::::::;.;KMMMMM
+                MMMMMMMMM0,lNWXO0NNNNXKKXXNXO0Ok0Oxl',:;;;;;;;..dMMMMM
+                MMMMMMMMMk,xWNOONNNX00XNNNWKOO0OkOxc'';;,,,,,,'.cMMMMM
+                MMMMMMMMMx,xWKkKWNXOKNWNNNX0xxOKxxx:..,,,,,''''.cMMMMM
+                MMMMMMMMM0,oWXkOWXOKNNNNN00Xkdx0kdd;..,'''''''..oMMMMM
+                MMMMMMMMMNl;0W0kKKkKWNNN0ONNOxdOOll,..'''......,0MMMMM
+                MMMMMMMMMMK::KN0kKOkNNWXk0WX0kdxkc:............xWMMMMM
+                MMMMMMMMMMMKl:kX0k0kONWNOONX0koxd:,..........'kWMMMMMM
+                MMMMMMMMMMMMNxccxOkkxkKWKx0NOoooc'..........lKWMMMMMMM
+                MMMMMMMMMMMMMWNklccclldk0OxOdcc;. .......;oKWWMMMMMMMM
+                MMMMMMMMMMMMMMMMWXOdl:::;cc;'... ..',:lx0NMMMMMMMMMMMM
+                MMMMMMMMMMMMMMMMMMMMMNKOkxddolloodk0XWMMMMMMMMMMMMMMMM
+                {FAIL} + {BOLD}
+                           __  ____  ____  __        ______
+                          / /_/ __ \/ __ \/ /_  ____/_  __/
+                         / __/ / / / /_/ / __ \/ __ \/ /
+                        / /_/ /_/ / _, _/ /_/ / /_/ / /
+                        \__/\____/_/ |_/_____/\____/_/  V{VERSION}
+                {FAIL} + {On_Black}
+                #######################################################
+                #  TorBot - A python Tor Crawler                      #
+                #  GitHub : https://github.com/DedsecInside/TorBot    #
+                #  Help : use -h for help text                        #
+                #######################################################
+                      {FAIL} + "LICENSE: GNU Public License" + {END}""".format(
+                D3DSEC=D3DSEC, INS1DE=INS1DE, FAIL=b_color.FAIL,
+                BOLD=b_color.BOLD, VERSION=__VERSION, END=b_color.ENDC,
+                On_Black=b_color.On_Black, WHITE=b_color.WHITE
+                )
+    print(header)
+
+
+def main(conn=False, link=False):
+    if conn:
+        connect(LOCALHOST, PORT)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", "--version",
+                        action="store_true",
+                        help="Show current version of TorBot.")
+    parser.add_argument("--update",
+                        action="store_true",
+                        help="Update TorBot to the latest stable version")
+    parser.add_argument("-q", "--quiet",
+                        action="store_true")
+    parser.add_argument("-u", "--url",
+                        help="Specifiy a website link to crawl")
+    parser.add_argument("-s", "--save",
+                        action="store_true",
+                        help="Save results in a file")
+    parser.add_argument("-m", "--mail",
+                        action="store_true",
+                        help="Get e-mail addresses from the crawled sites")
+    parser.add_argument("-e", "--extension",
+                        action='append',
+                        dest='extension',
+                        default=[],
+                        help=' '.join(("Specifiy additional website extensions",
+                                       "to the list(.com , .org etc)")))
+    parser.add_argument("-l", "--live",
+                        action="store_true",
+                        help="Check if websites are live or not (slow)")
+    parser.add_argument("-i", "--info",
+                        action="store_true",
+                        help=' '.join(("Info displays basic info of the",
+                                       "scanned site, (very slow)")))
+    args = parser.parse_args()
+
+    link = args.url
+
+    # If flag is -v, --update, -q/--quiet then user only runs that operation
+    # because these are single flags only
+    if args.version:
+        print("TorBot Version:" + __VERSION)
+        exit()
+    if args.update:
+        updater.updateTor()
+        exit()
+    if not args.quiet:
+        header()
+    # If url flag is set then check for accompanying flag set. Only one
+    # additional flag can be set with -u/--url flag
+    if args.url:
+        print("Tor IP Address :", pagereader.get_ip())
+        html_content, response = pagereader.read_first_page(link)
+        # -m/--mail
+        if args.mail:
+            emails = getemails.getMails(html_content)
+            print(emails)
+            if args.save:
+                savefile.saveJson('Emails', emails)
+        # -i/--info
+        elif args.info:
+            info.executeAll(link, html_content, response)
+            if args.save:
+                print('Nothing to save.\n')
+        else:
+            # Golang library is now being used
+            links = go_linker.GetLinks(link, LOCALHOST, PORT, 15, 1)
+            return links
+            # links = getweblinks.get_links(soup=html_content, ext=args.extension, live=args.live)
+            if args.save:
+                savefile.saveJson("Links", links)
+    else:
+        print("usage: torBot.py [-h] [-v] [--update] [-q] [-u URL] [-s] [-m] [-e EXTENSION] [-l] [-i]")
+
+    print("\n\n")
+
+
+if __name__ == '__main__':
+
+    try:
+        main(conn=True)
+
+    except KeyboardInterrupt:
+        print("Interrupt received! Exiting cleanly...")

EGG-INFO/top_level.txt

@@ -0,0 +1,2 @@
+modules
+tests

modules/bcolors.py

@@ -0,0 +1,14 @@
+class Bcolors:
+
+    def __init__(self):
+        self.HEADER = '\033[95m'
+        self.OKBLUE = '\033[94m'
+        self.OKGREEN = '\033[92m'
+        self.WARNING = '\033[93m'
+        self.FAIL = '\033[91m'
+        self.ENDC = '\033[0m'
+        self.BOLD = '\033[1m'
+        self.UNDERLINE = '\033[4m'
+        self.WHITE = '\033[97m'
+        self.On_Black = '\033[40m'
+        self.On_Red = '\033[41m'

modules/getemails.py

@@ -0,0 +1,41 @@
+from modules.bcolors import Bcolors
+from bs4 import BeautifulSoup
+
+
+def getMails(soup):
+
+    """
+        Searches for <a href> tags for links then checks if link contains the
+        substring 'mailto' indicating that it's an email. If it is determined
+        to be an email then the link is split and the username is appeneded to
+        the list
+
+        Args:
+            soup: BeautifulSoup isntance that will be used for parsing
+
+        Returns:
+            emails: list of email IDs
+    """
+    b_colors = Bcolors()
+
+    if isinstance(type(soup), type(BeautifulSoup)):
+
+        emails = []
+        links = soup.find_all('a')
+        for ref in links:
+            url = ref.get('href')
+            if url and 'mailto' in url:
+                """Split email address on"""
+                email_addr = url.split(':')
+                if (len(email_addr) > 1):
+                    emails.append(email_addr[1])
+
+        """Pretty print output as below"""
+        print ('')
+        print (b_colors.OKGREEN+'Mails Found - '+b_colors.ENDC+str(len(emails)))
+        print ('-------------------------------')
+
+        return emails
+
+    else:
+        raise('Method parameter is not of instance BeautifulSoup')

modules/getweblinks.py

@@ -0,0 +1,136 @@
+import re
+import requests
+import tldextract
+
+from modules import pagereader
+from bs4 import BeautifulSoup
+from modules.bcolors import Bcolors
+from requests.exceptions import ConnectionError, HTTPError
+
+
+def valid_url(url, extensions=False):
+    """Checks for any valid url using regular expression matching
+
+        Matches all possible url patterns with the url that is passed and
+        returns True if it is a url and returns False if it is not.
+
+        Args:
+            url: string representing url to be checked
+
+        Returns:
+            bool: True if valid url format and False if not
+    """
+    pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.([a-z, A-Z]+)(.*)"
+    regex = re.compile(pattern)
+    if not extensions:
+        if regex.match(url):
+            return True
+        return False
+
+    parts = tldextract.extract(url)
+    valid_sites = list()
+    for ext in extensions:
+        if regex.match(url) and '.'+parts.suffix in ext:
+            valid_sites.append(url)
+    return valid_sites
+
+
+def valid_onion_url(url):
+    """Checks for valid onion url using regular expression matching
+
+        Only matches onion urls
+
+        Args:
+            url: string representing url to be checked
+
+        Returns:
+            bool: True if valid onion url format, False if not
+    """
+    pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.onion/(.*)"
+    regex = re.compile(pattern)
+    if regex.match(url):
+        return True
+    return False
+
+
+def is_link_alive(link):
+    """Generator that yields links as they come
+
+        Uses head request because it uses less bandwith than get and timeout is
+        set to 10 seconds and then link is automatically declared as dead.
+
+        Args:
+            link: link to be tested
+            colors: object containing colors for link
+
+        Yields:
+            string: link with either no color or red which indicates failure
+    """
+
+    try:
+        resp = requests.head(link, timeout=10)
+        resp.raise_for_status()
+        return True
+    except (ConnectionError, HTTPError):
+        return False
+
+
+def add_green(link):
+    colors = Bcolors()
+    return '\t' + colors.OKGREEN + link + colors.ENDC
+
+
+def add_red(link):
+    colors = Bcolors()
+    return '\t' + colors.On_Red + link + colors.ENDC
+
+
+def get_links(soup, ext=False, live=False):
+    """
+        Searches through all <a ref> (hyperlinks) tags and stores them in a
+        list then validates if the url is formatted correctly.
+
+        Args:
+            soup: BeautifulSoup instance currently being used.
+
+        Returns:
+            websites: List of websites that were found
+    """
+    b_colors = Bcolors()
+    if isinstance(soup, BeautifulSoup):
+        websites = []
+
+        links = soup.find_all('a')
+        for ref in links:
+            url = ref.get('href')
+            if ext:
+                if url and valid_url(url, ext):
+                    websites.append(url)
+            else:
+                if url and valid_onion_url(url):
+                    websites.append(url)
+
+        """Pretty print output as below"""
+        print(''.join((b_colors.OKGREEN,
+              'Websites Found - ', b_colors.ENDC, str(len(websites)))))
+        print('------------------------------------')
+        
+        if live:
+            for link in websites:
+                if is_link_alive(link):
+                    coloredlink = add_green(link)
+                    page = pagereader.read_page(link)
+                    if page is not None and page.title is not None:
+                        print_row(coloredlink, page.title.string)
+                else:
+                    coloredlink = add_red(link)
+                    print_row(coloredlink, "Not found")
+
+        return websites
+
+    else:
+        raise(Exception('Method parameter is not of instance BeautifulSoup'))
+
+
+def print_row(url, description):
+    print("%-80s %-30s" % (url, description))

modules/go_linker.py

@@ -0,0 +1,26 @@
+import os.path
+
+from ctypes import cdll, c_char_p, c_longlong, c_int, Structure
+
+dll_path = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + "go_get_urls.so"
+go_linker = cdll.LoadLibrary(dll_path)
+
+
+class GoString(Structure):
+    _fields_ = [("p", c_char_p), ("n", c_longlong)]
+
+
+go_linker.GetLinks.argtypes = [GoString, GoString, GoString, c_int, c_int]
+
+
+def GetLinks(url, addr, port, timeout, extensions):
+
+    url = url.encode('utf-8')
+    addr = addr.encode('utf-8')
+    port = str(port).encode('utf-8')
+
+    go_linker.GetLinks(GoString(url, len(url)),
+                       GoString(addr, len(addr)),
+                       GoString(port, len(port)),
+                       timeout,
+                       extensions)

modules/info.py

@@ -0,0 +1,101 @@
+import requests
+
+from urllib.parse import urlsplit
+from termcolor import cprint
+
+
+def executeAll(target, soup, response):
+
+    try:
+        get_robots_txt(target)
+    except Exception:
+        cprint("No robots.txt file Found!""blue")
+    try:
+        get_dot_git(target)
+    except Exception:
+        cprint("Error !""red")
+    try:
+        get_dot_svn(target)
+    except Exception:
+        cprint("Error""red")
+    try:
+        get_dot_htaccess(target)
+    except Exception:
+        cprint("Error""red")
+    try:
+        get_webpage_description(soup)
+    except Exception:
+        cprint("Error""red")
+    try:
+        get_headers(response)
+    except Exception:
+        cprint("Error""red")
+
+
+def get_headers(response):
+    print('''
+          RESPONSE HEADERS
+          __________________
+          ''')
+    for key, val in response.headers.items():
+        print('*', key, ':', val)
+
+
+def get_robots_txt(target):
+    cprint("[*]Checking for Robots.txt"'yellow')
+    url = target
+    target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
+    requests.get(target+"/robots.txt")
+    cprint(r'blue')
+
+
+def get_dot_git(target):
+    cprint("[*]Checking for .git folder"'yellow')
+    url = target
+    target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
+    req = requests.get(target+"/.git/")
+    r = req.status_code
+    if r == 200:
+        cprint("Alert!"'red')
+        cprint(".git folder exposed publicly"'red')
+    else:
+        print("NO .git folder found"'blue')
+
+
+def get_dot_svn(target):
+    cprint("[*]Checking for .svn folder"'yellow')
+    url = target
+    target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
+    req = requests.get(target+"/.svn/entries")
+    r = req.status_code
+    if r == 200:
+        cprint("Alert!"'red')
+        cprint(".SVN folder exposed publicly"'red')
+    else:
+        cprint("NO .SVN folder found"'blue')
+
+
+def get_dot_htaccess(target):
+    cprint("[*]Checking for .htaccess"'yellow')
+    url = target
+    target = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
+    req = requests.get(target+"/.htaccess")
+    statcode = req.status_code
+    if statcode == 403:
+        cprint("403 Forbidden"'blue')
+    elif statcode == 200:
+        cprint("Alert!!"'blue')
+        cprint(".htaccess file found!"'blue')
+    else:
+        cprint("Status code"'blue')
+        cprint(statcode)
+
+
+def get_webpage_description(soup):
+    cprint("[*]Checking for description meta tag"'yellow')
+    metatags = soup.find_all('meta')
+    for meta in metatags:
+        if meta.has_attr('name'):
+            attributes = meta.attrs
+            if attributes['name'] == 'description':
+                cprint("Page description: " + attributes['content'])

modules/pagereader.py

@@ -0,0 +1,109 @@
+import requests
+
+from bs4 import BeautifulSoup
+from modules.bcolors import Bcolors
+from requests.exceptions import ConnectionError, HTTPError, MissingSchema
+from sys import exit
+
+
+def connection_msg(site):
+    yield "Attempting to connect to {site}".format(site=site)
+
+
+def read_first_page(site):
+    headers = {'User-Agent':
+               'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot'}
+    attempts_left = 3
+    err = " "
+    while attempts_left:
+        try:
+            if attempts_left == 3:
+                print(next(connection_msg(site)))
+                response = requests.get(site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page, response
+            if attempts_left == 2:
+                print(next(connection_msg('https://'+site)))
+                response = requests.get('https://'+site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page, response
+            if attempts_left == 1:
+                print(next(connection_msg('http://'+site)))
+                response = requests.get('http://'+site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page, response
+            if not attempts_left:
+                msg = ''.join(("There has been an {err} while attempting to ",
+                              "connect to {site}.")).format(err=err, site=site)
+                exit(msg)
+
+        except (HTTPError, MissingSchema, ConnectionError) as e:
+            attempts_left -= 1
+            err = e
+
+    if isinstance(err, HTTPError):
+        print ("There has been an HTTP error after three attempts.")
+        exit (1)
+    if isinstance(err, ConnectionError):
+        print("Got ConnectionError after three attempts... ",
+              "Please check if the TOR service is running or not.")
+        exit (1)
+
+
+def read_page(site):
+    headers = {'User-Agent':
+               'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot'}
+    attempts_left = 3
+    err = " "
+    while attempts_left:
+        try:
+            if attempts_left == 3:
+                #print(next(connection_msg(site)))
+                response = requests.get(site, headers=headers)
+                #print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if attempts_left == 2:
+                #print(next(connection_msg('https://'+site)))
+                response = requests.get('https://'+site, headers=headers)
+                #print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if attempts_left == 1:
+                #print(next(connection_msg('http://'+site)))
+                response = requests.get('http://'+site, headers=headers)
+                #print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if not attempts_left:
+                msg = ''.join(("There has been an {err} while attempting to ",
+                              "connect to {site}.")).format(err=err, site=site)
+                exit(msg)
+
+        except (HTTPError, MissingSchema, ConnectionError) as e:
+            attempts_left -= 1
+            err = e
+
+    if isinstance(err, HTTPError):
+        print("There has been an HTTP error after three attempts.")
+        exit (1)
+    if isinstance(err, ConnectionError):
+        print("There has been a connection error after three attempts.")
+        exit (1)
+
+def get_ip():
+    """Returns users tor ip address
+
+    https://check.torproject.org/ tells you if you are using tor and it
+    displays your IP address which we scape and return
+    """
+
+    b_colors = Bcolors()
+    page = read_first_page('https://check.torproject.org/')[0]
+    pg = page.find('strong')
+    ip_addr = pg.renderContents()
+
+    return b_colors.WARNING+b_colors.BOLD+ip_addr.decode("utf-8")+b_colors.ENDC

modules/savefile.py

@@ -0,0 +1,24 @@
+import json
+import time
+
+
+def saveJson(datatype, data):
+    """
+        Creates json file and stores json
+
+        Args:
+            datatype: the type of the object being passed
+            data = data that is being stored with object
+    """
+
+    timestr = time.strftime("%Y%m%d-%H%M%S")
+    file_name = "TorBot-Export-"+datatype+timestr+".json"
+    # Json File Creation
+    with open(file_name, "w+") as f:
+        # Store data in Json format
+        output = {datatype: data}
+        # Dump output to file
+        json.dump(output, f, indent=2)
+
+    print("\nData will be saved with a File Name :", file_name)
+    return file_name

modules/updater.py

@@ -0,0 +1,48 @@
+import subprocess
+
+
+def updateTor():
+
+    """
+        Currently updates Tor by calling terminal commands using subprocess
+        Not a great method and will be replaced in the future.
+
+    """
+
+    print("Checking for latest stable release")
+    isGit = subprocess.Popen(
+            ["git", "branch"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+    output = isGit.stdout.read()
+    branch = output[2:8].decode("utf-8")
+    print(branch)
+    if branch == 'master':
+        update = subprocess.Popen(
+                 ["git", "pull", "origin", "master"],
+                 stdout=subprocess.PIPE,
+                 stderr=subprocess.STDOUT)
+        update_out = update.stdout.read()
+        if update_out[90:109].decode("utf-8") == 'Already up-to-date.':
+            print("TorBot is already up-to-date.")
+        else:
+            print("TorBot has succesfully updated to latest stable version.")
+    else:
+        subprocess.Popen(
+            ["git", "init"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+        subprocess.Popen(
+            ["git", "remote", "add", "origin",
+                "https://github.com/DedSecInside/TorBoT.git"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+        update = subprocess.Popen(
+            ["git", "pull", "origin", "dev"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT)
+        update_out = update.stdout.read()
+        if update_out[90:109].decode("utf-8") == 'Already up-to-date.':
+            print("TorBot is already up-to-date.")
+        else:
+            print("TorBot has succesfully updated to latest stable version.")

tests/test_getemails.py

@@ -0,0 +1,20 @@
+import sys
+import os
+
+PACKAGE_PARENT = '..'
+SCRIPT_DIR = os.path.dirname(os.path.realpath(
+             os.path.join(os.getcwd(), os.path.expanduser(__file__))))
+
+sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))
+
+from modules import pagereader, getemails
+
+
+def test_get_emails_successful():
+    soup = pagereader.read_first_page('https://www.helloaddress.com/')[0]
+    test_emails = ["hello@helloaddress.com"]
+    emails = getemails.getMails(soup)
+    assert emails == test_emails
+
+if __name__ == '__main__':
+    test_get_emails_successful()

tests/test_getweblinks.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import sys
+import os
+PACKAGE_PARENT = '..'
+SCRIPT_DIR = os.path.dirname(os.path.realpath(
+             os.path.join(os.getcwd(), os.path.expanduser(__file__))))
+
+sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))
+from modules import getweblinks, pagereader
+
+
+def test_get_links_successful():
+    soup = pagereader.read_first_page('http://www.whatsmyip.net/')[0]
+    data = ['http://aff.ironsocket.com/SH7L',
+            'http://aff.ironsocket.com/SH7L',
+            'http://wsrs.net/',
+            'http://cmsgear.com/']
+
+    result = getweblinks.get_links(soup, ext=['.com', '.net'])
+    assert result == data
+
+
+if __name__ == '__main__':
+	test_get_links_successful()

tests/test_pagereader.py

@@ -0,0 +1,69 @@
+import pytest
+import requests
+import requests_mock
+
+from bs4 import BeautifulSoup
+from requests.exceptions import HTTPError, MissingSchema, ConnectionError
+
+
+@pytest.fixture
+def test_read_first_page(site):
+
+    with requests_mock.Mocker() as m:
+        m.get('https://www.test.com', text='This is a dot com site.')
+        m.get('https://www.test.org', text='This is a dot org site.')
+        m.get('https://www.test.net', text='This is a dot net site.')
+        m.get('https://www.test.onion', text='This is a dot onion site.')
+
+        m.register_uri('GET', 'https://www.test.cannotbefound', exc=HTTPError)
+        m.register_uri('GET', 'http://www.test.cannotbefound', exc=HTTPError)
+
+        headers = {'User-Agent':
+                   'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot'}
+        attempts_left = 3
+        err = " "
+
+        # Removed unnecessary code such as printing
+        while attempts_left:
+            try:
+                if attempts_left == 3:
+                    response = requests.get(site, headers=headers)
+                    page = BeautifulSoup(response.text, 'html.parser')
+                    return str(page)
+                if attempts_left == 2:
+                    response = requests.get('https://'+site, headers=headers)
+                    page = BeautifulSoup(response.text, 'html.parser')
+                    return str(page)
+                if attempts_left == 1:
+                    response = requests.get('http://'+site, headers=headers)
+                    page = BeautifulSoup(response.text, 'html.parser')
+                    return str(page)
+                if not attempts_left:
+                    raise err
+
+            except (HTTPError, MissingSchema, ConnectionError) as e:
+                err = e
+                attempts_left -= 1
+
+        raise err
+
+
+def test_run():
+    urls = ['www.test.com', 'www.test.org', 'www.test.net', 'www.test.onion',
+            'www.test.cannotbefound']
+
+    with pytest.raises(HTTPError):
+        for url in urls:
+            page = test_read_first_page(url)
+            if url[-4:] == '.com':
+                assert page == 'This is a dot com site.'
+            elif url[-4:] == '.org':
+                assert page == 'This is a dot org site.'
+            elif url[-4:] == '.net':
+                assert page == 'This is a dot net site.'
+            elif url[-6:] == '.onion':
+                assert page == 'This is a dot onion site.'
+
+
+if __name__ == '__main__':
+    test_run()

tests/test_savetofile.py

@@ -0,0 +1,40 @@
+import sys
+import os
+import json
+PACKAGE_PARENT = '..'
+SCRIPT_DIR = os.path.dirname(os.path.realpath(
+             os.path.join(os.getcwd(), os.path.expanduser(__file__))))
+
+sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))
+
+from modules import savefile
+
+
+def test_save_links_successful():
+    mock_data = ['http://aff.ironsocket.com/SH7L',
+                 'http://aff.ironsocket.com/SH7L',
+                 'http://wsrs.net/',
+                 'http://cmsgear.com/']
+    try:
+        file_name = savefile.saveJson('Links', mock_data)
+        mock_output = {'Links': mock_data}
+
+        with open('test_file.json', 'w+') as test_file:
+            json.dump(mock_output, test_file, indent=2)
+
+        os.chdir(os.getcwd())
+        assert os.path.isfile(file_name) is True
+        mock_file = open(file_name, 'r')
+        test_file = open('test_file.json', 'r')
+
+        mock_data = mock_file.read()
+        test_data = test_file.read()
+
+    finally:
+        os.remove(file_name)
+        os.remove('test_file.json')
+
+    assert mock_data == test_data
+
+if __name__ == '__main__':
+    test_save_links_successful()