anubis-netsec 1.4.0__py3-none-any.whl

anubis/API.py

@@ -0,0 +1,5 @@
+# Public Shodan key bundled with Anubis
+CENSYS_ID = ""
+CENSYS_SECRET = ""
+# Spyse Token - https://spyse.com/user
+SPYSE_TOKEN = ""

anubis/__init__.py

@@ -0,0 +1 @@
+__version__ = '1.4.0'

anubis/cli.py

@@ -0,0 +1,132 @@
+"""
+Usage:
+  anubis (-t TARGET | -f FILE) [-o FILENAME]  [-bdinoprsSv] [-w SCAN] [-q NUM]
+  anubis -h
+  anubis (--version | -V)
+  
+Options:
+  -h --help                       show this help message and exit
+  -t --target                     set target (comma separated, no spaces, if multiple)
+  -f --file                       set target (reads from file, one domain per line)
+  -n --with-nmap                  perform an nmap service/script scan
+  -o --output                     save to filename
+  -i --additional-info            show additional information about the host from Shodan (requires API key)
+  -p --ip                         outputs the resolved IPs for each subdomain, and a full list of unique ips
+  -d --send-to-anubis-db          send results to Anubis-DB
+  -r --recursive                  recursively search over all subdomains
+  -s --ssl                        run an ssl scan and output cipher + chain info
+  -S --silent                     only out put subdomains, one per line
+  -w --overwrite-nmap-scan SCAN   overwrite default nmap scan (default -nPn -sV -sC)
+  -v --verbose                    print debug info and full request output
+  -q --queue-workers NUM          override number of queue workers (default: 10, max: 100)
+  -V --version                       show version and exit
+
+Help:
+  For help using this tool, please open an issue on the Github repository:
+  https://github.com/jonluca/anubis
+"""
+
+import sys
+import time
+from functools import reduce
+
+from docopt import docopt
+
+import anubis.commands
+# noinspection PyPep8Naming
+from . import __version__ as VERSION
+
+
+# Overload stdout to save output and change colors on filewrite
+class StdOutHook:
+  lines = []
+  filename = ""
+
+  def __init__(self, filename, silent, output):
+    self.filename = filename
+    self.silent = silent
+    self.output = output
+
+  def write(self, text, override=False, **kwargs):
+    if not self.silent or override:
+      sys.__stdout__.write(text)
+    self.lines.append(text)
+
+  def write_out(self):
+    if self.output:
+      with open(self.filename, "w") as file:
+        for line in self.lines:
+          # remove stdout colors
+          line = line.replace('\033[91m', '')
+          line = line.replace('\033[92m', '')
+          line = line.replace('\033[93m', '')
+          line = line.replace('\033[94m', '')
+          line = line.replace('\033[95m', '')
+          line = line.replace('\033[0m', '')
+          file.write(line)
+
+  def flush(self):
+    # python3 compatability, does nothing
+    pass
+
+
+# credit to https://stackoverflow.com/questions/1557571/how-do-i-get-time-of-a-python-programs-execution
+def seconds_to_str(t):
+  return "%d:%02d:%02d.%03d" % reduce(lambda ll, b: divmod(ll[0], b) + ll[1:],
+                                      [(t * 1000,), 1000, 60, 60])
+
+
+def main():
+  try:
+    if sys.version_info < (3, 0):
+      sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
+      sys.exit(1)
+
+    start_time = time.time()
+
+    options = docopt(__doc__, version=VERSION)
+
+    if options["--output"] or options['--silent']:
+      sys.stdout = StdOutHook(options["FILENAME"], options['--silent'],
+                              options['--output'])
+
+    if options["--queue-workers"]:
+      if not options["--recursive"]:
+        print("Queue workers override only works with recursive option!")
+        sys.exit(1)
+      if int(options["--queue-workers"]) > 100:
+        print("Max queue worker override is 100!")
+        sys.exit(1)
+      if int(options["--queue-workers"]) < 1:
+        print("Queue workers can't be negative!")
+        sys.exit(1)
+
+    if not options["--target"] and not options['--file']:
+      print(
+        "Target required! Run with -h for usage instructions. Either -t target.host or -f file.txt required")
+      return
+
+    if options["--target"] and options['--file']:
+      print(
+        "Please only supply one target method - either read by file with -f or as an argument to -t, not both.")
+      return
+
+    print("""
+          d8888                   888      d8b
+         d88888                   888      Y8P
+        d88P888                   888
+       d88P 888 88888b.  888  888 88888b.  888 .d8888b
+      d88P  888 888 "88b 888  888 888 "88b 888 88K
+     d88P   888 888  888 888  888 888  888 888 "Y8888b.
+    d8888888888 888  888 Y88b 888 888 d88P 888      X88
+   d88P     888 888  888  "Y88888 88888P"  888  88888P'
+    """)
+
+    command = anubis.commands.Target(options)
+    command.run()
+    print("Subdomain search took %s" % seconds_to_str(time.time() - start_time))
+    if options["--output"]:
+      sys.stdout.write_out()
+  except KeyboardInterrupt:
+    print("\nQuitting...")
+    sys.exit(0)

anubis/commands/__init__.py

@@ -0,0 +1 @@
+from .target import *

anubis/commands/base.py

@@ -0,0 +1,14 @@
+"""The base command."""
+
+
+class Base(object):
+  """A base command."""
+
+  def __init__(self, options, *args, **kwargs):
+    self.options = options
+    self.args = args
+    self.kwargs = kwargs
+
+  def run(self):
+    raise NotImplementedError(
+      'run() method must be implemented by the overloading class')

anubis/commands/target.py

@@ -0,0 +1,174 @@
+"""The target command."""
+import os
+import re
+import socket
+import sys
+import threading
+from urllib.parse import urlsplit
+
+from anubis.scanners.anubis_db import search_anubisdb, send_to_anubisdb
+from anubis.scanners.crt import search_crtsh
+from anubis.scanners.hudson_rock import search_hudsonrock
+from anubis.scanners.dnssec import dnssecc_subdomain_enum
+from anubis.scanners.hackertarget import subdomain_hackertarget
+from anubis.scanners.netcraft import search_netcraft
+from anubis.scanners.nmap import scan_host
+from anubis.scanners.recursive import recursive_search
+from anubis.scanners.shodan import search_shodan
+from anubis.scanners.spyse import search_spyse
+from anubis.scanners.ssl import search_subject_alt_name
+from anubis.scanners.zonetransfer import dns_zonetransfer
+from anubis.utils.color_print import ColorPrint
+from .base import Base
+
+
+class Target(Base):
+  """Main enumeration module"""
+  domains = list()
+  dedupe = set()
+
+  stdout = sys.stdout
+
+  def handle_exception(self, e, message=""):
+    if self.options["--verbose"]:
+      print(e)
+    if message:
+      ColorPrint.red(message)
+
+  def init(self):
+    if self.options["FILE"]:
+      full_path = os.path.join(os.getcwd(), self.options["FILE"])
+      with open(full_path) as file:
+        self.options["TARGET"] = list(filter(None, file.read().split('\n')))
+    else:
+      self.options["TARGET"] = list(
+        filter(None, self.options["TARGET"].split(",")))
+    # Clean up targets
+    for i in range(len(self.options["TARGET"])):
+      url = self.options["TARGET"][i]
+      # Inject protocol if not there
+      if not re.match(r'http(s?):', url):
+        url = 'http://' + url
+
+      parsed = urlsplit(url)
+      host = parsed.netloc
+
+      self.options["TARGET"][i] = host
+
+      try:
+        ip_str = socket.gethostbyname(host)
+        ColorPrint.green(f"Searching for subdomains for {ip_str} ({host})")
+      except Exception as e:
+        self.handle_exception(e,
+                              "Error connecting to target! Make sure you spelled it correctly and it is a resolvable address")
+
+  def run(self):
+    # Retrieve IP of target and run initial configurations
+    self.init()
+    # If multiple targets, create scans for each
+    for i in range(len(self.options["TARGET"])):
+      # Default scans that run every time
+      target = self.options["TARGET"][i]
+      ColorPrint.green(f"Working on target: {target}")
+      threads = [threading.Thread(target=dns_zonetransfer, args=(self, target)),
+                 threading.Thread(target=subdomain_hackertarget,
+                                  args=(self, target)),
+                 threading.Thread(target=search_subject_alt_name,
+                                  args=(self, target)),
+                 threading.Thread(target=search_netcraft, args=(self, target)),
+                 threading.Thread(target=search_hudsonrock, args=(self, target)),
+                 threading.Thread(target=search_crtsh, args=(self, target)),
+                 threading.Thread(target=search_spyse,
+                                  args=(self, target)),
+                 threading.Thread(target=search_anubisdb, args=(self, target))]
+
+      # Additional options - shodan.io scan
+      if self.options["--additional-info"]:
+        threads.append(threading.Thread(target=search_shodan, args=(self,)))
+
+      # Additional options - nmap scan of dnssec script and a host/port scan
+      if self.options["--with-nmap"]:
+        threads.append(
+          threading.Thread(target=dnssecc_subdomain_enum, args=(self, target)))
+        threads.append(threading.Thread(target=scan_host, args=(self, target)))
+
+      # Start all threads and wait for them to finish
+      for x in threads:
+        x.start()
+
+      for x in threads:
+        x.join()
+
+      # Run a recursive search on each subdomain - rarely useful, but nice to have
+      # just in case
+      if self.options["--recursive"]:
+        recursive_search(self)
+
+      # remove duplicates and clean up
+      self.domains = self.clean_domains(self.domains)
+      self.dedupe = set(self.domains)
+
+      print("Found", len(self.dedupe), "subdomains")
+      print("----------------")
+
+      if self.options["--ip"]:
+        self.resolve_ips()
+      else:
+        for domain in self.dedupe:
+          cleaned_domain = domain.strip()
+          ColorPrint.green(cleaned_domain)
+          if self.options['--silent']:
+            sys.stdout.write(cleaned_domain + '\n', override=True)
+
+      if self.options["--send-to-anubis-db"]:
+        send_to_anubisdb(self, [target])
+      # reset per domain
+      self.domains = list()
+
+  def resolve_ips(self):
+    unique_ips = set()
+    for domain in self.dedupe:
+      try:
+        # Attempt to get IP
+        resolved_ip = socket.gethostbyname(domain)
+      except Exception as e:
+        # If getting IP fails, fallback to empty string
+        resolved_ip = ""
+      # TODO - Align domains and ips in stdout
+      ColorPrint.green(domain + ": " + resolved_ip)
+      if self.options['--silent']:
+        sys.stdout.write(domain + '\n', override=True)
+
+      if resolved_ip:
+        unique_ips.add(resolved_ip)
+    print("Found %s unique IPs" % len(unique_ips))
+    for ip in unique_ips:
+      # Ignore empty strings, final sanity check
+      if ip:
+        ColorPrint.green(ip)
+
+  @staticmethod
+  def clean_domains(domains):
+    cleaned = []
+    for subdomain in domains:
+      subdomain = subdomain.lower()
+      if subdomain.find("//") != -1:
+        subdomain = subdomain[subdomain.find("//") + 2:]
+      # Some pkey return instances like example.com. - remove the final .
+      if subdomain.endswith('.'):
+        subdomain = subdomain[:-1]
+      # sometimes we'll get something like /www.example.com
+      if subdomain[0] in ["\\", ".", "/", "#", "$", "%"]:
+        subdomain = subdomain[1:]
+      # If it's an email address, only take the domain part
+      if "@" in subdomain:
+        subdomain = subdomain.split("@")
+        # If it's an actual email like mail@example.com, take example.com
+        if len(subdomain) > 1:
+          subdomain = subdomain[1]
+        else:
+          # If for some reason it's example.com@, take example.com
+          subdomain = subdomain[0]
+
+      cleaned.append(subdomain.strip())
+    return cleaned

anubis/scanners/anubis_db.py

@@ -0,0 +1,29 @@
+from json import loads
+
+import requests
+
+from anubis.utils.color_print import ColorPrint
+
+
+def search_anubisdb(self, target):
+  print("Searching Anubis-DB")
+  res = requests.get("https://anubisdb.com/subdomains/" + target)
+  if res.status_code == 200 and res.text:
+    subdomains = loads(res.text)
+    for subdomain in subdomains:
+      if subdomain not in self.domains:
+        self.domains.append(subdomain)
+
+
+def send_to_anubisdb(self, target):
+  if len(target) == 1:
+    print("Sending to AnubisDB")
+    data = {'subdomains': self.domains}
+    # Sends found subdomains to Anubis (max 10,000/post)
+    res = requests.post("https://anubisdb.com/subdomains/" + target[0],
+                        json=data)
+    if res.status_code != 200:
+      ColorPrint.red("Error sending results to AnubisDB - Status Code: " + str(
+        res.status_code))
+  else:
+    print("Cannot send multiple domains to AnubisDB")

anubis/scanners/censys.py

@@ -0,0 +1,21 @@
+import censys
+
+from anubis.utils.color_print import ColorPrint
+
+
+def search_censys(self, target):
+  print("Searching Censys")
+  try:
+    from anubis.API import CENSYS_ID, CENSYS_SECRET
+  except ImportError:
+    ColorPrint.red(
+      "To run a Censys scan, you must add your API keys to anubis/API.py")
+    return
+  if not CENSYS_SECRET or not CENSYS_ID:
+    ColorPrint.red(
+      "To run a Censys scan, you must add your API keys to anubis/API.py")
+    return
+  # Print certificate information for domains
+  c = censys.certificates.CensysCertificates(CENSYS_ID, CENSYS_SECRET)
+  for cert in c.search("." + target):
+    print(cert)

anubis/scanners/crt.py

@@ -0,0 +1,132 @@
+import re
+from html.parser import HTMLParser
+
+import requests
+
+
+def search_crtsh(self, target):
+    """
+    Search crt.sh for certificates related to the target and extract both
+    subdomains and root domains.
+    """
+    print("Searching crt.sh")
+    headers = {
+        'authority': 'crt.sh',
+        'cache-control': 'max-age=0',
+        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.28 Safari/537.36',
+        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+        'accept-encoding': 'gzip, deflate, br',
+        'accept-language': 'en-US,en;q=0.9',
+    }
+
+    # Clean target input to ensure we have just the domain
+    target = clean_domain(target)
+
+    # Search for both wildcard and explicit domain
+    params = (('q', '%.' + target),)
+    try:
+        res = requests.get('https://crt.sh/', headers=headers, params=params)
+        scraped = res.text
+
+        # Extract domains from response
+        domains = extract_domains(scraped, target)
+
+        # Add all found domains to our list
+        for domain in domains:
+            if domain not in self.domains:
+                self.domains.append(domain)
+                if self.options["--verbose"]:
+                    print("Crt.sh Found Domain:", domain)
+
+    except Exception as e:
+        self.handle_exception(e, "Error searching crt.sh")
+
+
+def clean_domain(domain):
+    """Clean input to get the base domain without protocol or path."""
+    # Remove protocol if present
+    domain = re.sub(r'^(http|https)://', '', domain)
+    # Remove path, query params, etc.
+    domain = domain.split('/')[0].strip()
+    return domain
+
+
+class _CertificateNamesParser(HTMLParser):
+  """Collect direct cell text, separating names at BR and cell boundaries.
+
+  Unsupported nested markup discards the current name and the rest of its cell.
+  """
+
+  _CELL_BOUNDARIES = ('td', 'th', 'tr', 'thead', 'tbody', 'tfoot', 'table')
+
+  def __init__(self):
+    super().__init__(convert_charrefs=True)
+    self.names = []
+    self._parts = None
+
+  def _finish_name(self):
+    if self._parts is not None:
+      self.names.append(''.join(self._parts))
+    self._parts = None
+
+  def handle_starttag(self, tag, attrs):
+    if tag == 'td':
+      self._finish_name()
+      self._parts = []
+    elif tag == 'br' and self._parts is not None:
+      self._finish_name()
+      self._parts = []
+    elif tag in self._CELL_BOUNDARIES:
+      self._finish_name()
+    elif self._parts is not None:
+      # Do not interpret scripts, links, or other nested markup as names.
+      self._parts = None
+
+  def handle_endtag(self, tag):
+    if tag in self._CELL_BOUNDARIES:
+      self._finish_name()
+    elif tag == 'br' and self._parts is not None:
+      self._finish_name()
+      self._parts = []
+    elif self._parts is not None:
+      self._parts = None
+
+  def handle_data(self, data):
+    if self._parts is not None:
+      self._parts.append(data)
+
+  def close(self):
+    super().close()
+    self._finish_name()
+
+
+def extract_domains(html_content, target):
+  """Return concrete in-scope hostnames, always including the normalized target.
+
+  Wildcard certificate names are skipped: stripping ``*.`` would invent a
+  concrete hostname. As before, labels preceding the caller's target must be
+  ASCII DNS labels. The target suffix is not revalidated or converted via IDNA.
+  """
+  target = target.strip().lower()
+  domains = {target}
+  parser = _CertificateNamesParser()
+  parser.feed(html_content)
+  parser.close()
+  label_pattern = re.compile(r'[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?')
+  target_labels = len(target.split('.'))
+
+  for name in parser.names:
+    domain = name.strip()
+    if '*' in domain or len(domain.rstrip('.')) > 253:
+      continue
+    if target.isascii() and not domain.isascii():
+      continue
+    prefix_labels = domain.split('.')[:-target_labels]
+    domain = domain.lower()
+    if domain != target and not domain.endswith('.' + target):
+      continue
+    if all(label.isascii() and label_pattern.fullmatch(label.lower())
+           for label in prefix_labels):
+      domains.add(domain)
+
+  return list(domains)

anubis/scanners/dnssec.py

@@ -0,0 +1,23 @@
+import os
+
+from nmap import nmap
+
+from anubis.utils.color_print import ColorPrint
+
+
+def dnssecc_subdomain_enum(self, target):
+  # Must run as root
+  if os.getuid() == 0:
+    print("Starting DNSSEC Enum")
+    nm = nmap.PortScanner()
+    arguments = '-sSU -p 53 --script dns-nsec-enum --script-args dns-nsec-enum.domains=' + target
+
+    nm.scan(hosts=target, arguments=arguments)
+    for host in nm.all_hosts():
+      try:
+        print(nm[host]['udp'][53]['script']['dns-nsec-enum'])
+      except:
+        pass
+  else:
+    ColorPrint.red(
+      "To run a DNSSEC subdomain enumeration, Anubis must be run as root")

anubis/scanners/hackertarget.py

@@ -0,0 +1,24 @@
+import requests
+
+
+def subdomain_hackertarget(self, target):
+  print("Searching HackerTarget")
+  headers = {
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36', }
+  params = (('q', target),)
+
+  results = requests.get('http://api.hackertarget.com/hostsearch/',
+                         headers=headers, params=params)
+  results = results.text.split('\n')
+  for res in results:
+    try:
+      if res.split(",")[0] != "":
+        domain = res.split(",")[0]
+        domain = domain.strip()
+        if domain not in self.domains and domain.endswith("." + target):
+          self.domains.append(domain)
+          if self.options["--verbose"]:
+            print("HackerTarget Found Domain:", domain.strip())
+    except:
+      print("except")
+      pass

anubis/scanners/hudson_rock.py

@@ -0,0 +1,24 @@
+from urllib.parse import urlsplit
+
+import requests
+
+
+def search_hudsonrock(self, target):
+  print("Searching HudsonRock")
+  try:
+    res = requests.get("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-domain?domain=" + target)
+    urls = res.json().get('data', {}).get('all_urls', [])
+  except Exception as e:
+    self.handle_exception(e, "Error searching HudsonRock")
+    return
+
+  for url_entry in urls:
+    try:
+      # Entries are full URLs (https://sso.example.com/login); keep the host
+      domain = urlsplit(url_entry.get('url', '')).hostname
+    except (AttributeError, ValueError):
+      continue
+    if domain and domain.endswith("." + target) and domain not in self.domains:
+      self.domains.append(domain)
+      if self.options["--verbose"]:
+        print("HudsonRock Found Domain:", domain)

anubis/scanners/netcraft.py

@@ -0,0 +1,34 @@
+import re
+
+import requests
+
+
+def search_netcraft(self, target):
+  print("Searching NetCraft.com")
+  headers = {'Pragma': 'no-cache', 'DNT': '1',
+             'Accept-Encoding': 'gzip, deflate, br',
+             'Accept-Language': 'en-US,en;q=0.9,it;q=0.8',
+             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',
+             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+             'Cache-Control': 'no-cache',
+             'Referer': 'https://searchdns.netcraft.com/?restriction=site+ends+with&host=',
+             'Connection': 'keep-alive', }
+
+  params = (('restriction', 'site contains'), ('host', target))
+  try:
+    res = requests.get('https://searchdns.netcraft.com/', headers=headers,
+                       params=params)
+    scraped = res.text
+    trimmed = scraped[scraped.find('<div class="blogtitle">'):scraped.rfind(
+      '<div id="copyright">')]
+    subdomain_finder = re.compile(
+      r'<a href="http://toolbar.netcraft.com/site_report\?url=(.*)">')
+    links = subdomain_finder.findall(trimmed)
+    for domain in links:
+      if domain.strip() not in self.domains and domain.endswith("." + target):
+        self.domains.append(domain.strip())
+        if self.options["--verbose"]:
+          print("Netcraft Found Domain:", domain.strip())
+  except Exception as e:
+    self.handle_exception(e, "Error searching NetCraft")
+    pass

anubis/scanners/nmap.py

@@ -0,0 +1,53 @@
+import shutil
+from json import dumps
+
+import nmap
+
+
+# Performs an nmap scan of a target, and outputs interesting services/ssl information
+def scan_host(self, target):
+  if shutil.which("nmap") is None:
+    print("Nmap must be installed to run --with-nmap!")
+    return
+
+  print("Starting Nmap Scan")
+  nm = nmap.PortScanner()
+  arguments = self.options["--overwrite-nmap-scan"] or '-nPn -sV -sC'
+  nm.scan(hosts=target, arguments=arguments)
+  # Could be extended to scan multiple hosts in the future
+  for host in nm.all_hosts():
+    if self.options["--verbose"]:
+      print(dumps(nm[host], indent=2, sort_keys=True))
+    print('Host : %s (%s)' % (host, nm[host].hostname()))
+    for proto in nm[host].all_protocols():
+      print('----------')
+    print('Protocol: %s' % proto)
+    lport = nm[host][proto].keys()
+    for port in lport:
+      print('port: %s\tstate: %s' % (port, nm[host][proto][port]['state']))
+      try:
+        if nm[host][proto][port]['product']:
+          print('\tservice: %s' % nm[host][proto][port]['product'], end='')
+          if nm[host][proto][port]['version']:
+            print('\tversion: %s' % nm[host][proto][port]['version'])
+          else:
+            print('')
+      except Exception as e:
+        self.handle_exception(e)
+      try:
+        fix_newline = nm[host][proto][port]['script']['ssl-cert'].split('\n')
+        print('\tCertificate:')
+        for i in range(len(fix_newline)):
+          print('\t', fix_newline[i])
+          if fix_newline[i].startswith("Subject Alternative Name: "):
+            content = fix_newline[i].replace("Subject Alternative Name: ", '')
+            content = content.replace("DNS:", '')
+            new_domains = content.split(",")
+            for domain in new_domains:
+              domain = domain.strip()
+              if domain not in self.domains:
+                self.domains.append(domain)
+                if self.options["--verbose"]:
+                  print("Nmap Found Domain:", domain.strip())
+      except Exception as e:
+        self.handle_exception(e)