oneforall-kjl 0.1.1__py3-none-any.whl

OneForAll/modules/check/axfr.py

@@ -0,0 +1,97 @@
+"""
+查询域名的NS记录(域名服务器记录，记录该域名由哪台域名服务器解析)，检查查出的域名服务器是
+否开启DNS域传送，如果开启且没做访问控制和身份验证便加以利用获取域名的所有记录。
+
+DNS域传送(DNS zone transfer)指的是一台备用域名服务器使用来自主域名服务器的数据刷新自己
+的域数据库，目的是为了做冗余备份，防止主域名服务器出现故障时 dns 解析不可用。
+当主服务器开启DNS域传送同时又对来请求的备用服务器未作访问控制和身份验证便可以利用此漏洞获
+取某个域的所有记录。
+"""
+import dns.resolver
+import dns.zone
+
+from common import utils
+from common.check import Check
+from config.log import logger
+
+
+class AXFR(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = 'AXFRCheck'
+        self.results = []
+
+    def axfr(self, server):
+        """
+        Perform domain transfer
+
+        :param server: domain server
+        """
+        logger.log('DEBUG', f'Trying to perform domain transfer in {server} '
+                            f'of {self.domain}')
+        try:
+            xfr = dns.query.xfr(where=server, zone=self.domain,
+                                timeout=5.0, lifetime=10.0)
+            zone = dns.zone.from_xfr(xfr)
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+            logger.log('DEBUG', f'Domain transfer to server {server} of '
+                                f'{self.domain} failed')
+            return
+        names = zone.nodes.keys()
+        for name in names:
+            full_domain = str(name) + '.' + self.domain
+            subdomain = self.match_subdomains(full_domain)
+            self.subdomains.update(subdomain)
+            record = zone[name].to_text(name)
+            self.results.append(record)
+        if self.results:
+            logger.log('DEBUG', f'Found the domain transfer record of '
+                                f'{self.domain} on {server}')
+            logger.log('DEBUG', '\n'.join(self.results))
+            self.results = []
+
+    def check(self):
+        """
+        check
+        """
+        resolver = utils.dns_resolver()
+        try:
+            answers = resolver.query(self.domain, "NS")
+        except Exception as e:
+            logger.log('ERROR', e.args)
+            return
+        nsservers = [str(answer) for answer in answers]
+        if not len(nsservers):
+            logger.log('ALERT', f'No name server record found for {self.domain}')
+            return
+        for nsserver in nsservers:
+            self.axfr(nsserver)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = AXFR(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    run('ZoneTransfer.me')
+    # run('example.com')

OneForAll/modules/check/cdx.py

@@ -0,0 +1,44 @@
+"""
+检查crossdomain.xml文件收集子域名
+"""
+from common.check import Check
+
+
+class CrossDomain(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = "CrossDomainCheck"
+
+    def check(self):
+        """
+        检查crossdomain.xml收集子域名
+        """
+        filenames = {'crossdomain.xml'}
+        self.to_check(filenames)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param domain: 域名
+    """
+    check = CrossDomain(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/check/cert.py

@@ -0,0 +1,58 @@
+"""
+检查域名证书收集子域名
+"""
+import socket
+import ssl
+
+from config.log import logger
+from common.check import Check
+
+
+class CertInfo(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = 'CertInfo'
+
+    def check(self):
+        """
+        获取域名证书并匹配证书中的子域名
+        """
+        try:
+            ctx = ssl.create_default_context()
+            sock = socket.socket()
+            sock.settimeout(10)
+            wrap_sock = ctx.wrap_socket(sock, server_hostname=self.domain)
+            wrap_sock.connect((self.domain, 443))
+            cert_dict = wrap_sock.getpeercert()
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+            return
+        subdomains = self.match_subdomains(str(cert_dict))
+        self.subdomains.update(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = CertInfo(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/check/csp.py

@@ -0,0 +1,94 @@
+"""
+Collect subdomains from ContentSecurityPolicy
+"""
+import requests
+
+from config.log import logger
+from common.check import Check
+
+
+class CSP(Check):
+    """
+    Collect subdomains from ContentSecurityPolicy
+    """
+
+    def __init__(self, domain, header):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = 'CSPCheck'
+        self.csp_header = header
+
+    @property
+    def grab_header(self):
+        """
+        Get header
+
+        :return: ContentSecurityPolicy header
+        """
+        csp_header = dict()
+        urls = [f'http://{self.domain}',
+                f'https://{self.domain}']
+        urls_www = [f'http://www.{self.domain}',
+                    f'https://www.{self.domain}']
+        header = self.grab_loop(csp_header, urls)
+        if header:
+            return header
+        header = self.grab_loop(csp_header, urls_www)
+        return header
+
+    def grab_loop(self, csp_header, urls):
+        for url in urls:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            try:
+                response = self.get(url, check=False, ignore=True, raise_error=True)
+            except requests.exceptions.ConnectTimeout:
+                logger.log('DEBUG', f'Connection to {url} timed out, so break check')
+                break
+            if response:
+                return response.headers
+        return csp_header
+
+    def check(self):
+        """
+        正则匹配响应头中的内容安全策略字段以发现子域名
+        """
+        if not self.csp_header:
+            self.csp_header = self.grab_header
+        csp = self.csp_header.get('Content-Security-Policy')
+        if not self.csp_header:
+            logger.log('DEBUG', f'Failed to get header of {self.domain} domain')
+            return
+        if not csp:
+            logger.log('DEBUG', f'There is no Content-Security-Policy in the header '
+                                f'of {self.domain}')
+            return
+        self.subdomains = self.match_subdomains(csp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain, header=None):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    :param dict or None header: 响应头
+    """
+    check = CSP(domain, header)
+    check.run()
+
+
+if __name__ == '__main__':
+    resp = requests.get('https://content-security-policy.com/')
+    run('google-analytics.com', dict(resp.headers))

OneForAll/modules/check/nsec.py

@@ -0,0 +1,58 @@
+# https://www.icann.org/resources/pages/dnssec-what-is-it-why-important-2019-03-20-zh
+# https://appsecco.com/books/subdomain-enumeration/active_techniques/zone_walking.html
+
+from common import utils
+from common.check import Check
+
+
+class NSEC(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = "NSECCheck"
+
+    def walk(self):
+        domain = self.domain
+        while True:
+            answer = utils.dns_query(domain, 'NSEC')
+            if answer is None:
+                break
+            subdomain = str()
+            for item in answer:
+                record = item.to_text()
+                subdomains = self.match_subdomains(record)
+                subdomain = ''.join(subdomains)  # 其实这里的subdomains的长度为1 也就是说只会有一个子域
+                self.subdomains.update(subdomains)
+            if subdomain == self.domain:  # 当查出子域为主域 说明完成了一个循环 不再继续查询
+                break
+            if domain != self.domain:  # 防止出现wwdmas.cn 000.000.wwdmas.cn 000.000.000.wwdmas.cn情况
+                if domain.split('.')[0] == subdomain.split('.')[0]:
+                    break
+            domain = subdomain
+        return self.subdomains
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.walk()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    brute = NSEC(domain)
+    brute.run()
+
+
+if __name__ == '__main__':
+    run('iana.org')

OneForAll/modules/check/robots.py

@@ -0,0 +1,44 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+from common.check import Check
+
+
+class Robots(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = 'RobotsCheck'
+
+    def check(self):
+        """
+        正则匹配域名的robots.txt文件中的子域
+        """
+        filenames = {'robots.txt'}
+        self.to_check(filenames)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = Robots(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

OneForAll/modules/check/sitemap.py

@@ -0,0 +1,44 @@
+"""
+检查内容安全策略收集子域名收集子域名
+"""
+from common.check import Check
+
+
+class Sitemap(Check):
+    def __init__(self, domain):
+        Check.__init__(self)
+        self.domain = domain
+        self.module = 'check'
+        self.source = 'SitemapCheck'
+
+    def check(self):
+        """
+        正则匹配域名的sitemap文件中的子域
+        """
+        filenames = {'sitemap.xml', 'sitemap.txt', 'sitemap.html', 'sitemapindex.xml'}
+        self.to_check(filenames)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.check()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    check = Sitemap(domain)
+    check.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

OneForAll/modules/collect.py

@@ -0,0 +1,70 @@
+import threading
+import importlib
+
+from config.log import logger
+from config import settings
+
+
+class Collect(object):
+    def __init__(self, domain):
+        self.domain = domain
+        self.modules = []
+        self.collect_funcs = []
+
+    def get_mod(self):
+        """
+        Get modules
+        """
+        if settings.enable_all_module:
+            # The crawl module has some problems
+            modules = ['certificates', 'check', 'datasets',
+                       'dnsquery', 'intelligence', 'search']
+            for module in modules:
+                module_path = settings.module_dir.joinpath(module)
+                for path in module_path.rglob('*.py'):
+                    import_module = f'modules.{module}.{path.stem}'
+                    self.modules.append(import_module)
+        else:
+            self.modules = settings.enable_partial_module
+
+    def import_func(self):
+        """
+        Import do function
+        """
+        for module in self.modules:
+            name = module.split('.')[-1]
+            import_object = importlib.import_module(module)
+            func = getattr(import_object, 'run')
+            self.collect_funcs.append([func, name])
+
+    def run(self):
+        """
+        Class entrance
+        """
+        logger.log('INFOR', f'Start collecting subdomains of {self.domain}')
+        self.get_mod()
+        self.import_func()
+
+        threads = []
+        # Create subdomain collection threads
+        for func_obj, func_name in self.collect_funcs:
+            thread = threading.Thread(target=func_obj, name=func_name,
+                                      args=(self.domain,), daemon=True)
+            threads.append(thread)
+        # Start all threads
+        for thread in threads:
+            thread.start()
+        # Wait for all threads to finish
+        for thread in threads:
+            # 挨个线程判断超时 最坏情况主线程阻塞时间=线程数*module_thread_timeout
+            # 超时线程将脱离主线程 由于创建线程时已添加守护属于 所有超时线程会随着主线程结束
+            thread.join(settings.module_thread_timeout)
+
+        for thread in threads:
+            if thread.is_alive():
+                logger.log('ALERT', f'{thread.name} module thread timed out')
+
+
+if __name__ == '__main__':
+    collect = Collect('example.com')
+    collect.run()

OneForAll/modules/crawl/archivecrawl.py

@@ -0,0 +1,59 @@
+import cdx_toolkit
+from common.crawl import Crawl
+from config.log import logger
+
+
+class ArchiveCrawl(Crawl):
+    def __init__(self, domain):
+        Crawl.__init__(self)
+        self.domain = domain
+        self.module = 'Crawl'
+        self.source = 'ArchiveCrawl'
+
+    def crawl(self, domain, limit):
+        """
+
+        :param domain:
+        :param limit:
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        cdx = cdx_toolkit.CDXFetcher(source='ia')
+        url = f'*.{domain}/*'
+        size = cdx.get_size_estimate(url)
+        logger.log('DEBUG', f'{url} ArchiveCrawl size estimate {size}')
+
+        for resp in cdx.iter(url, limit=limit):
+            if resp.data.get('status') not in ['301', '302']:
+                url = resp.data.get('url')
+                subdomains = self.match_subdomains(domain, url + resp.text)
+                self.subdomains.update(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.crawl(self.domain, 50)
+        # 爬取已发现的子域以发现新的子域
+        for subdomain in self.subdomains:
+            if subdomain != self.domain:
+                self.crawl(subdomain, 10)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    crawl = ArchiveCrawl(domain)
+    crawl.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/crawl/commoncrawl.py

@@ -0,0 +1,59 @@
+import cdx_toolkit
+from tqdm import tqdm
+
+from common.crawl import Crawl
+
+
+class CommonCrawl(Crawl):
+    def __init__(self, domain):
+        Crawl.__init__(self)
+        self.domain = domain
+        self.module = 'Crawl'
+        self.source = 'CommonCrawl'
+
+    def crawl(self, domain, limit):
+        """
+
+        :param domain:
+        :param limit:
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        cdx = cdx_toolkit.CDXFetcher()
+        url = f'*.{domain}/*'
+        size = cdx.get_size_estimate(url)
+        print(url, 'CommonCrawl size estimate', size)
+
+        for resp in tqdm(cdx.iter(url, limit=limit), total=limit):
+            if resp.data.get('status') not in ['301', '302']:
+                subdomains = self.match_subdomains(domain, resp.text)
+                self.subdomains.update(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.crawl(self.domain, 50)
+        # 爬取已发现的子域以发现新的子域
+        for subdomain in self.subdomains:
+            if subdomain != self.domain:
+                self.crawl(subdomain, 10)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    crawl = CommonCrawl(domain)
+    crawl.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/anubis.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Anubis(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'AnubisQuery'
+        self.addr = 'https://jldc.me/anubis/subdomains/'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr + self.domain
+        resp = self.get(self.addr)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Anubis(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('hackerone.com')