oneforall-kjl 0.1.1__py3-none-any.whl

OneForAll/modules/datasets/urlscan.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Urlscan(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'UrlscanQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = 'https://urlscan.io/api/v1/search/'
+        params = {'q': 'domain:' + self.domain}
+        resp = self.get(url, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Urlscan(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('sangfor.com')

OneForAll/modules/datasets/windvane.py

@@ -0,0 +1,92 @@
+from config import settings
+from common.query import Query
+
+
+class Windvane(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = "WindvaneQuery"
+        self.addr = 'https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain'
+        self.api_key = settings.windvane_api_token
+        self.page_size = 1000
+        
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({
+            'Content-Type': 'application/json',
+            'Referer': 'https://windvane.lichoin.com'
+        })
+        
+        if self.api_key:
+            self.header.update({'X-Api-Key': self.api_key})
+            
+        self.proxy = self.get_proxy(self.source)
+        
+        page = 1
+        total_pages = 1
+        all_subdomains = []
+        
+        while page <= total_pages:
+            data = {
+                "domain": self.domain,
+                "page_request": {
+                    "page": page,
+                    "count": self.page_size
+                }
+            }
+            
+            resp = self.post(self.addr, json=data)
+            if not resp:
+                break
+                
+            try:
+                result = resp.json()
+                
+                if result.get('code') != 0:
+                    break
+                
+                data_section = result.get('data', {})
+
+                subdomains = self.match_subdomains(resp)
+                if not subdomains:
+                    break
+                self.subdomains.update(subdomains)
+                
+                page_info = data_section.get('page_response', {})
+                total_pages = int(page_info.get('total_page', 1))
+                
+                page += 1
+                
+            except:
+                break
+        
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Windvane(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('baidu.com')

OneForAll/modules/dnsquery/mx.py

@@ -0,0 +1,35 @@
+from common.lookup import Lookup
+
+
+class QueryMX(Lookup):
+    def __init__(self, domain):
+        Lookup.__init__(self)
+        self.domain = domain
+        self.module = 'dnsquery'
+        self.source = "QueryMX"
+        self.qtype = 'MX'  # 利用的DNS记录的MX记录收集子域
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QueryMX(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('cuit.edu.cn')

OneForAll/modules/dnsquery/ns.py

@@ -0,0 +1,35 @@
+from common.lookup import Lookup
+
+
+class QueryNS(Lookup):
+    def __init__(self, domain):
+        Lookup.__init__(self)
+        self.domain = domain
+        self.module = 'dnsquery'
+        self.source = "QueryNS"
+        self.qtype = 'NS'  # 利用的DNS记录的NS记录收集子域
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QueryNS(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('cuit.edu.cn')

OneForAll/modules/dnsquery/soa.py

@@ -0,0 +1,35 @@
+from common.lookup import Lookup
+
+
+class QuerySOA(Lookup):
+    def __init__(self, domain):
+        Lookup.__init__(self)
+        self.domain = domain
+        self.module = 'dnsquery'
+        self.source = "QuerySOA"
+        self.qtype = 'SOA'  # 利用的DNS记录的SOA记录收集子域
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QuerySOA(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('cuit.edu.cn')

OneForAll/modules/dnsquery/spf.py

@@ -0,0 +1,35 @@
+from common.lookup import Lookup
+
+
+class QuerySPF(Lookup):
+    def __init__(self, domain):
+        Lookup.__init__(self)
+        self.domain = domain
+        self.module = 'dnsquery'
+        self.source = "QuerySPF"
+        self.qtype = 'SPF'  # 利用的DNS记录的SPF记录收集子域
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    brute = QuerySPF(domain)
+    brute.run()
+
+
+if __name__ == '__main__':
+    run('qq.com')

OneForAll/modules/dnsquery/txt.py

@@ -0,0 +1,35 @@
+from common.lookup import Lookup
+
+
+class QueryTXT(Lookup):
+    def __init__(self, domain):
+        Lookup.__init__(self)
+        self.domain = domain
+        self.module = 'dnsquery'
+        self.source = "QueryTXT"
+        self.qtype = 'TXT'  # 利用的DNS记录的TXT记录收集子域
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QueryTXT(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('cuit.edu.cn')

OneForAll/modules/enrich.py

@@ -0,0 +1,72 @@
+from modules import iscdn
+from common import utils
+from common.database import Database
+from common.ipasn import IPAsnInfo
+from common.ipreg import IpRegData
+
+
+def get_ips(info):
+    ip = info.get('ip')
+    if not ip:
+        return None
+    ips = ip.split(',')
+    return ips
+
+
+def enrich_info(data):
+    ip_asn = IPAsnInfo()
+    ip_reg = IpRegData()
+    for index, info in enumerate(data):
+        ips = get_ips(info)
+        if not ips:
+            continue
+        public = list()
+        cidr = list()
+        asn = list()
+        org = list()
+        addr = list()
+        isp = list()
+        for ip in ips:
+            public.append(str(utils.ip_is_public(ip)))
+            asn_info = ip_asn.find(ip)
+            cidr.append(asn_info.get('cidr'))
+            asn.append(asn_info.get('asn'))
+            org.append(asn_info.get('org'))
+            ip_info = ip_reg.query(ip)
+            addr.append(ip_info.get('addr'))
+            isp.append(ip_info.get('isp'))
+        data[index]['public'] = ','.join(public)
+        data[index]['cidr'] = ','.join(cidr)
+        data[index]['asn'] = ','.join(asn)
+        data[index]['org'] = ','.join(org)
+        data[index]['addr'] = ','.join(addr)
+        data[index]['isp'] = ','.join(isp)
+    return data
+
+
+class Enrich(object):
+    def __init__(self, domain):
+        self.domain = domain
+
+    def get_data(self):
+        db = Database()
+        fields = ['url', 'cname', 'ip', 'public', 'cdn', 'header',
+                  'cidr', 'asn', 'org', 'addr', 'isp']
+        results = db.get_data_by_fields(self.domain, fields)
+        return results.as_dict()
+
+    def save_db(self, data):
+        db = Database()
+        for info in data:
+            url = info.pop('url')
+            info.pop('cname')
+            info.pop('ip')
+            info.pop('header')
+            db.update_data_by_url(self.domain, info, url)
+        db.close()
+
+    def run(self):
+        data = self.get_data()
+        data = enrich_info(data)
+        data = iscdn.do_check(data)
+        self.save_db(data)

OneForAll/modules/finder.py

@@ -0,0 +1,206 @@
+import re
+import time
+from urllib import parse
+from requests import Response
+
+from common import utils
+from common import resolve
+from common import request
+from common.module import Module
+from common.database import Database
+from config import settings
+from config.log import logger
+
+
+class Finder(Module):
+    def __init__(self):
+        Module.__init__(self)
+        self.module = 'Finder'
+        self.source = 'Finder'
+        self.start = time.time()  # 模块开始执行时间
+
+    def run(self, domain, data, port):
+        logger.log('INFOR', f'Start Finder module')
+        existing_subdomains = set(map(lambda x: x.get('subdomain'), data))  # 已有的子域
+        found_subdomains = find_subdomains(domain, data)
+        new_subdomains = found_subdomains - existing_subdomains
+        if not len(new_subdomains):
+            self.finish()  # 未发现新的子域就直接返回
+        self.subdomains = new_subdomains
+        self.finish()
+        self.gen_result()
+        resolved_data = resolve.run_resolve(domain, self.results)
+        request.run_request(domain, resolved_data, port)
+
+
+file_path = settings.data_storage_dir.joinpath('common_js_library.json')
+black_name = utils.load_json(file_path)
+# Regular expression comes from https://github.com/GerbenJavado/LinkFinder
+expression = r"""
+    (?:"|')                               # Start newline delimiter
+    (
+        ((?:[a-zA-Z]{1,10}://|//)           # Match a scheme [a-Z]*1-10 or //
+        [^"'/]{1,}\.                        # Match a domain name (any character + dot)
+        [a-zA-Z]{2,}[^"']{0,})              # The domain extension and/or path
+        |
+        ((?:/|\.\./|\./)                    # Start with /,../,./
+        [^"'><,;| *()(%%$^/\\\[\]]          # Next character can't be...
+        [^"'><,;|()]{1,})                   # Rest of the characters can't be
+        |
+        ([a-zA-Z0-9_\-/]{1,}/               # Relative endpoint with /
+        [a-zA-Z0-9_\-/]{1,}                 # Resource name
+        \.(?:[a-zA-Z]{1,4}|action)          # Rest + extension (length 1-4 or action)
+        (?:[\?|/][^"|']{0,}|))              # ? mark with parameters
+        |
+        ([a-zA-Z0-9_\-]{1,}                 # filename
+        \.(?:js)                            # . + extension
+        (?:\?[^"|']{0,}|))                  # ? mark with parameters
+    )
+    (?:"|')                                 # End newline delimiter
+    """
+url_pattern = re.compile(expression, re.VERBOSE)
+
+
+def find_new_urls(html):
+    result = re.finditer(url_pattern, html)
+    if result is None:
+        return None
+    urls = set()
+    for match in result:
+        url = match.group().strip('"').strip("'")
+        urls.add(url)
+    return urls
+
+
+def convert_url(req_url, rel_url):
+    black_url = ["javascript:"]  # Add some keyword for filter url.
+    raw_url = parse.urlparse(req_url)
+    netloc = raw_url.netloc
+    scheme = raw_url.scheme
+    if rel_url[0:2] == "//":
+        result = scheme + ":" + rel_url
+    elif rel_url[0:4] == "http":
+        result = rel_url
+    elif rel_url[0:2] != "//" and rel_url not in black_url:
+        if rel_url[0:1] == "/":
+            result = scheme + "://" + netloc + rel_url
+        else:
+            if rel_url[0:1] == ".":
+                if rel_url[0:2] == "..":
+                    result = scheme + "://" + netloc + rel_url[2:]
+                else:
+                    result = scheme + "://" + netloc + rel_url[1:]
+            else:
+                result = scheme + "://" + netloc + "/" + rel_url
+    else:
+        result = req_url
+    return result
+
+
+def filter_name(path):
+    for name in black_name:
+        if path.endswith(name):
+            return True
+    black_ext = ['io.js', 'ui.js', 'fp.js', 'en.js', 'en-us,js', 'zh.js', 'zh-cn.js',
+                 'zh_cn.js', 'dev.js', 'min.js', 'umd.js', 'esm.js', 'all.js', 'cjs.js',
+                 'prod.js', 'slim.js', 'core.js', 'global.js', 'bundle.js', 'browser.js',
+                 'brands.js', 'simple.js', 'common.js', 'development.js', 'banner.js',
+                 'production.js']
+    for ext in black_ext:
+        if path.endswith(ext):
+            return True
+    r = re.compile(r'\d+.\d+.\d+')
+    if r.search(path):
+        return True
+    return False
+
+
+def filter_url(domain, url):
+    try:
+        raw_url = parse.urlparse(url)
+    except Exception as e:  # 解析失败则跳过该URL
+        logger.log('DEBUG', e.args)
+        return True
+    scheme = raw_url.scheme.lower()
+    if not scheme:
+        return True
+    if scheme not in ['http', 'https']:
+        return True
+    netloc = raw_url.netloc.lower()
+    if not netloc:
+        return True
+    if not netloc.endswith(domain):
+        return True
+    path = raw_url.path.lower()
+    if not path:
+        return True
+    if not path.endswith('.js'):
+        return True
+    if path.endswith('min.js'):
+        return True
+    return filter_name(path)
+
+
+def match_subdomains(domain, text):
+    if isinstance(text, str):
+        subdomains = utils.match_subdomains(domain, text, fuzzy=False)
+    else:
+        logger.log('DEBUG', f'abnormal object: {type(text)}')
+        subdomains = set()
+    logger.log('TRACE', f'matched subdomains: {subdomains}')
+    return subdomains
+
+
+def find_in_resp(domain, url, html):
+    logger.log('TRACE', f'matching subdomains from response of {url}')
+    return match_subdomains(domain, html)
+
+
+def find_in_history(domain, url, history):
+    logger.log('TRACE', f'matching subdomains from history of {url}')
+    return match_subdomains(domain, history)
+
+
+def find_js_urls(domain, req_url, rsp_html):
+    js_urls = set()
+    new_urls = find_new_urls(rsp_html)
+    if not new_urls:
+        return js_urls
+    for rel_url in new_urls:
+        url = convert_url(req_url, rel_url)
+        if not filter_url(domain, url):
+            js_urls.add(url)
+    return js_urls
+
+
+def convert_to_dict(url_list):
+    url_dict = []
+    for url in url_list:
+        url_dict.append({'url': url})
+    return url_dict
+
+
+def find_subdomains(domain, data):
+    subdomains = set()
+    js_urls = set()
+    db = Database()
+    for infos in data:
+        jump_history = infos.get('history')
+        req_url = infos.get('url')
+        subdomains.update(find_in_history(domain, req_url, jump_history))
+        rsp_html = db.get_resp_by_url(domain, req_url)
+        if not rsp_html:
+            logger.log('DEBUG', f'an abnormal response occurred in the request {req_url}')
+            continue
+        subdomains.update(find_in_resp(domain, req_url, rsp_html))
+        js_urls.update(find_js_urls(domain, req_url, rsp_html))
+
+    req_data = convert_to_dict(js_urls)
+    resp_data = request.bulk_request(domain, req_data, ret=True)
+    while not resp_data.empty():
+        _, resp = resp_data.get()
+        if not isinstance(resp, Response):
+            continue
+        text = utils.decode_resp_text(resp)
+        subdomains.update(find_in_resp(domain, resp.url, text))
+    return subdomains

OneForAll/modules/intelligence/alienvault.py

@@ -0,0 +1,50 @@
+from common.query import Query
+
+
+class AlienVault(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Intelligence'
+        self.source = 'AlienVaultQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+
+        base = 'https://otx.alienvault.com/api/v1/indicators/domain'
+        dns = f'{base}/{self.domain}/passive_dns'
+        resp = self.get(dns)
+        self.subdomains = self.collect_subdomains(resp)
+
+        url = f'{base}/{self.domain}/url_list'
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = AlienVault(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/intelligence/riskiq_api.py

@@ -0,0 +1,58 @@
+from config import settings
+from common.query import Query
+
+
+class RiskIQ(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Intelligence'
+        self.source = 'RiskIQAPIQuery'
+        self.addr = 'https://api.riskiq.net/pt/v2/enrichment/subdomains'
+        self.user = settings.riskiq_api_username
+        self.key = settings.riskiq_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'Accept': 'application/json'})
+        self.proxy = self.get_proxy(self.source)
+        params = {'query': self.domain}
+        resp = self.get(url=self.addr,
+                        params=params,
+                        auth=(self.user, self.key))
+        if not resp:
+            return
+        data = resp.json()
+        names = data.get('subdomains')
+        subdomain_str = str(set(map(lambda name: f'{name}.{self.domain}', names)))
+        self.subdomains = self.collect_subdomains(subdomain_str)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.user, self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RiskIQ(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('alibabagroup.com')