oneforall-kjl 0.1.1__py3-none-any.whl

OneForAll/modules/intelligence/threatbook_api.py

@@ -0,0 +1,50 @@
+from config import settings
+from common.query import Query
+
+
+class ThreatBookAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Intelligence'
+        self.source = 'ThreatBookAPIQuery'
+        self.addr = 'https://api.threatbook.cn/v3/domain/sub_domains'
+        self.key = settings.threatbook_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.key,
+                  'resource': self.domain}
+        resp = self.post(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatBookAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/intelligence/threatminer.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class ThreatMiner(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Intelligence'
+        self.source = 'ThreatMinerQuery'
+        self.addr = 'https://api.threatminer.org/v2/domain.php'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': self.domain, 'rt': 5}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = ThreatMiner(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/intelligence/virustotal.py

@@ -0,0 +1,60 @@
+from common.query import Query
+
+'''
+最多查询100条
+'''
+
+
+class VirusTotal(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.source = 'VirusTotalQuery'
+        self.module = 'Intelligence'
+        self.domain = domain
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        next_cursor = ''
+        while True:
+            self.header = self.get_header()
+            self.header.update({'Referer': 'https://www.virustotal.com/',
+                                'TE': 'Trailers'})
+            self.proxy = self.get_proxy(self.source)
+            params = {'limit': '40', 'cursor': next_cursor}
+            addr = f'https://www.virustotal.com/ui/domains/{self.domain}/subdomains'
+            resp = self.get(url=addr, params=params)
+            if not resp:
+                break
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:
+                break
+            self.subdomains.update(subdomains)
+            data = resp.json()
+            next_cursor = data.get('meta').get('cursor')
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = VirusTotal(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('mi.com')

OneForAll/modules/intelligence/virustotal_api.py

@@ -0,0 +1,59 @@
+from config import settings
+from common.query import Query
+
+
+class VirusTotalAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Intelligence'
+        self.source = 'VirusTotalAPIQuery'
+        self.key = settings.virustotal_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        next_cursor = ''
+        while True:
+            self.header = self.get_header()
+            self.header.update({'x-apikey': self.key})
+            self.proxy = self.get_proxy(self.source)
+            params = {'limit': '40', 'cursor': next_cursor}
+            addr = f'https://www.virustotal.com/api/v3/domains/{self.domain}/subdomains'
+            resp = self.get(url=addr, params=params)
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:
+                break
+            self.subdomains.update(subdomains)
+            data = resp.json()
+            next_cursor = data.get('meta').get('cursor')
+            if not next_cursor:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.key):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = VirusTotalAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('mi.com')

OneForAll/modules/iscdn.py

@@ -0,0 +1,86 @@
+import json
+import ipaddress
+
+from config import settings
+from common import utils
+from common.database import Database
+from config.log import logger
+
+data_dir = settings.data_storage_dir
+
+# from https://github.com/al0ne/Vxscan/blob/master/lib/iscdn.py
+cdn_ip_cidr = utils.load_json(data_dir.joinpath('cdn_ip_cidr.json'))
+cdn_asn_list = utils.load_json(data_dir.joinpath('cdn_asn_list.json'))
+
+# from https://github.com/Qclover/CDNCheck/blob/master/checkCDN/cdn3_check.py
+cdn_cname_keyword = utils.load_json(data_dir.joinpath('cdn_cname_keywords.json'))
+
+cdn_header_key = utils.load_json(data_dir.joinpath('cdn_header_keys.json'))
+
+
+def check_cname_keyword(cname):
+    if not cname:
+        return False
+    names = cname.lower().split(',')
+    for name in names:
+        for keyword in cdn_cname_keyword.keys():
+            if keyword in name:
+                return True
+
+
+def check_header_key(header):
+    if isinstance(header, str):
+        header = json.loads(header)
+    if isinstance(header, dict):
+        header = set(map(lambda x: x.lower(), header.keys()))
+        for key in cdn_header_key:
+            if key in header:
+                return True
+    else:
+        return False
+
+
+def check_cdn_cidr(ips):
+    if isinstance(ips, str):
+        ips = set(ips.split(','))
+    else:
+        return False
+    for ip in ips:
+        try:
+            ip = ipaddress.ip_address(ip)
+        except Exception as e:
+            logger.log('DEBUG', e.args)
+            return False
+        for cidr in cdn_ip_cidr:
+            if ip in ipaddress.ip_network(cidr):
+                return True
+
+
+def check_cdn_asn(asn):
+    if isinstance(asn, str):
+        if asn in cdn_asn_list:
+            return True
+    return False
+
+
+def do_check(data):
+    logger.log('DEBUG', f'Checking cdn')
+    for index, item in enumerate(data):
+        cname = item.get('cname')
+        if check_cname_keyword(cname):
+            data[index]['cdn'] = 1
+            continue
+        header = item.get('header')
+        if check_header_key(header):
+            data[index]['cdn'] = 1
+            continue
+        ip = item.get('ip')
+        if check_cdn_cidr(ip):
+            data[index]['cdn'] = 1
+            continue
+        asn = item.get('asn')
+        if check_cdn_asn(asn):
+            data[index]['cdn'] = 1
+            continue
+        data[index]['cdn'] = 0
+    return data

OneForAll/modules/search/ask.py

@@ -0,0 +1,69 @@
+import time
+from common.search import Search
+
+
+class Ask(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'AskSearch'
+        self.addr = 'https://www.search.ask.com/web'
+        self.limit_num = 200  # 限制搜索条数
+        self.per_page_num = 10  # 默认每页显示10页
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.page_num = 1
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'page': self.page_num}
+            resp = self.get(self.addr, params)
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            self.page_num += 1
+            if '>Next<' not in resp.text:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Ask(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/search/baidu.py

@@ -0,0 +1,96 @@
+import time
+from bs4 import BeautifulSoup
+from common.search import Search
+
+
+class Baidu(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.module = 'Search'
+        self.source = 'BaiduSearch'
+        self.addr = 'https://www.baidu.com/s'
+        self.domain = domain
+        self.limit_num = 750  # 限制搜索条数
+
+    def redirect_match(self, html):
+        """
+        获取跳转地址并传递地址进行跳转head请求
+
+        :param html: 响应体
+        :return: 子域
+        """
+        bs = BeautifulSoup(html, 'html.parser')
+        subdomains_all = set()
+        # 获取搜索结果中所有的跳转URL地址
+        for find_res in bs.find_all('a', {'class': 'c-showurl'}):
+            url = find_res.get('href')
+            subdomains = self.match_location(url)
+            subdomains_all.update(subdomains)
+        return subdomains_all
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'wd': query,
+                      'pn': self.page_num,
+                      'rn': self.per_page_num}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if len(domain) > 12:  # 解决百度搜索结果中域名过长会显示不全的问题
+                # 获取百度跳转URL响应头的Location字段获取直链
+                subdomains = self.redirect_match(resp.text)
+            else:
+                subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            self.page_num += self.per_page_num
+            # 搜索页面没有出现下一页时停止搜索
+            if f'&pn={self.page_num}&' not in resp.text:
+                break
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain)
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Baidu(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('mi.com')

OneForAll/modules/search/bing.py

@@ -0,0 +1,79 @@
+import time
+from common.search import Search
+
+
+class Bing(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingSearch'
+        self.init = 'https://www.bing.com/'
+        self.addr = 'https://www.bing.com/search'
+        self.limit_num = 1000  # 限制搜索条数
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.page_num = 0  # 二次搜索重新置0
+        resp = self.get(self.init)
+        if not resp:
+            return
+        self.cookie = resp.cookies  # 获取cookie bing在搜索时需要带上cookie
+        while True:
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'first': self.page_num,
+                      'count': self.per_page_num}
+            resp = self.get(self.addr, params)
+            subdomains = self.match_subdomains(resp, fuzzy=False)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if '<div class="sw_next">' not in resp.text:
+                break
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.search(self.domain)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = Bing(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/search/bing_api.py

@@ -0,0 +1,78 @@
+import time
+from config import settings
+from common.search import Search
+
+
+class BingAPI(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'BingAPISearch'
+        self.addr = 'https://api.bing.microsoft.com/v7.0/search'
+        self.id = settings.bing_api_id
+        self.key = settings.bing_api_key
+        self.limit_num = 1000  # 必应同一个搜索关键词限制搜索条数
+        self.delay = 1  # 必应自定义搜索限制时延1秒
+
+    def search(self, domain, filtered_subdomain=''):
+        """
+        发送搜索请求并做子域匹配
+
+        :param str domain: 域名
+        :param str filtered_subdomain: 过滤的子域
+        """
+        self.page_num = 0  # 二次搜索重新置0
+        while True:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.header = {'Ocp-Apim-Subscription-Key': self.key}
+            self.proxy = self.get_proxy(self.source)
+            query = 'site:.' + domain + filtered_subdomain
+            params = {'q': query, 'safesearch': 'Off',
+                      'count': self.per_page_num,
+                      'offset': self.page_num}
+            resp = self.get(self.addr, params)
+            subdomains = self.match_subdomains(resp)
+            if not self.check_subdomains(subdomains):
+                break
+            self.subdomains.update(subdomains)
+            self.page_num += self.per_page_num
+            if self.page_num >= self.limit_num:  # 搜索条数限制
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.id, self.key):
+            return
+        self.begin()
+        self.search(self.domain)
+
+        # 排除同一子域搜索结果过多的子域以发现新的子域
+        for statement in self.filter(self.domain, self.subdomains):
+            self.search(self.domain, filtered_subdomain=statement)
+
+        # 递归搜索下一层的子域
+        if self.recursive_search:
+            for subdomain in self.recursive_subdomain():
+                self.search(subdomain)
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = BingAPI(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/search/fofa_api.py

@@ -0,0 +1,74 @@
+import base64
+import time
+
+from config import settings
+from common.search import Search
+
+
+class FoFa(Search):
+    def __init__(self, domain):
+        Search.__init__(self)
+        self.domain = domain
+        self.module = 'Search'
+        self.source = 'FoFaAPISearch'
+        self.addr = 'https://fofa.info/api/v1/search/all'
+        self.delay = 1
+        self.email = settings.fofa_api_email
+        self.key = settings.fofa_api_key
+
+    def search(self):
+        """
+        发送搜索请求并做子域匹配
+        """
+        self.page_num = 1
+        subdomain_encode = f'domain="{self.domain}"'.encode('utf-8')
+        query_data = base64.b64encode(subdomain_encode)
+        while 100 * self.page_num < settings.cam_records_maximum_per_domain:
+            time.sleep(self.delay)
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            query = {'email': self.email,
+                     'key': self.key,
+                     'qbase64': query_data,
+                     'page': self.page_num,
+                     'full': 'true',
+                     'size': min(1000, settings.cam_records_maximum_per_domain)}
+            resp = self.get(self.addr, query)
+            if not resp:
+                return
+            resp_json = resp.json()
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            size = resp_json.get('size')
+            if size < min(1000, settings.cam_records_maximum_per_domain):
+                break
+            self.page_num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.email, self.key):
+            return
+        self.begin()
+        self.search()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    search = FoFa(domain)
+    search.run()
+
+
+if __name__ == '__main__':
+    run('example.com')