oneforall-kjl 0.1.1__py3-none-any.whl

OneForAll/modules/datasets/ip138.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class IP138(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'IP138Query'
+        self.addr = 'https://site.ip138.com/{domain}/domain.htm'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        self.addr = self.addr.format(domain=self.domain)
+        resp = self.get(self.addr)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IP138(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/ipv4info_api.py

@@ -0,0 +1,73 @@
+from config import settings
+from common.query import Query
+from config.log import logger
+
+
+class IPv4InfoAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'IPv4InfoAPIQuery'
+        self.addr = ' http://ipv4info.com/api_v1/'
+        self.api = settings.ipv4info_api_key
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        page = 0
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            params = {'type': 'SUBDOMAINS', 'key': self.api,
+                      'value': self.domain, 'page': page}
+            resp = self.get(self.addr, params)
+            if not resp:
+                return
+            if resp.status_code != 200:
+                break  # 请求不正常通常网络是有问题，不再继续请求下去
+            try:
+                json = resp.json()
+            except Exception as e:
+                logger.log('DEBUG', e.args)
+                break
+            subdomains = self.match_subdomains(str(json))
+            if not subdomains:
+                break
+            self.subdomains.update(subdomains)
+            # 不直接使用subdomains是因为可能里面会出现不符合标准的子域名
+            subdomains = json.get('Subdomains')
+            if subdomains and len(subdomains) < 300:
+                # ipv4info子域查询接口每次最多返回300个 用来判断是否还有下一页
+                break
+            page += 1
+            if page >= 50:  # ipv4info子域查询接口最多允许查询50页
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = IPv4InfoAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/netcraft.py

@@ -0,0 +1,66 @@
+import hashlib
+import re
+import time
+from urllib import parse
+
+from common.query import Query
+
+
+class NetCraft(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'NetCraftQuery'
+        self.addr = 'https://searchdns.netcraft.com/?restriction=site+contains&position=limited'
+        self.page_num = 1
+        self.per_page_num = 20
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()  # NetCraft会检查User-Agent
+        self.proxy = self.get_proxy(self.source)
+        last = ''
+        while True:
+            time.sleep(self.delay)
+            self.proxy = self.get_proxy(self.source)
+            params = {'host': '*.' + self.domain,
+                      'from': self.page_num}
+            resp = self.get(self.addr + last, params)
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 搜索没有发现子域名则停止搜索
+                break
+            self.subdomains.update(subdomains)
+            if 'Next Page' not in resp.text:  # 搜索页面没有出现下一页时停止搜索
+                break
+            last = re.search(r'&last=.*' + self.domain, resp.text).group(0)
+            self.page_num += self.per_page_num
+            if self.page_num > 500:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = NetCraft(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/passivedns_api.py

@@ -0,0 +1,51 @@
+from config import settings
+from common.query import Query
+
+
+class PassiveDnsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'PassiveDnsQuery'
+        self.addr = settings.passivedns_api_addr or 'http://api.passivedns.cn'
+        self.token = settings.passivedns_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.header.update({'X-AuthToken': self.token})
+        self.proxy = self.get_proxy(self.source)
+        url = self.addr + '/flint/rrset/*.' + self.domain
+        resp = self.get(url)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if 'api.passivedns.cn' in self.addr:
+            if not self.have_api(self.token):
+                return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = PassiveDnsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/qianxun.py

@@ -0,0 +1,61 @@
+from common.query import Query
+
+
+class QianXun(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Query'
+        self.source = 'QianXunQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+
+        num = 1
+        while True:
+            data = {'ecmsfrom': '',
+                    'show': '',
+                    'num': '',
+                    'classid': '0',
+                    'keywords': self.domain}
+            url = f'https://www.dnsscan.cn/dns.html?' \
+                  f'keywords={self.domain}&page={num}'
+            resp = self.post(url, data)
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 没有发现子域名则停止查询
+                break
+            self.subdomains.update(subdomains)
+            if '<div id="page" class="pagelist">' not in resp.text:
+                break
+            if '<li class="disabled"><span>&raquo;</span></li>' in resp.text:
+                break
+            num += 1
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = QianXun(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/rapiddns.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class RapidDNS(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'RapidDNSQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        url = f'http://rapiddns.io/subdomain/{self.domain}'
+        params = {'full': '1'}
+        resp = self.get(url, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = RapidDNS(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/riddler.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Riddler(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'RiddlerQuery'
+        self.addr = 'https://riddler.io/search'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'q': 'pld:' + self.domain}
+        resp = self.get(self.addr, params)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Riddler(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/robtex.py

@@ -0,0 +1,58 @@
+import json
+import time
+
+from common.query import Query
+
+
+class Robtex(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = "RobtexQuery"
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        base_addr = 'https://freeapi.robtex.com/pdns'
+        url = f'{base_addr}/forward/{self.domain}'
+        resp = self.get(url)
+        if not resp:
+            return
+        text_list = resp.text.splitlines()
+        for item in text_list:
+            record = json.loads(item)
+            if record.get('rrtype') in ['A', 'AAAA']:
+                time.sleep(self.delay)  # Robtex有查询频率限制
+                ip = record.get('rrdata')
+                url = f'{base_addr}/reverse/{ip}'
+                resp = self.get(url)
+                self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Robtex(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('google.com')

OneForAll/modules/datasets/securitytrails_api.py

@@ -0,0 +1,56 @@
+from config import settings
+from common.query import Query
+
+
+class SecurityTrailsAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'SecurityTrailsAPIQuery'
+        self.addr = 'https://api.securitytrails.com/v1/domain/'
+        self.api = settings.securitytrails_api
+        self.delay = 2  # SecurityTrails查询时延至少2秒
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        params = {'apikey': self.api}
+        url = f'{self.addr}{self.domain}/subdomains'
+        resp = self.get(url, params)
+        if not resp:
+            return
+        prefixs = resp.json()['subdomains']
+        subdomains = [f'{prefix}.{self.domain}' for prefix in prefixs]
+        if subdomains:
+            self.subdomains.update(subdomains)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.api):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SecurityTrailsAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/sitedossier.py

@@ -0,0 +1,57 @@
+
+from common.query import Query
+
+
+class SiteDossier(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'SiteDossierQuery'
+        self.addr = 'http://www.sitedossier.com/parentdomain/'
+        self.delay = 2
+        self.page_num = 1
+        self.per_page_num = 100
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        while True:
+            self.header = self.get_header()
+            self.proxy = self.get_proxy(self.source)
+            url = f'{self.addr}{self.domain}/{self.page_num}'
+            resp = self.get(url)
+            subdomains = self.match_subdomains(resp)
+            if not subdomains:  # 没有发现子域名则停止查询
+                break
+            self.subdomains.update(subdomains)
+            # 搜索页面没有出现下一页时停止搜索
+            if 'Show next 100 items' not in resp.text:
+                break
+            self.page_num += self.per_page_num
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SiteDossier(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/spyse_api.py

@@ -0,0 +1,62 @@
+from config import settings
+from common.query import Query
+
+
+class SpyseAPI(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'SpyseAPIQuery'
+        self.token = settings.spyse_api_token
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        limit = 100
+        offset = 0
+        while True:
+            self.header = self.get_header()
+            self.header.update({'Authorization': 'Bearer ' + self.token})
+            self.proxy = self.get_proxy(self.source)
+            addr = 'https://api.spyse.com/v3/data/domain/subdomain'
+            params = {'domain': self.domain, 'offset': offset, 'limit': limit}
+            resp = self.get(addr, params)
+            if not resp:
+                return
+            json = resp.json()
+            subdomains = self.match_subdomains(str(json))
+            if not subdomains:  # 没有发现子域名则停止查询
+                break
+            self.subdomains.update(subdomains)
+            offset += limit
+            if len(json.get('data').get('items')) < limit:
+                break
+
+    def run(self):
+        """
+        类执行入口
+        """
+        if not self.have_api(self.token):
+            return
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = SpyseAPI(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')

OneForAll/modules/datasets/sublist3r.py

@@ -0,0 +1,45 @@
+from common.query import Query
+
+
+class Sublist3r(Query):
+    def __init__(self, domain):
+        Query.__init__(self)
+        self.domain = domain
+        self.module = 'Dataset'
+        self.source = 'Sublist3rQuery'
+
+    def query(self):
+        """
+        向接口查询子域并做子域匹配
+        """
+        self.header = self.get_header()
+        self.proxy = self.get_proxy(self.source)
+        addr = 'https://api.sublist3r.com/search.php'
+        param = {'domain': self.domain}
+        resp = self.get(addr, param)
+        self.subdomains = self.collect_subdomains(resp)
+
+    def run(self):
+        """
+        类执行入口
+        """
+        self.begin()
+        self.query()
+        self.finish()
+        self.save_json()
+        self.gen_result()
+        self.save_db()
+
+
+def run(domain):
+    """
+    类统一调用入口
+
+    :param str domain: 域名
+    """
+    query = Sublist3r(domain)
+    query.run()
+
+
+if __name__ == '__main__':
+    run('example.com')