oneforall-kjl 0.1.1__py3-none-any.whl

OneForAll/common/resolve.py

@@ -0,0 +1,173 @@
+import gc
+import json
+
+from config.log import logger
+from config import settings
+from common import utils
+
+
+def filter_subdomain(data):
+    """
+    过滤出无解析内容的子域到新的子域列表
+
+    :param list data: 待过滤的数据列表
+    :return: 符合条件的子域列表
+    """
+    logger.log('DEBUG', f'Filtering subdomains to be resolved')
+    subdomains = []
+    for infos in data:
+        if not infos.get('ip'):
+            subdomain = infos.get('subdomain')
+            if subdomain:
+                subdomains.append(subdomain)
+    return subdomains
+
+
+def update_data(data, infos):
+    """
+    更新解析结果
+
+    :param list data: 待更新的数据列表
+    :param dict infos: 子域有关结果信息
+    :return: 更新后的数据列表
+    """
+    logger.log('DEBUG', f'Updating resolved results')
+    if not infos:
+        logger.log('ALERT', f'No valid resolved result')
+        return data
+    new_data = list()
+    for index, items in enumerate(data):
+        if items.get('ip'):
+            new_data.append(items)
+            continue
+        subdomain = items.get('subdomain')
+        record = infos.get(subdomain)
+        if record:
+            items.update(record)
+            new_data.append(items)
+        else:
+            subdomain = items.get('subdomain')
+            logger.log('DEBUG', f'{subdomain} resolution has no result')
+    return new_data
+
+
+def save_db(name, data):
+    """
+    Save resolved results to database
+
+    :param str  name: table name
+    :param list data: data to be saved
+    """
+    logger.log('INFOR', f'Saving resolved results')
+    utils.save_to_db(name, data, 'resolve')
+
+
+def save_subdomains(save_path, subdomain_list):
+    logger.log('DEBUG', f'Saving resolved subdomain')
+    subdomain_data = '\n'.join(subdomain_list)
+    if not utils.save_to_file(save_path, subdomain_data):
+        logger.log('FATAL', 'Save resolved subdomain error')
+        exit(1)
+
+
+def gen_infos(data, qname, info, infos):
+    flag = False
+    cnames = list()
+    ips = list()
+    ttl = list()
+    answers = data.get('answers')
+    for answer in answers:
+        if answer.get('type') == 'A':
+            flag = True
+            name = answer.get('name')
+            cname = name[:-1].lower()  # 去除最右边的`.`点号
+            cnames.append(cname)
+            ip = answer.get('data')
+            ips.append(ip)
+            ttl.append(str(answer.get('ttl')))
+            info['resolve'] = 1
+            info['reason'] = 'OK'
+            info['cname'] = ','.join(cnames)
+            info['ip'] = ','.join(ips)
+            info['ttl'] = ','.join(ttl)
+            infos[qname] = info
+    if not flag:
+        logger.log('DEBUG', f'Resolving {qname} have not a record')
+        info['alive'] = 0
+        info['resolve'] = 0
+        info['reason'] = 'NoARecord'
+        infos[qname] = info
+    return infos
+
+
+def deal_output(output_path):
+    logger.log('INFOR', f'Processing resolved results')
+    infos = dict()  # 用来记录所有域名有关信息
+    with open(output_path) as fd:
+        for line in fd:
+            line = line.strip()
+            try:
+                items = json.loads(line)
+            except Exception as e:
+                logger.log('ERROR', e.args)
+                logger.log('ERROR', f'Error resolve line {line}, skip this line')
+                continue
+            info = dict()
+            info['resolver'] = items.get('resolver')
+            qname = items.get('name')[:-1]  # 去除最右边的`.`点号
+            status = items.get('status')
+            if status != 'NOERROR':
+                logger.log('DEBUG', f'Resolving {qname}: {status}')
+                continue
+            data = items.get('data')
+            if 'answers' not in data:
+                logger.log('DEBUG', f'Resolving {qname} have not any answers')
+                info['alive'] = 0
+                info['resolve'] = 0
+                info['reason'] = 'NoAnswer'
+                infos[qname] = info
+                continue
+            infos = gen_infos(data, qname, info, infos)
+    return infos
+
+
+def run_resolve(domain, data):
+    """
+    调用子域解析入口函数
+
+    :param str domain: 待解析的主域
+    :param list data: 待解析的子域数据列表
+    :return: 解析得到的结果列表
+    :rtype: list
+    """
+    logger.log('INFOR', f'Start resolving subdomains of {domain}')
+    subdomains = filter_subdomain(data)
+    if not subdomains:
+        return data
+
+    massdns_dir = settings.third_party_dir.joinpath('massdns')
+    result_dir = settings.result_save_dir
+    temp_dir = result_dir.joinpath('temp')
+    utils.check_dir(temp_dir)
+    massdns_path = utils.get_massdns_path(massdns_dir)
+    timestring = utils.get_timestring()
+
+    save_name = f'collected_subdomains_{domain}_{timestring}.txt'
+    save_path = temp_dir.joinpath(save_name)
+    save_subdomains(save_path, subdomains)
+    del subdomains
+    gc.collect()
+
+    output_name = f'resolved_result_{domain}_{timestring}.json'
+    output_path = temp_dir.joinpath(output_name)
+    log_path = result_dir.joinpath('massdns.log')
+    ns_path = utils.get_ns_path()
+
+    logger.log('INFOR', f'Running massdns to resolve subdomains')
+    utils.call_massdns(massdns_path, save_path, ns_path,
+                       output_path, log_path, quiet_mode=True)
+
+    infos = deal_output(output_path)
+    data = update_data(data, infos)
+    logger.log('INFOR', f'Finished resolve subdomains of {domain}')
+    return data

OneForAll/common/search.py

@@ -0,0 +1,78 @@
+from config import settings
+from common.module import Module
+
+
+class Search(Module):
+    """
+    Search base class
+    """
+    def __init__(self):
+        Module.__init__(self)
+        self.page_num = 0  # 要显示搜索起始条数
+        self.per_page_num = 50  # 每页显示搜索条数
+        self.recursive_search = settings.enable_recursive_search
+        self.recursive_times = settings.search_recursive_times
+        self.full_search = settings.enable_full_search
+
+    @staticmethod
+    def filter(domain, subdomain):
+        """
+        生成搜索过滤语句
+        使用搜索引擎支持的-site:语法过滤掉搜索页面较多的子域以发现新域
+
+        :param str domain: 域名
+        :param set subdomain: 子域名集合
+        :return: 过滤语句
+        :rtype: str
+        """
+        statements_list = []
+        subdomains_temp = set(map(lambda x: x + '.' + domain, settings.common_subnames))
+        subdomains_temp = list(subdomain.intersection(subdomains_temp))
+        for i in range(0, len(subdomains_temp), 2):  # 同时排除2个子域
+            statements_list.append(''.join(set(map(lambda s: ' -site:' + s,
+                                                   subdomains_temp[i:i + 2]))))
+        return statements_list
+
+    def match_location(self, url):
+        """
+        匹配跳转之后的url
+        针对部分搜索引擎(如百度搜索)搜索展示url时有显示不全的情况
+        此函数会向每条结果的链接发送head请求获取响应头的location值并做子域匹配
+
+        :param str url: 展示结果的url链接
+        :return: 匹配的子域
+        :rtype set
+        """
+        resp = self.head(url, check=False, allow_redirects=False)
+        if not resp:
+            return set()
+        location = resp.headers.get('location')
+        if not location:
+            return set()
+        return set(self.match_subdomains(location))
+
+    def check_subdomains(self, subdomains):
+        """
+        检查搜索出的子域结果是否满足条件
+
+        :param subdomains: 子域结果
+        :return:
+        """
+        if not subdomains:
+            # 搜索没有发现子域名则停止搜索
+            return False
+        if not self.full_search and subdomains.issubset(self.subdomains):
+            # 在全搜索过程中发现搜索出的结果有完全重复的结果就停止搜索
+            return False
+        return True
+
+    def recursive_subdomain(self):
+        # 递归搜索下一层的子域
+        # 从1开始是之前已经做过1层子域搜索了,当前实际递归层数是layer+1
+        subdomains = self.subdomains.copy()
+        for layer_num in range(1, self.recursive_times):
+            for subdomain in subdomains:
+                # 进行下一层子域搜索的限制条件
+                count = subdomain.count('.') - self.domain.count('.')
+                if count == layer_num:
+                    yield subdomain

OneForAll/common/similarity.py

@@ -0,0 +1,138 @@
+"""
+根据网页结构判断页面相似性(Determine page similarity based on HTML page structure)
+判断方法：根据网页的DOM树确定网页的模板特征向量，对模板特征向量计算网页结构相似性。
+来源地址：https://github.com/SPuerBRead/HTMLSimilarity
+计算参考: https://patents.google.com/patent/CN101694668B/zh
+"""
+from treelib import Tree
+from bs4 import BeautifulSoup
+import bs4
+
+
+class DOMTree(object):
+    def __init__(self, label, attrs):
+        self.label = label
+        self.attrs = attrs
+
+
+class HTMLParser(object):
+    def __init__(self, html):
+        self.dom_id = 1
+        self.dom_tree = Tree()
+        self.bs_html = BeautifulSoup(html, 'html.parser')
+
+    def get_dom_structure_tree(self):
+        for content in self.bs_html.contents:
+            if isinstance(content, bs4.element.Tag):
+                self.bs_html = content
+        self.recursive_descendants(self.bs_html, 1)
+        return self.dom_tree
+
+    def recursive_descendants(self, descendants, parent_id):
+        if self.dom_id == 1:
+            self.dom_tree.create_node(descendants.name, self.dom_id,
+                                      data=DOMTree(descendants.name, descendants.attrs))
+            self.dom_id = self.dom_id + 1
+        for child in descendants.contents:
+            if isinstance(child, bs4.element.Tag):
+                self.dom_tree.create_node(child.name, self.dom_id, parent_id,
+                                          data=DOMTree(child.name, child.attrs))
+                self.dom_id = self.dom_id + 1
+                self.recursive_descendants(child, self.dom_id - 1)
+
+
+class Converter(object):
+    def __init__(self, dom_tree, dimension):
+        self.dom_tree = dom_tree
+        self.node_info_list = []
+        self.dimension = dimension
+        self.initial_weight = 1
+        self.attenuation_ratio = 0.6
+        self.dom_eigenvector = {}.fromkeys(range(0, dimension), 0)
+
+    def get_eigenvector(self):
+        for node_id in range(1, self.dom_tree.size() + 1):
+            node = self.dom_tree.get_node(node_id)
+            node_feature = self.create_feature(node)
+            feature_hash = self.feature_hash(node_feature)
+            node_weight = self.calculate_weight(node, node_id, feature_hash)
+            self.construct_eigenvector(feature_hash, node_weight)
+        return self.dom_eigenvector
+
+    @staticmethod
+    def create_feature(node):
+        node_attr_list = []
+        node_feature = node.data.label + '|'
+        for attr in node.data.attrs.keys():
+            node_attr_list.append(attr + ':' + str(node.data.attrs[attr]))
+        node_feature += '|'.join(node_attr_list)
+        return node_feature
+
+    @staticmethod
+    def feature_hash(node_feature):
+        return abs(hash(node_feature)) % (10 ** 8)
+
+    def calculate_weight(self, node, node_id, feature_hash):
+        brother_node_count = 0
+        depth = self.dom_tree.depth(node)
+        for brother_node in self.dom_tree.siblings(node_id):
+            brother_node_feature = self.create_feature(brother_node)
+            brother_node_feature_hash = self.feature_hash(brother_node_feature)
+            if brother_node_feature_hash == feature_hash:
+                brother_node_count = brother_node_count + 1
+        if brother_node_count:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth \
+                          * self.attenuation_ratio ** brother_node_count
+        else:
+            node_weight = self.initial_weight * self.attenuation_ratio ** depth
+        return node_weight
+
+    def construct_eigenvector(self, feature_hash, node_weight):
+        feature_hash = feature_hash % self.dimension
+        self.dom_eigenvector[feature_hash] += node_weight
+
+
+def calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension):
+    a, b = 0, 0
+    for i in range(dimension):
+        a += dom1_eigenvector[i]-dom2_eigenvector[i]
+        if dom1_eigenvector[i] and dom2_eigenvector[i]:
+            b += dom1_eigenvector[i] + dom2_eigenvector[i]
+    pseudodistance = abs(a)/b
+    return pseudodistance
+
+
+def get_pseudodistance(html_doc1, html_doc2, dimension=5000):
+    """
+    获取html文档结构相似度
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 伪距离
+    """
+    hp1 = HTMLParser(html_doc1)
+    html_doc1_dom_tree = hp1.get_dom_structure_tree()
+    hp2 = HTMLParser(html_doc2)
+    html_doc2_dom_tree = hp2.get_dom_structure_tree()
+    converter = Converter(html_doc1_dom_tree, dimension)
+    dom1_eigenvector = converter.get_eigenvector()
+    converter = Converter(html_doc2_dom_tree, dimension)
+    dom2_eigenvector = converter.get_eigenvector()
+    return calc_pseudodistance(dom1_eigenvector, dom2_eigenvector, dimension)
+
+
+def is_similar(html_doc1, html_doc2, dimension=5000):
+    """
+    根据计算出的伪距离来判断是否html页面结构相似
+
+    :param str html_doc1: html文档
+    :param str html_doc2: html文档
+    :param int dimension: 降维后的维数
+    :return 是否相似（伪距离value<0.2时相似，value>0.2时不相似）
+    """
+    value = get_pseudodistance(html_doc1, html_doc2, dimension)
+    if value > 0.2:
+        return False
+    else:
+        return True

OneForAll/common/tablib/format.py

@@ -0,0 +1,89 @@
+import decimal
+import json
+import csv
+from io import StringIO
+from uuid import UUID
+
+""" Tablib - formats
+"""
+from collections import OrderedDict
+
+
+class Registry:
+    _formats = OrderedDict()
+
+    def register(self, key, format_or_path):
+        # Create Databook.<format> read or read/write properties
+
+        # Create Dataset.<format> read or read/write properties,
+        # and Dataset.get_<format>/set_<format> methods.
+        self._formats[key] = format_or_path
+
+    def register_builtins(self):
+        # Registration ordering matters for autodetection.
+        self.register('csv', CSVFormat())
+        self.register('json', JSONFormat())
+
+    def get_format(self, key):
+        if key not in self._formats:
+            raise Exception("OneForAll has no format '%s'." % key)
+        return self._formats[key]
+
+
+registry = Registry()
+
+
+def serialize_objects_handler(obj):
+    if isinstance(obj, (decimal.Decimal, UUID)):
+        return str(obj)
+    elif hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    else:
+        return obj
+
+
+"""
+Tablib - JSON Support
+"""
+
+
+class JSONFormat:
+    title = 'json'
+    extensions = ('json',)
+
+    @classmethod
+    def export_set(cls, dataset):
+        """Returns JSON representation of Dataset."""
+        return json.dumps(dataset.dict, default=serialize_objects_handler)
+
+
+""" Tablib - CSV Support.
+"""
+
+
+class CSVFormat:
+    title = 'csv'
+    extensions = ('csv',)
+
+    DEFAULT_DELIMITER = ','
+
+    @classmethod
+    def export_stream_set(cls, dataset, **kwargs):
+        """Returns CSV representation of Dataset as file-like."""
+        stream = StringIO()
+
+        kwargs.setdefault('delimiter', cls.DEFAULT_DELIMITER)
+
+        _csv = csv.writer(stream, **kwargs)
+
+        for row in dataset._package(dicts=False):
+            _csv.writerow(row)
+
+        stream.seek(0)
+        return stream
+
+    @classmethod
+    def export_set(cls, dataset, **kwargs):
+        """Returns CSV representation of Dataset."""
+        stream = cls.export_stream_set(dataset, **kwargs)
+        return stream.getvalue()