crawlo 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

crawlo/utils/tools.py

@@ -0,0 +1,303 @@
+import json
+import re
+from pprint import pformat
+from datetime import date, time, datetime
+
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def make_insert_sql(
+        table, data, auto_update=False, update_columns=(), insert_ignore=False
+):
+    """
+    @summary: 适用于mysql
+    ---------
+    @param table:
+    @param data: 表数据 json格式
+    @param auto_update: 使用的是replace into， 为完全覆盖已存在的数据
+    @param update_columns: 需要更新的列 默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
+    @param insert_ignore: 数据存在忽略
+    ---------
+    @result:
+    """
+
+    keys = ["`{}`".format(key) for key in data.keys()]
+    keys = list2str(keys).replace("'", "")
+
+    values = [format_sql_value(value) for value in data.values()]
+    values = list2str(values)
+
+    if update_columns:
+        if not isinstance(update_columns, (tuple, list)):
+            update_columns = [update_columns]
+        update_columns_ = ", ".join(
+            ["{key}=values({key})".format(key=key) for key in update_columns]
+        )
+        sql = (
+                "insert%s into `{table}` {keys} values {values} on duplicate key update %s"
+                % (" ignore" if insert_ignore else "", update_columns_)
+        )
+
+    elif auto_update:
+        sql = "replace into `{table}` {keys} values {values}"
+    else:
+        sql = "insert%s into `{table}` {keys} values {values}" % (
+            " ignore" if insert_ignore else ""
+        )
+
+    sql = sql.format(table=table, keys=keys, values=values).replace("None", "null")
+    return sql
+
+
+def make_update_sql(table, data, condition):
+    """
+    @summary: 适用于mysql， oracle数据库时间需要to_date 处理（TODO）
+    ---------
+    @param table:
+    @param data: 表数据 json格式
+    @param condition: where 条件
+    ---------
+    @result:
+    """
+    key_values = []
+
+    for key, value in data.items():
+        value = format_sql_value(value)
+        if isinstance(value, str):
+            key_values.append("`{}`={}".format(key, repr(value)))
+        elif value is None:
+            key_values.append("`{}`={}".format(key, "null"))
+        else:
+            key_values.append("`{}`={}".format(key, value))
+
+    key_values = ", ".join(key_values)
+
+    sql = "update `{table}` set {key_values} where {condition}"
+    sql = sql.format(table=table, key_values=key_values, condition=condition)
+    return sql
+
+
+def make_batch_sql(
+        table, datas, auto_update=False, update_columns=(), update_columns_value=()
+):
+    """
+    @summary: 生成批量的SQL
+    ---------
+    @param table:
+    @param datas: 表数据 [{...}]
+    @param auto_update: 使用的是replace into，为完全覆盖已存在的数据
+    @param update_columns: 需要更新的列，默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
+    @param update_columns_value: 需要更新的列的值，默认为datas里边对应的值，注意如果值为字符串类型需要主动加单引号，如 update_columns_value=("'test'",)
+    ---------
+    @result:
+    """
+    if not datas:
+        return
+
+    keys = list(set([key for data in datas for key in data]))
+    # values_placeholder = ["%s"] * len(keys)
+    values = []
+    for data in datas:
+        # 检查 data 是否是字典类型
+        if not isinstance(data, dict):
+            # 如果 data 不是字典，记录错误日志并打印 data 的内容和类型
+            # logger.error(f"期望的数据类型是字典，但实际得到: {data} (类型: {type(data)})")
+            continue  # 跳过非字典类型的 data，继续处理下一个数据
+
+        value = []
+        for key in keys:
+            # 从字典中获取当前 key 对应的值
+            current_data = data.get(key)
+            try:
+                # 对值进行格式化处理
+                current_data = format_sql_value(current_data)
+                value.append(current_data)  # 将处理后的值添加到列表中
+            except Exception as e:
+                # 如果格式化失败，记录错误日志
+                logger.error(f"{key}: {current_data} (类型: {type(current_data)}) -> {e}")
+
+        # 将处理后的值列表添加到 values 中
+        values.append(value)
+    keys_str = ", ".join(["`{}`".format(key) for key in keys])
+    placeholders_str = ", ".join(["%s"] * len(keys))
+
+    if update_columns:
+        if not isinstance(update_columns, (tuple, list)):
+            update_columns = [update_columns]
+        if update_columns_value:
+            update_columns_ = ", ".join(
+                [
+                    "`{key}`={value}".format(key=key, value=value)
+                    for key, value in zip(update_columns, update_columns_value)
+                ]
+            )
+        else:
+            # 修改这里，使用 VALUES() 函数来引用插入的值
+            update_columns_ = ", ".join(
+                ["`{key}`=VALUES(`{key}`)".format(key=key) for key in update_columns]
+            )
+
+        sql = f"INSERT INTO `{table}` ({keys_str}) VALUES ({placeholders_str}) ON DUPLICATE KEY UPDATE {update_columns_}"
+    elif auto_update:
+        sql = "REPLACE INTO `{table}` ({keys}) VALUES ({values_placeholder})".format(
+            table=table, keys=keys_str, values_placeholder=placeholders_str
+        )
+    else:
+        sql = "INSERT IGNORE INTO `{table}` ({keys}) VALUES ({values_placeholder})".format(
+            table=table, keys=keys_str, values_placeholder=placeholders_str
+        )
+    return sql, values
+
+
+def format_sql_value(value):
+    """
+    格式化 SQL 值
+    """
+    if value is None:
+        return None  # 处理 NULL 值
+
+    # 确保处理字符串
+    if isinstance(value, str):
+        return value.strip()  # 去除首尾空格
+
+    # 处理列表或元组类型
+    elif isinstance(value, (list, tuple)):
+        try:
+            return dumps_json(value)  # 将其转为 JSON 字符串
+        except Exception as e:
+            raise ValueError(f"Failed to serialize list/tuple to JSON: {value}, error: {e}")
+
+    # 处理字典类型
+    elif isinstance(value, dict):
+        try:
+            return dumps_json(value)  # 将其转为 JSON 字符串
+        except Exception as e:
+            raise ValueError(f"Failed to serialize dict to JSON: {value}, error: {e}")
+
+    # 处理布尔类型
+    elif isinstance(value, bool):
+        return int(value)  # 转为整数
+
+    # 确保数值类型优先匹配
+    elif isinstance(value, (int, float)):
+        return value  # 返回数值
+
+    # 处理日期、时间类型
+    elif isinstance(value, (date, time, datetime)):
+        return str(value)  # 转换为字符串表示
+
+    # 如果遇到无法处理的类型，抛出异常
+    else:
+        raise TypeError(f"Unsupported value type: {type(value)}, value: {value}")
+
+
+
+
+def list2str(datas):
+    """
+    列表转字符串
+    :param datas: [1, 2]
+    :return: (1, 2)
+    """
+    data_str = str(tuple(datas))
+    data_str = re.sub(r",\)$", ")", data_str)
+    return data_str
+
+_REGEXPS = {}
+
+def get_info(html, regexps, allow_repeat=True, fetch_one=False, split=None):
+    regexps = isinstance(regexps, str) and [regexps] or regexps
+
+    infos = []
+    for regex in regexps:
+        if regex == "":
+            continue
+
+        if regex not in _REGEXPS.keys():
+            _REGEXPS[regex] = re.compile(regex, re.S)
+
+        if fetch_one:
+            infos = _REGEXPS[regex].search(html)
+            if infos:
+                infos = infos.groups()
+            else:
+                continue
+        else:
+            infos = _REGEXPS[regex].findall(str(html))
+
+        if len(infos) > 0:
+            break
+
+    if fetch_one:
+        infos = infos if infos else ("",)
+        return infos if len(infos) > 1 else infos[0]
+    else:
+        infos = allow_repeat and infos or sorted(set(infos), key=infos.index)
+        infos = split.join(infos) if split else infos
+        return infos
+
+
+def get_json(json_str):
+    """
+    @summary: 取json对象
+    ---------
+    @param json_str: json格式的字符串
+    ---------
+    @result: 返回json对象
+    """
+
+    try:
+        return json.loads(json_str) if json_str else {}
+    except Exception as e1:
+        try:
+            json_str = json_str.strip()
+            json_str = json_str.replace("'", '"')
+            keys = get_info(json_str, r"(\w+):")
+            for key in keys:
+                json_str = json_str.replace(key, '"%s"' % key)
+
+            return json.loads(json_str) if json_str else {}
+
+        except Exception as e2:
+            logger.error(
+                """
+                e1: %s
+                format json_str: %s
+                e2: %s
+                """
+                % (e1, json_str, e2)
+            )
+
+        return {}
+
+
+def dumps_json(data, indent=4, sort_keys=False):
+    """
+    @summary: 格式化json 用于打印
+    ---------
+    @param data: json格式的字符串或json对象
+    @param indent:
+    @param sort_keys:
+    ---------
+    @result: 格式化后的字符串
+    """
+    try:
+        if isinstance(data, str):
+            data = get_json(data)
+
+        data = json.dumps(
+            data,
+            ensure_ascii=False,
+            indent=indent,
+            skipkeys=True,
+            sort_keys=sort_keys,
+            default=str,
+        )
+
+    except Exception as e:
+        data = pformat(data)
+
+    return data

crawlo/utils/url.py

@@ -0,0 +1,40 @@
+from urllib.parse import urldefrag
+from w3lib.url import add_or_replace_parameter
+
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
+
+
+if __name__ == '__main__':
+    f = escape_ajax('http://example.com/page#!')
+    print(f)

{crawlo-1.0.0.dist-info → crawlo-1.0.2.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.0
-Summary: feapder是一款支持异步的python爬虫框架
+Version: 1.0.2
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder
 Author-email: crawlo@qq.com
@@ -11,13 +11,26 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: aiohttp>=3.12.6
-Requires-Dist: httpx>=0.28.1
-Requires-Dist: DBUtils>=2.0
-Requires-Dist: parsel>=1.10.0
-Requires-Dist: pymysql>=1.1.1
-Requires-Dist: ujson>=5.10.0
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
 Provides-Extra: render
 Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
 Requires-Dist: playwright; extra == "render"
@@ -30,7 +43,6 @@ Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
 Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
 Requires-Dist: playwright; extra == "all"
 Requires-Dist: selenium>=3.141.0; extra == "all"
-Dynamic: license-file
 
 # Crawlo
-异步通用爬虫框架
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.2.dist-info/RECORD

@@ -0,0 +1,68 @@
+crawlo/__init__.py,sha256=BoRtaB19VFlByP3JKzXQbmg4Jb6i6yVnpTR3jCSrCig,208
+crawlo/__version__.py,sha256=FlR3yFykEvKzEITpEQq_qx6Uq29lYlhxcnSHeRP1LgI,23
+crawlo/crawler.py,sha256=XLiDyFyoiJr8BzhLbCB15zbccVHQiiLGDM1zFUkdrAI,8544
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=trxM2c0jw50QsGSoFAKC2RrKpapOFHQDq0wQuLWqmKE,980
+crawlo/stats_collector.py,sha256=jhAW8k0SzjqelkpiWpfGmMw2DBkgTjpwnObqTNDOp6A,1286
+crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
+crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
+crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
+crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
+crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
+crawlo/core/scheduler.py,sha256=ZwPoU_QRjs9wwrxdt-MGPwsSmKhvvhgmcnBllDrXnhg,2014
+crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
+crawlo/downloader/aiohttp_downloader.py,sha256=4C2BDloKzwss16kfD7tH0WPugPbSSFxl-5-_DLWB0vM,3676
+crawlo/downloader/httpx_downloader.py,sha256=ra6Ae_lv8pNyvLzPQYBgTNuBdMVBYi86kNt2OdZlcSo,1704
+crawlo/downloader/playwright_downloader.py,sha256=mEGlSd6A6sN0Wyq-TDkownIElOgxnwVfY3rS5wtLoYY,6726
+crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
+crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
+crawlo/extension/log_stats.py,sha256=-V7ipdIfYMQdp1ZDc4kvNEAIHIR74U1ZHV5FhlLyGGU,1786
+crawlo/filters/__init__.py,sha256=9fJQRVkxWWPChajYbAGe1O6UYB639xWt0hiLUGBs4hQ,1014
+crawlo/filters/aioredis_filter.py,sha256=H_HAFfE9mHnPrzQcjyXXCseQ77iN4nT9a3lukHiu8M8,4874
+crawlo/filters/memory_filter.py,sha256=pk2o0kbX0zrGLJ6poKhSa-cfOmsp68fA65hXEGQCQ5M,6895
+crawlo/filters/redis_filter.py,sha256=m1nRsf_3slnWSb80RtTEURi5kwjdI0xotoFxnscx974,4211
+crawlo/items/__init__.py,sha256=o5BSpS1Byivr-bpdfFgc9GCoGi8ThNuPJiTW7lz85-I,2125
+crawlo/items/items.py,sha256=myOOjWaSByKW0r8SxIQ0bxS3PXjUDE1c-Pe38z6fSF8,4108
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
+crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
+crawlo/middleware/middleware_manager.py,sha256=T4axTY89Z0BOwaWDWcUTABeDNTvyPFiyrbwj-H4sbSA,6629
+crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
+crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
+crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
+crawlo/middleware/retry.py,sha256=9SnE7l3Nhh143AqCiL0gfE6dl_gF1Kr6CjoNxvMH_Ps,3465
+crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
+crawlo/network/request.py,sha256=yUAL6oecm1TniD9dsBmRedEaEHzh3rtr_6p3dMK2EfQ,8195
+crawlo/network/response.py,sha256=amnLEExKq11dLh6m_YdSqdKLl0srh1eeY5uPz15fwyo,6055
+crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
+crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
+crawlo/pipelines/mongo_pipeline.py,sha256=Yr48D0T61-_Y-EpgWXf7BUn9w8e-Pj5P07QDSPZ0pYU,4558
+crawlo/pipelines/mysql_batch_pipline.py,sha256=7KXd0IUV0h3IViD8R0iruyWv5XdZR1pANB8EY9z6iMI,5022
+crawlo/pipelines/mysql_pipeline.py,sha256=TzyaBg1oBj9pgzuVUrb5VVtkiwZH6gqP_6IYM2QCkLs,8052
+crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7aARE5W3k,2174
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=JS1QKYe7jkdFlOjqZ-eOeOcVS3AXCZynoNH95GuEnds,2556
+crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
+crawlo/spider/__init__.py,sha256=1tmKkr2-oJi0w9r2ho9nn6Z_VDn18pjXHXU0Hv2eheY,941
+crawlo/templates/item_template.tmpl,sha256=0bGFnlwJRqstxMNEj1H_pEICybwoueRhs31QaDPXrS0,372
+crawlo/templates/spider_template.tmpl,sha256=JzphuA87Yl_F1xR9zOIi_ZSazyT8eSNPxYYPMv3Uiko,835
+crawlo/templates/project_template/main.py,sha256=BcCP294ycCPsHi_AMN7OAJtcrLvQdf91meH93PqbQgs,626
+crawlo/templates/project_template/setting.py,sha256=Ce4nMbrdhL1ioRdTcB0vV_vK_50cfnwVqSvt49QsNkA,9395
+crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
+crawlo/utils/concurrency_manager.py,sha256=o-_cfeUHdlBOM3eAXF857MtekSrRcVTBJ2jWZvY6weQ,5230
+crawlo/utils/date_tools.py,sha256=Y7pnGNn1-5vkiHtydAgmQ-qR3pSO30k5WEYigOPifPQ,5496
+crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
+crawlo/utils/log.py,sha256=LU0J3boPCL-Kynx3wR_CAryRgScNmPPn4pBitLrrsX4,1028
+crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
+crawlo/utils/project.py,sha256=FfBaMfxcau4yL59O-DfD7FAii8k6gXWQmQ1YU6aaUCE,1544
+crawlo/utils/request.py,sha256=BEBtxwejvQw5euEiSclHCbqkNcBwUMY7KPGGkvj8BjE,2936
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo/utils/tools.py,sha256=U7xGKgXe2PmMyvNxZ1vlJYcv4Ei1WhIBMf8XcJZ7XCY,9764
+crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
+crawlo-1.0.2.dist-info/METADATA,sha256=kzNv4kckIbSZCADon5g2Ik5QJhubNxWra0-aXX6bMug,1784
+crawlo-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.2.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
+crawlo-1.0.2.dist-info/top_level.txt,sha256=Dwuv-Y1aGSJD3mjFrCdNGQ8EHroMj7RgVcxDdcczx4k,7
+crawlo-1.0.2.dist-info/RECORD,,

crawlo-1.0.0.dist-info/RECORD

@@ -1,59 +0,0 @@
-crawlo/__init__.py,sha256=yzg6DkuCNkz27Tn5YDdaboZ1Ah96FwV6eGNEDO48lxA,170
-crawlo/__version__.py,sha256=j4YpqEiBQsl3obE5LH55xbuejIMjUwUNp-k4JXOjEFU,23
-crawlo/crawler.py,sha256=lIx3-_TKXdCDhpfw4lYcizpyoclapjAHfGchBJ9DmTU,3679
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=trxM2c0jw50QsGSoFAKC2RrKpapOFHQDq0wQuLWqmKE,980
-crawlo/stats_collector.py,sha256=jhAW8k0SzjqelkpiWpfGmMw2DBkgTjpwnObqTNDOp6A,1286
-crawlo/subscriber.py,sha256=4stxeXqNK6RB7oqo0wKJdUw2Ym3b6UBMhZ4pRrjSMEU,1001
-crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
-crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
-crawlo/core/engine.py,sha256=mQfzY4Bm0Ysb5bPjoH3lMVFbn9WJuQ1UjBTReuOljzQ,5873
-crawlo/core/processor.py,sha256=5RQPfffE8AnKnuajL9J2oc7QAeoSjATxwdxRU3_EoKM,1183
-crawlo/core/scheduler.py,sha256=2z8qzZmj6zn9w5pE1yux6Vi7fDMq8Cto6n9Wyyo_DP8,1074
-crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
-crawlo/downloader/aiohttp_downloader.py,sha256=4C2BDloKzwss16kfD7tH0WPugPbSSFxl-5-_DLWB0vM,3676
-crawlo/downloader/httpx_downloader.py,sha256=ra6Ae_lv8pNyvLzPQYBgTNuBdMVBYi86kNt2OdZlcSo,1704
-crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
-crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
-crawlo/extension/log_stats.py,sha256=ZIIB6WKvYSxDIw7Mr52QkOSiX-tMGWU8Ifpd1EXiTUY,1677
-crawlo/items/__init__.py,sha256=Ir04T486XuJJ-OCItxqO-78Jyt7B8JGLBsrgCO605PU,535
-crawlo/items/items.py,sha256=7w0uojKu1GzY5SDfrlJtAhqVv6alHQjwkBkthSoJ-04,2856
-crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
-crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
-crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
-crawlo/middleware/middleware_manager.py,sha256=T4axTY89Z0BOwaWDWcUTABeDNTvyPFiyrbwj-H4sbSA,6629
-crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
-crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
-crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
-crawlo/middleware/retry.py,sha256=sCRO5uh9cG8Ui3PODJEg_6PlxDD_lAoopaF2P7u6bUo,3148
-crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
-crawlo/network/request.py,sha256=YokEk6fEBQF2ii-Oe18ZzVc262Ih6KcUZPLOpxcs6Mg,1474
-crawlo/network/response.py,sha256=QikuOCgxS5yMh4Uh0QuMpqWfyR78vossCs-Va-sQ2YE,2993
-crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
-crawlo/pipelines/console_pipeline.py,sha256=1zorg1xrIiCctiTkd38jm7Q-miKMnRFnJBjv7Tg0gyo,537
-crawlo/pipelines/mongo_pipeline.py,sha256=DfXD10Ee-3q685EdS6du3Nn-f0EaBKJgdgsVlpcYhd8,67
-crawlo/pipelines/mysql_pipeline.py,sha256=QAZToJAmy3xyTLOh7DkwCfxMDWQTLiTOLdfvZhkMP6E,67
-crawlo/pipelines/pipeline_manager.py,sha256=FvpMuHwB5DWGzPsy1mZFdcyxzPN2mVL2aivUWlzrHx4,2183
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=4J8Kzc3dk09ogy5_FeaGA1KHUzHPMYg9LFIVFIrhuXY,979
-crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
-crawlo/spider/__init__.py,sha256=9haJYfxX4Gx2iV3KK5Fry2fO9R-gEmAkR_pQjZTIErE,905
-crawlo/templates/item_template.tmpl,sha256=0bGFnlwJRqstxMNEj1H_pEICybwoueRhs31QaDPXrS0,372
-crawlo/templates/spider_template.tmpl,sha256=JzphuA87Yl_F1xR9zOIi_ZSazyT8eSNPxYYPMv3Uiko,835
-crawlo/templates/project_template/main.py,sha256=BcCP294ycCPsHi_AMN7OAJtcrLvQdf91meH93PqbQgs,626
-crawlo/templates/project_template/setting.py,sha256=Ce4nMbrdhL1ioRdTcB0vV_vK_50cfnwVqSvt49QsNkA,9395
-crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
-crawlo/utils/date_tools.py,sha256=bW3j9LxxuAUwykzuzaOYR3iY8U0g7wBPZL1ELf4lXVo,482
-crawlo/utils/func_tools.py,sha256=sWnoa2SDAMK2NBdoIO2o3Ew6SqeJXcSNEzmWZUJcE6o,616
-crawlo/utils/log.py,sha256=LU0J3boPCL-Kynx3wR_CAryRgScNmPPn4pBitLrrsX4,1028
-crawlo/utils/pqueue.py,sha256=W1n23t9roVvDB84jQ4j7sk_Z4FoD8q3LbRRbH4B6Ej4,446
-crawlo/utils/project.py,sha256=JHMV7SLDWMaPCwuwVY9KQqvu2VT1ZlXEQvpIzzNaSeo,1506
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo-1.0.0.dist-info/licenses/LICENSE,sha256=f96mrub4oLJnuMcNwMjHWMksci5brQ2a2Fu0R7YEgnk,1125
-crawlo-1.0.0.dist-info/METADATA,sha256=wS6yr3avxrdzm4x4WPN7fBKgO5jobpQ8UpvrCUC1aQw,1266
-crawlo-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.0.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
-crawlo-1.0.0.dist-info/top_level.txt,sha256=Dwuv-Y1aGSJD3mjFrCdNGQ8EHroMj7RgVcxDdcczx4k,7
-crawlo-1.0.0.dist-info/RECORD,,

crawlo-1.0.0.dist-info/licenses/LICENSE

@@ -1,23 +0,0 @@
-MIT License
-
-Modifications:
-
-Copyright (c) 2020 crawl-coder <2251018029@qq.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.