crawlo 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

crawlo/utils/tools.py

@@ -1,303 +1,5 @@
-import json
-import re
-from pprint import pformat
-from datetime import date, time, datetime
-
-from crawlo.utils.log import get_logger
-
-
-logger = get_logger(__name__)
-
-
-def make_insert_sql(
-        table, data, auto_update=False, update_columns=(), insert_ignore=False
-):
-    """
-    @summary: 适用于mysql
-    ---------
-    @param table:
-    @param data: 表数据 json格式
-    @param auto_update: 使用的是replace into， 为完全覆盖已存在的数据
-    @param update_columns: 需要更新的列 默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
-    @param insert_ignore: 数据存在忽略
-    ---------
-    @result:
-    """
-
-    keys = ["`{}`".format(key) for key in data.keys()]
-    keys = list2str(keys).replace("'", "")
-
-    values = [format_sql_value(value) for value in data.values()]
-    values = list2str(values)
-
-    if update_columns:
-        if not isinstance(update_columns, (tuple, list)):
-            update_columns = [update_columns]
-        update_columns_ = ", ".join(
-            ["{key}=values({key})".format(key=key) for key in update_columns]
-        )
-        sql = (
-                "insert%s into `{table}` {keys} values {values} on duplicate key update %s"
-                % (" ignore" if insert_ignore else "", update_columns_)
-        )
-
-    elif auto_update:
-        sql = "replace into `{table}` {keys} values {values}"
-    else:
-        sql = "insert%s into `{table}` {keys} values {values}" % (
-            " ignore" if insert_ignore else ""
-        )
-
-    sql = sql.format(table=table, keys=keys, values=values).replace("None", "null")
-    return sql
-
-
-def make_update_sql(table, data, condition):
-    """
-    @summary: 适用于mysql， oracle数据库时间需要to_date 处理（TODO）
-    ---------
-    @param table:
-    @param data: 表数据 json格式
-    @param condition: where 条件
-    ---------
-    @result:
-    """
-    key_values = []
-
-    for key, value in data.items():
-        value = format_sql_value(value)
-        if isinstance(value, str):
-            key_values.append("`{}`={}".format(key, repr(value)))
-        elif value is None:
-            key_values.append("`{}`={}".format(key, "null"))
-        else:
-            key_values.append("`{}`={}".format(key, value))
-
-    key_values = ", ".join(key_values)
-
-    sql = "update `{table}` set {key_values} where {condition}"
-    sql = sql.format(table=table, key_values=key_values, condition=condition)
-    return sql
-
-
-def make_batch_sql(
-        table, datas, auto_update=False, update_columns=(), update_columns_value=()
-):
-    """
-    @summary: 生成批量的SQL
-    ---------
-    @param table:
-    @param datas: 表数据 [{...}]
-    @param auto_update: 使用的是replace into，为完全覆盖已存在的数据
-    @param update_columns: 需要更新的列，默认全部，当指定值时，auto_update设置无效，当duplicate key冲突时更新指定的列
-    @param update_columns_value: 需要更新的列的值，默认为datas里边对应的值，注意如果值为字符串类型需要主动加单引号，如 update_columns_value=("'test'",)
-    ---------
-    @result:
-    """
-    if not datas:
-        return
-
-    keys = list(set([key for data in datas for key in data]))
-    # values_placeholder = ["%s"] * len(keys)
-    values = []
-    for data in datas:
-        # 检查 data 是否是字典类型
-        if not isinstance(data, dict):
-            # 如果 data 不是字典，记录错误日志并打印 data 的内容和类型
-            # logger.error(f"期望的数据类型是字典，但实际得到: {data} (类型: {type(data)})")
-            continue  # 跳过非字典类型的 data，继续处理下一个数据
-
-        value = []
-        for key in keys:
-            # 从字典中获取当前 key 对应的值
-            current_data = data.get(key)
-            try:
-                # 对值进行格式化处理
-                current_data = format_sql_value(current_data)
-                value.append(current_data)  # 将处理后的值添加到列表中
-            except Exception as e:
-                # 如果格式化失败，记录错误日志
-                logger.error(f"{key}: {current_data} (类型: {type(current_data)}) -> {e}")
-
-        # 将处理后的值列表添加到 values 中
-        values.append(value)
-    keys_str = ", ".join(["`{}`".format(key) for key in keys])
-    placeholders_str = ", ".join(["%s"] * len(keys))
-
-    if update_columns:
-        if not isinstance(update_columns, (tuple, list)):
-            update_columns = [update_columns]
-        if update_columns_value:
-            update_columns_ = ", ".join(
-                [
-                    "`{key}`={value}".format(key=key, value=value)
-                    for key, value in zip(update_columns, update_columns_value)
-                ]
-            )
-        else:
-            # 修改这里，使用 VALUES() 函数来引用插入的值
-            update_columns_ = ", ".join(
-                ["`{key}`=VALUES(`{key}`)".format(key=key) for key in update_columns]
-            )
-
-        sql = f"INSERT INTO `{table}` ({keys_str}) VALUES ({placeholders_str}) ON DUPLICATE KEY UPDATE {update_columns_}"
-    elif auto_update:
-        sql = "REPLACE INTO `{table}` ({keys}) VALUES ({values_placeholder})".format(
-            table=table, keys=keys_str, values_placeholder=placeholders_str
-        )
-    else:
-        sql = "INSERT IGNORE INTO `{table}` ({keys}) VALUES ({values_placeholder})".format(
-            table=table, keys=keys_str, values_placeholder=placeholders_str
-        )
-    return sql, values
-
-
-def format_sql_value(value):
-    """
-    格式化 SQL 值
-    """
-    if value is None:
-        return None  # 处理 NULL 值
-
-    # 确保处理字符串
-    if isinstance(value, str):
-        return value.strip()  # 去除首尾空格
-
-    # 处理列表或元组类型
-    elif isinstance(value, (list, tuple)):
-        try:
-            return dumps_json(value)  # 将其转为 JSON 字符串
-        except Exception as e:
-            raise ValueError(f"Failed to serialize list/tuple to JSON: {value}, error: {e}")
-
-    # 处理字典类型
-    elif isinstance(value, dict):
-        try:
-            return dumps_json(value)  # 将其转为 JSON 字符串
-        except Exception as e:
-            raise ValueError(f"Failed to serialize dict to JSON: {value}, error: {e}")
-
-    # 处理布尔类型
-    elif isinstance(value, bool):
-        return int(value)  # 转为整数
-
-    # 确保数值类型优先匹配
-    elif isinstance(value, (int, float)):
-        return value  # 返回数值
-
-    # 处理日期、时间类型
-    elif isinstance(value, (date, time, datetime)):
-        return str(value)  # 转换为字符串表示
-
-    # 如果遇到无法处理的类型，抛出异常
-    else:
-        raise TypeError(f"Unsupported value type: {type(value)}, value: {value}")
-
-
-
-
-def list2str(datas):
-    """
-    列表转字符串
-    :param datas: [1, 2]
-    :return: (1, 2)
-    """
-    data_str = str(tuple(datas))
-    data_str = re.sub(r",\)$", ")", data_str)
-    return data_str
-
-_REGEXPS = {}
-
-def get_info(html, regexps, allow_repeat=True, fetch_one=False, split=None):
-    regexps = isinstance(regexps, str) and [regexps] or regexps
-
-    infos = []
-    for regex in regexps:
-        if regex == "":
-            continue
-
-        if regex not in _REGEXPS.keys():
-            _REGEXPS[regex] = re.compile(regex, re.S)
-
-        if fetch_one:
-            infos = _REGEXPS[regex].search(html)
-            if infos:
-                infos = infos.groups()
-            else:
-                continue
-        else:
-            infos = _REGEXPS[regex].findall(str(html))
-
-        if len(infos) > 0:
-            break
-
-    if fetch_one:
-        infos = infos if infos else ("",)
-        return infos if len(infos) > 1 else infos[0]
-    else:
-        infos = allow_repeat and infos or sorted(set(infos), key=infos.index)
-        infos = split.join(infos) if split else infos
-        return infos
-
-
-def get_json(json_str):
-    """
-    @summary: 取json对象
-    ---------
-    @param json_str: json格式的字符串
-    ---------
-    @result: 返回json对象
-    """
-
-    try:
-        return json.loads(json_str) if json_str else {}
-    except Exception as e1:
-        try:
-            json_str = json_str.strip()
-            json_str = json_str.replace("'", '"')
-            keys = get_info(json_str, r"(\w+):")
-            for key in keys:
-                json_str = json_str.replace(key, '"%s"' % key)
-
-            return json.loads(json_str) if json_str else {}
-
-        except Exception as e2:
-            logger.error(
-                """
-                e1: %s
-                format json_str: %s
-                e2: %s
-                """
-                % (e1, json_str, e2)
-            )
-
-        return {}
-
-
-def dumps_json(data, indent=4, sort_keys=False):
-    """
-    @summary: 格式化json 用于打印
-    ---------
-    @param data: json格式的字符串或json对象
-    @param indent:
-    @param sort_keys:
-    ---------
-    @result: 格式化后的字符串
-    """
-    try:
-        if isinstance(data, str):
-            data = get_json(data)
-
-        data = json.dumps(
-            data,
-            ensure_ascii=False,
-            indent=indent,
-            skipkeys=True,
-            sort_keys=sort_keys,
-            default=str,
-        )
-
-    except Exception as e:
-        data = pformat(data)
-
-    return data
+def custom_extractor_proxy(data: dict, key: str='proxy') -> dict | str | None:
+    """只负责从 API 返回数据中提取代理部分"""
+    if data.get("status") == 0:
+        return data.get(key)  # 返回 {"http": "...", "https": "..."} 整个字典
+    return None

crawlo/utils/url.py

@@ -1,40 +1,40 @@
-from urllib.parse import urldefrag
-from w3lib.url import add_or_replace_parameter
-
-
-def escape_ajax(url: str) -> str:
-    """
-    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
-    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
-
-    规则说明：
-    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
-    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
-    3. 保留原始查询参数（如果有）
-
-    示例：
-    >>> escape_ajax("www.example.com/ajax.html#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
-    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html#!")
-    'www.example.com/ajax.html?_escaped_fragment_='
-
-    非AJAX可爬取的URL（无#!）原样返回：
-    >>> escape_ajax("www.example.com/ajax.html#normal")
-    'www.example.com/ajax.html#normal'
-    """
-    # 分离URL的基础部分和哈希片段
-    de_frag, frag = urldefrag(url)
-
-    # 仅处理以"!"开头的哈希片段（Google规范）
-    if not frag.startswith("!"):
-        return url  # 不符合规则则原样返回
-
-    # 调用辅助函数添加 `_escaped_fragment_` 参数
-    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
-
-
-if __name__ == '__main__':
-    f = escape_ajax('http://example.com/page#!')
+from urllib.parse import urldefrag
+from w3lib.url import add_or_replace_parameter
+
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
+
+
+if __name__ == '__main__':
+    f = escape_ajax('http://example.com/page#!')
     print(f)

{crawlo-1.0.4.dist-info → crawlo-1.0.6.dist-info}/METADATA

@@ -1,48 +1,49 @@
-Metadata-Version: 2.4
-Name: crawlo
-Version: 1.0.4
-Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
-Home-page: https://github.com/crawl-coder/Crawlo.git
-Author: crawl-coder
-Author-email: crawlo@qq.com
-License: MIT
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp>=3.12.14
-Requires-Dist: aiomysql>=0.2.0
-Requires-Dist: aioredis>=2.0.1
-Requires-Dist: asyncmy>=0.2.10
-Requires-Dist: cssselect>=1.2.0
-Requires-Dist: dateparser>=1.2.2
-Requires-Dist: httpx[http2]>=0.27.0
-Requires-Dist: lxml>=5.2.1
-Requires-Dist: motor>=3.7.0
-Requires-Dist: parsel>=1.9.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: pymongo>=4.11
-Requires-Dist: PyMySQL>=1.1.1
-Requires-Dist: python-dateutil>=2.9.0.post0
-Requires-Dist: redis>=6.2.0
-Requires-Dist: requests>=2.32.4
-Requires-Dist: six>=1.17.0
-Requires-Dist: ujson>=5.9.0
-Requires-Dist: urllib3>=2.5.0
-Requires-Dist: w3lib>=2.1.2
-Provides-Extra: render
-Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
-Requires-Dist: playwright; extra == "render"
-Requires-Dist: selenium>=3.141.0; extra == "render"
-Provides-Extra: all
-Requires-Dist: bitarray>=1.5.3; extra == "all"
-Requires-Dist: PyExecJS>=1.5.1; extra == "all"
-Requires-Dist: pymongo>=3.10.1; extra == "all"
-Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
-Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: selenium>=3.141.0; extra == "all"
-
-# Crawlo
-Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.0.6
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.6.dist-info/RECORD

@@ -0,0 +1,94 @@
+crawlo/__init__.py,sha256=xpiIAZbSG3CzneJuDLPCbwfRcvw2wyHYl2kJjaNfNGY,584
+crawlo/__version__.py,sha256=1HqFYnow__4MUVRI_OMjvzTBzKkReNozOdA96kH53cA,23
+crawlo/cli.py,sha256=hjAJKx9pba375sATvvcy-dtZyBIgXj8fRBq9RFIZHA4,1206
+crawlo/crawler.py,sha256=nfuA_f8QnuIp2F4ZbaJv8Fceo_QPwqV1jYdD_edkMjg,8527
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=xdyZkvVcLEJ-19sWMHvn9IJsu30-hAY2jJhA2kYIims,1207
+crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
+crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
+crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
+crawlo/commands/__init__.py,sha256=dRu3ipuhDM7M1eTb6zJtQZ_u7N_tZumGfH5_I92xno8,252
+crawlo/commands/genspider.py,sha256=kSHYsAGHRoxU6Qf_MGpR_VS-Ua5NUGY2KGm_Wapn0sw,3529
+crawlo/commands/run.py,sha256=Upv8K4sM0c0I1fIwTFK18VDcSHF7xabqfXtQ82fk56g,4628
+crawlo/commands/startproject.py,sha256=1KOq_CALy01oklr0dAUYhGFzu4f7w45q2H0O3qafLX4,3494
+crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
+crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
+crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
+crawlo/core/scheduler.py,sha256=ZMPs4LSs69FsFfDTvaOMJKqpSQQGvIEE9pMyYVVAA64,1948
+crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
+crawlo/downloader/aiohttp_downloader.py,sha256=YfvYCDp3y0OsVyfdYX1XJC3EcCrbNLKOcFY8b7JC3_w,7675
+crawlo/downloader/cffi_downloader.py,sha256=QthBmZOE0cjYNRTM-449EuaFuqxxdc19kp93iqOlwB8,12678
+crawlo/downloader/httpx_downloader.py,sha256=yshb1JZa4B6hcVwIT97SrxCXkj3HJqT9IDpxSxjkJm4,11754
+crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
+crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
+crawlo/extension/log_stats.py,sha256=JFJsdK7UWkhP4TEAF-H-S7SpQbDpBryS0AT6e6jZCBo,1721
+crawlo/extension/logging_extension.py,sha256=rty2_up53KV05nCazuBuz2ZapHKq0ti7mGVBzMTr0ak,1236
+crawlo/filters/__init__.py,sha256=9fJQRVkxWWPChajYbAGe1O6UYB639xWt0hiLUGBs4hQ,1014
+crawlo/filters/aioredis_filter.py,sha256=phBFW9Z28oylbik9Kb2WHM65Wo5yRAH2w9Yz0_2HaOQ,5621
+crawlo/filters/memory_filter.py,sha256=L8XEJkObOxs4BzYpQvk9PVM969k2LE61VFsnEOTEf_E,6841
+crawlo/items/__init__.py,sha256=HLDShSwAQUrgwt9_Ec2SIwzpIDZnNOCg9nSYqqEQdp8,407
+crawlo/items/base.py,sha256=DZG0qENdukJExRtKjqdNkSlzUoWR3ucjyF73LYLANFo,754
+crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
+crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
+crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
+crawlo/middleware/middleware_manager.py,sha256=Vfkasi8YaLxzGrOrFYfxOMEGRS8XocqeQMruLtVxL_c,6360
+crawlo/middleware/proxy.py,sha256=PiIfhRXfcMzBtW_p7jfR8rGxcM4VT68Mk54swbaV2H4,9801
+crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
+crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
+crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
+crawlo/middleware/retry.py,sha256=a2EmigYFzL8oxd50JhrSe5XbYJyx8yDjOjE5fXAOFhY,3459
+crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
+crawlo/network/request.py,sha256=qd50mmrXS6yZKmAb6ERAMHzm2Ln80Wu5NSMwx_t1IGc,7247
+crawlo/network/response.py,sha256=z2Owti_9ds567jLvfuX8hrfdQL8JKn5lkt2QOc-Gi3Y,6200
+crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
+crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
+crawlo/pipelines/mongo_pipeline.py,sha256=Yr48D0T61-_Y-EpgWXf7BUn9w8e-Pj5P07QDSPZ0pYU,4558
+crawlo/pipelines/mysql_batch_pipline.py,sha256=Mj3PReDRw22JhJ5hZxnka4cirKq3kEbOCNhgpq1gvfA,10611
+crawlo/pipelines/mysql_pipeline.py,sha256=bsAFqpxrCijzvX-IusxOtvTvQEUCt5uHNTyYMo_pIq4,8056
+crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7aARE5W3k,2174
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=urj4XJ--ZpVRbbo3fWUT71bYQLmElx43AC9KeHtqHBs,7310
+crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
+crawlo/spider/__init__.py,sha256=lWi0bCR7HLT5bnj7_e9UIgFJjuqoeWtbwADfNkaajug,1139
+crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
+crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
+crawlo/templates/project/items.py.tmpl,sha256=bXx-oCldMr2EgBKUAH9LH5gMnbyLiWX-EySAaMzcu2g,318
+crawlo/templates/project/middlewares.py.tmpl,sha256=VAolmMTC6HBmJT5XvWB0ag6ig9iaDBS32adIQ1zPdw0,2177
+crawlo/templates/project/pipelines.py.tmpl,sha256=xK1Yl7wYxiyUCm07GZvMnCS_cxJ5LF7z1YBBdkLlWys,1880
+crawlo/templates/project/settings.py.tmpl,sha256=985Z-jiU6A31f5s1IVU4PvkC_QGlFlRRfTF6rZ_G4ek,1771
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
+crawlo/templates/spider/spider.py.tmpl,sha256=SkNv1kOwet7ZdxoNXpj-o1iRETB30bcwPP16Uy8lyXg,869
+crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
+crawlo/utils/concurrency_manager.py,sha256=o-_cfeUHdlBOM3eAXF857MtekSrRcVTBJ2jWZvY6weQ,5230
+crawlo/utils/date_tools.py,sha256=lcEFP2Z5b-6pUTHczrzCCuqiHP_4_2zamomMGPZrExo,7194
+crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
+crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
+crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
+crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
+crawlo/utils/project.py,sha256=qAiCmpIxiB7RxCLG-U5lGV6k4UCa21uRdykTfnAF834,7669
+crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
+crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
+crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
+examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+examples/gxb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+examples/gxb/items.py,sha256=3-1Lxpi7EqMzheDJoO0MPyHky5nHG_nqQGgKlm8y6mQ,989
+examples/gxb/run.py,sha256=9kJlR8f-tZ3BqP5PW7sCLTw6PAFWo3x4cG5lc-6GWqI,333
+examples/gxb/settings.py,sha256=_nbXj9HV2e0F6liUzK0ueygLcaMM_IUlkuwL6mJqUfc,2345
+examples/gxb/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+examples/gxb/spider/miit_spider.py,sha256=tcQnuyUHfu-Re1QbKKSI9DXW3Sp1vyBW8qBzKLf_RC4,6666
+examples/gxb/spider/telecom_device.py,sha256=58iG6BQtQjjDHOF7-DXH0u5_XnppP5AJTQwaVJVyBEo,4929
+tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
+tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
+tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
+tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
+tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
+tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
+crawlo-1.0.6.dist-info/METADATA,sha256=_TDAivxDg2R8omq5gG1kUiODY2tZ3UEp5aH0SwshOjI,1825
+crawlo-1.0.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.6.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.0.6.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.0.6.dist-info/RECORD,,

crawlo-1.0.6.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+crawlo = crawlo.cli:main

{crawlo-1.0.4.dist-info → crawlo-1.0.6.dist-info}/top_level.txt

@@ -1,2 +1,3 @@
 crawlo
+examples
 tests

examples/gxb/items.py

@@ -0,0 +1,36 @@
+from crawlo.items import Item, Field
+
+class RadioApprovalItem(Item):
+    approval_number = Field()
+    device_name = Field()
+    device_model = Field()
+    applicant = Field()
+    remarks = Field()
+    validity_period = Field()
+    frequency_tolerance = Field()
+    frequency_range = Field()
+    transmit_power = Field()
+    occupied_bandwidth = Field()
+    spurious_emission_limit = Field()
+    issue_date = Field()
+    approval_code = Field()
+    cmiit_id = Field()
+    modulation_mode = Field()
+    technology_system = Field()
+    mid = Field()
+
+class TelecomLicenseItem(Item):
+    license_number = Field()
+    device_name = Field()
+    device_model = Field()
+    applicant = Field()
+    manufacturer = Field()
+    issue_date = Field()
+    expiry_date = Field()
+    certificate_type = Field()
+    remarks = Field()
+    certificate_status = Field()
+    origin = Field()
+    article_id = Field()
+    article_edit_date = Field()
+    create_time = Field()

examples/gxb/run.py

@@ -0,0 +1,16 @@
+import asyncio
+
+from crawlo.crawler import CrawlerProcess
+from examples.gxb.spider.telecom_device import TelecomDeviceLicensesSpider
+
+async def main():
+    process = CrawlerProcess()
+    await process.crawl(
+        [TelecomDeviceLicensesSpider]
+    )
+
+
+
+if __name__ == '__main__':
+    asyncio.run(main())
+    # 132023