crawlo 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

tests/test_proxy_strategies.py

@@ -0,0 +1,60 @@
+# tests/test_proxy_strategies.py
+import pytest
+from crawlo import Request
+from crawlo.proxy.strategies import STRATEGIES
+
+
+@pytest.fixture
+def mock_proxies():
+    """提供测试用的代理列表"""
+    return [
+        {'url': 'http://p1:8080'},
+        {'url': 'http://p2:8080'},
+        {'url': 'http://p3:8080'},
+    ]
+
+
+@pytest.fixture
+def mock_stats():
+    """提供测试用的统计信息"""
+    return {
+        'http://p1:8080': {'total': 10},
+        'http://p2:8080': {'total': 5},
+        'http://p3:8080': {'total': 1},
+    }
+
+
+@pytest.fixture
+def mock_request():
+    """提供测试用的请求对象"""
+    return Request("https://example.com")
+
+
+def test_random_strategy(mock_proxies, mock_request, mock_stats):
+    """测试随机策略"""
+    strategy = STRATEGIES['random']
+    chosen = strategy(mock_proxies, mock_request, mock_stats)
+    assert chosen in [p['url'] for p in mock_proxies]
+
+
+def test_least_used_strategy(mock_proxies, mock_request, mock_stats):
+    """测试最少使用策略"""
+    strategy = STRATEGIES['least_used']
+    chosen = strategy(mock_proxies, mock_request, mock_stats)
+    assert chosen == 'http://p3:8080'  # total=1
+
+
+def test_domain_rule_strategy(mock_proxies, mock_request, mock_stats):
+    """测试域名规则策略"""
+    from crawlo.proxy.strategies.domain_rule import domain_rule_strategy
+    request = Request("https://taobao.com/item/123")
+    rules = {'taobao.com': 'http://special:8080'}
+
+    # Monkey patch 确保有回退策略
+    old_strategy = STRATEGIES['least_used']
+    try:
+        STRATEGIES['least_used'] = lambda p, r, s: 'http://fallback:8080'
+        chosen = domain_rule_strategy(mock_proxies, request, mock_stats, rules)
+        assert chosen == 'http://special:8080'
+    finally:
+        STRATEGIES['least_used'] = old_strategy

crawlo/downloader/playwright_downloader.py

@@ -1,161 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import Optional, Dict, Any
-from playwright.async_api import Browser, Page, Response as PlaywrightResponse
-from crawlo import Response, Request
-from crawlo.downloader import DownloaderBase
-
-
-class PlaywrightDownloader(DownloaderBase):
-    def __init__(self, crawler):
-        super().__init__(crawler)
-        # Playwright 核心对象
-        self.browser: Optional[Browser] = None  # 浏览器实例
-        self.context: Optional[Any] = None  # 浏览器上下文（隔离cookies等）
-
-        # 可配置参数（通过crawler.settings覆盖默认值）
-        self._browser_type: str = "chromium"  # 浏览器类型（chromium/firefox/webkit）
-        self._headless: bool = True  # 是否无头模式
-        self._timeout: int = 30000  # 操作超时（毫秒）
-        self._viewport: Dict[str, int] = {"width": 1280, "height": 720}  # 视口大小
-        self._extra_launch_args: Dict[str, Any] = {}  # 浏览器启动额外参数
-
-    async def _init_browser(self):
-        """初始化Playwright浏览器实例"""
-        from playwright.async_api import async_playwright
-
-        # 启动Playwright引擎
-        playwright = await async_playwright().start()
-
-        # 根据配置选择浏览器类型
-        browser_launcher = {
-            "chromium": playwright.chromium,
-            "firefox": playwright.firefox,
-            "webkit": playwright.webkit
-        }.get(self._browser_type, playwright.chromium)  # 默认chromium
-
-        # 启动浏览器（含启动参数）
-        self.browser = await browser_launcher.launch(
-            headless=self._headless,  # 无头模式开关
-            timeout=self._timeout,  # 启动超时
-            **self._extra_launch_args  # 透传额外参数（如代理配置）
-        )
-
-        # 创建浏览器上下文（隔离环境）
-        self.context = await self.browser.new_context(
-            viewport=self._viewport,  # 设置窗口大小
-            user_agent=self.crawler.settings.get("USER_AGENT")  # 自定义UA
-        )
-
-    def open(self):
-        """从crawler配置加载参数"""
-        super().open()  # 调用父类初始化
-
-        # 读取配置（支持在settings.py中覆盖）
-        self._browser_type = self.crawler.settings.get("PLAYWRIGHT_BROWSER", "chromium")
-        self._headless = self.crawler.settings.get_bool("HEADLESS", True)
-        self._timeout = self.crawler.settings.get_int("PLAYWRIGHT_TIMEOUT", 30000)
-        self._viewport = self.crawler.settings.get_dict("VIEWPORT", {"width": 1280, "height": 720})
-        self._extra_launch_args = self.crawler.settings.get_dict("PLAYWRIGHT_LAUNCH_ARGS", {})
-
-    async def download(self, request: Request) -> Response:
-        """
-        核心下载方法：
-        1. 创建新页面Tab
-        2. 加载目标URL
-        3. 获取渲染后的内容
-        """
-        if not self.browser:
-            await self._init_browser()  # 懒加载浏览器
-
-        page = await self.context.new_page()  # 每个请求独立Page（自动隔离）
-
-        try:
-            # 设置请求头（模拟浏览器）
-            if request.headers:
-                await page.set_extra_http_headers(request.headers)
-
-            # 导航到目标URL（支持等待策略配置）
-            response = await page.goto(
-                request.url,
-                timeout=self._timeout,
-                wait_until="domcontentloaded"  # 等待策略：domcontentloaded/networkidle/load
-            )
-
-            # 特殊处理POST请求（Playwright限制需用API方式）
-            if request.method.lower() == "post":
-                return await self._handle_post_request(request, page)
-
-            # 执行自定义JavaScript（用于提取动态数据）
-            if request.meta.get("execute_js"):
-                result = await page.evaluate(request.meta["execute_js"])
-                request.meta["js_result"] = result  # 存储JS执行结果
-
-            # 获取渲染后的完整HTML（含动态生成内容）
-            body = await page.content()
-
-            # 调试模式下截图（用于排查页面问题）
-            if self.crawler.settings.get_bool("DEBUG"):
-                screenshot = await page.screenshot(type="png")
-                request.meta["screenshot"] = screenshot  # 截图存入request.meta
-
-            # 构造统一响应对象
-            return self._structure_response(request, response, body)
-
-        except Exception as e:
-            self.logger.error(f"页面下载失败: {str(e)}")
-            raise
-        finally:
-            await page.close()  # 确保页面关闭，避免资源泄漏
-
-    async def _handle_post_request(self, request: Request, page: Page) -> Response:
-        """
-        处理POST请求的特殊方法：
-        通过页面内fetch API发送POST请求，并监听响应
-        """
-        async with page.expect_response(request.url) as response_info:
-            # 在页面上下文中执行fetch
-            await page.evaluate(
-                """async ({url, headers, body}) => {
-                    await fetch(url, {
-                        method: 'POST',
-                        headers: headers,
-                        body: body
-                    });
-                }""",
-                {
-                    "url": request.url,
-                    "headers": request.headers or {},
-                    "body": request.body or ""
-                }
-            )
-
-        response = await response_info.value  # 获取API响应
-        body = await response.text()  # 读取响应体
-        return self._structure_response(request, response, body)
-
-    @staticmethod
-    def _structure_response(
-            request: Request,
-            response: PlaywrightResponse,
-            body: str
-    ) -> Response:
-        """
-        标准化响应格式：
-        将Playwright的响应转换为crawlo的统一Response对象
-        """
-        return Response(
-            url=str(response.url),  # 最终URL（含重定向）
-            headers=response.headers,  # 响应头
-            status_code=response.status,  # HTTP状态码
-            body=body.encode('utf-8'),  # 响应体（转bytes）
-            request=request  # 关联的请求对象
-        )
-
-    async def close(self) -> None:
-        """资源清理：关闭浏览器实例和上下文"""
-        if self.context:
-            await self.context.close()
-        if self.browser:
-            await self.browser.close()
-        await super().close()  # 调用父类清理逻辑

crawlo/templates/item_template.tmpl

@@ -1,22 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary:
----------
-@author: {USER}
-"""
-
-from crawlo import Item
-
-
-class ${item_name}Item(Item):
-    """
-    This class was generated by feapder
-    command: feapder create -i ${command}
-    """
-
-    __table_name__ = "${table_name}"
-
-    def __init__(self, *args, **kwargs):
-        ${propertys}

crawlo/templates/project_template/main.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary: 爬虫入口
----------
-@author: {USER}
-"""
-
-from crawlo import ArgumentParser
-
-from spiders import *
-
-
-
-def crawl_xxx():
-    """
-    Spider爬虫
-    """
-    spider = xxx.XXXSpider(redis_key="xxx:xxx")
-    spider.start()
-
-
-
-if __name__ == "__main__":
-    parser = ArgumentParser(description="xxx爬虫")
-
-    parser.add_argument(
-        "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx
-    )
-    parser.start()
-
-    # main.py作为爬虫启动的统一入口，提供命令行的方式启动多个爬虫

crawlo/templates/project_template/setting.py

@@ -1,190 +0,0 @@
-# -*- coding: utf-8 -*-
-"""爬虫配置文件"""
-# import os
-# import sys
-#
-# # MYSQL
-# MYSQL_IP = "localhost"
-# MYSQL_PORT = 3306
-# MYSQL_DB = ""
-# MYSQL_USER_NAME = ""
-# MYSQL_USER_PASS = ""
-#
-# # MONGODB
-# MONGO_IP = "localhost"
-# MONGO_PORT = 27017
-# MONGO_DB = ""
-# MONGO_USER_NAME = ""
-# MONGO_USER_PASS = ""
-# MONGO_URL = "
-#
-# # REDIS
-# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"]
-# REDISDB_IP_PORTS = "localhost:6379"
-# REDISDB_USER_PASS = ""
-# REDISDB_DB = 0
-# # 连接redis时携带的其他参数，如ssl=True
-# REDISDB_KWARGS = dict()
-# # 适用于redis哨兵模式
-# REDISDB_SERVICE_NAME = ""
-#
-# # 数据入库的pipeline，可自定义，默认MysqlPipeline
-# ITEM_PIPELINES = [
-#     "feapder.pipelines.mysql_pipeline.MysqlPipeline",
-#     # "feapder.pipelines.mongo_pipeline.MongoPipeline",
-#     # "feapder.pipelines.console_pipeline.ConsolePipeline",
-# ]
-# EXPORT_DATA_MAX_FAILED_TIMES = 10  # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警
-# EXPORT_DATA_MAX_RETRY_TIMES = 10  # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试
-#
-# # 爬虫相关
-# # COLLECTOR
-# COLLECTOR_TASK_COUNT = 32  # 每次获取任务数量，追求速度推荐32
-#
-# # SPIDER
-# SPIDER_THREAD_COUNT = 1  # 爬虫并发数，追求速度推荐32
-# # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5
-# SPIDER_SLEEP_TIME = 0
-# SPIDER_MAX_RETRY_TIMES = 10  # 每个请求最大重试次数
-# KEEP_ALIVE = False  # 爬虫是否常驻
-
-# 下载
-# DOWNLOADER = "feapder.network.downloader.RequestsDownloader"  # 请求下载器
-# SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader"
-# RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader"  # 渲染下载器
-# # RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader"
-# MAKE_ABSOLUTE_LINKS = True  # 自动转成绝对连接
-
-# # 浏览器渲染
-# WEBDRIVER = dict(
-#     pool_size=1,  # 浏览器的数量
-#     load_images=True,  # 是否加载图片
-#     user_agent=None,  # 字符串 或 无参函数，返回值为user_agent
-#     proxy=None,  # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
-#     headless=False,  # 是否为无头浏览器
-#     driver_type="CHROME",  # CHROME、EDGE、PHANTOMJS、FIREFOX
-#     timeout=30,  # 请求超时时间
-#     window_size=(1024, 800),  # 窗口大小
-#     executable_path=None,  # 浏览器路径，默认为默认路径
-#     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
-#     custom_argument=[
-#         "--ignore-certificate-errors",
-#         "--disable-blink-features=AutomationControlled",
-#     ],  # 自定义浏览器渲染参数
-#     xhr_url_regexes=None,  # 拦截xhr接口，支持正则，数组类型
-#     auto_install_driver=True,  # 自动下载浏览器驱动 支持chrome 和 firefox
-#     download_path=None,  # 下载文件的路径
-#     use_stealth_js=False,  # 使用stealth.min.js隐藏浏览器特征
-# )
-#
-# PLAYWRIGHT = dict(
-#     user_agent=None,  # 字符串 或 无参函数，返回值为user_agent
-#     proxy=None,  # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
-#     headless=False,  # 是否为无头浏览器
-#     driver_type="chromium",  # chromium、firefox、webkit
-#     timeout=30,  # 请求超时时间
-#     window_size=(1024, 800),  # 窗口大小
-#     executable_path=None,  # 浏览器路径，默认为默认路径
-#     download_path=None,  # 下载文件的路径
-#     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
-#     wait_until="networkidle",  # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle"
-#     use_stealth_js=False,  # 使用stealth.min.js隐藏浏览器特征
-#     page_on_event_callback=None,  # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()}
-#     storage_state_path=None,  # 保存浏览器状态的路径
-#     url_regexes=None,  # 拦截接口，支持正则，数组类型
-#     save_all=False,  # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口
-# )
-#
-# # 爬虫启动时，重新抓取失败的requests
-# RETRY_FAILED_REQUESTS = False
-# # 爬虫启动时，重新入库失败的item
-# RETRY_FAILED_ITEMS = False
-# # 保存失败的request
-# SAVE_FAILED_REQUEST = True
-# # request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做）
-# REQUEST_LOST_TIMEOUT = 600  # 10分钟
-# # request网络请求超时时间
-# REQUEST_TIMEOUT = 22  # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组
-# # item在内存队列中最大缓存数量
-# ITEM_MAX_CACHED_COUNT = 5000
-# # item每批入库的最大数量
-# ITEM_UPLOAD_BATCH_MAX_SIZE = 1000
-# # item入库时间间隔
-# ITEM_UPLOAD_INTERVAL = 1
-# # 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。
-# TASK_MAX_CACHED_SIZE = 0
-#
-# # 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求
-# RESPONSE_CACHED_ENABLE = False  # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True
-# RESPONSE_CACHED_EXPIRE_TIME = 3600  # 缓存时间 秒
-# RESPONSE_CACHED_USED = False  # 是否使用缓存 补采数据时可设置为True
-#
-# # 设置代理
-# PROXY_EXTRACT_API = None  # 代理提取API ，返回的代理分割符为\r\n
-# PROXY_ENABLE = True
-# PROXY_MAX_FAILED_TIMES = 5  # 代理最大失败次数，超过则不使用，自动删除
-# PROXY_POOL = "feapder.network.proxy_pool.ProxyPool"  # 代理池
-#
-# # 随机headers
-# RANDOM_HEADERS = True
-# # UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型
-# USER_AGENT_TYPE = "chrome"
-# # 默认使用的浏览器头
-# DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"
-# # requests 使用session
-# USE_SESSION = False
-#
-# # 去重
-# ITEM_FILTER_ENABLE = False  # item 去重
-# REQUEST_FILTER_ENABLE = False  # request 去重
-# ITEM_FILTER_SETTING = dict(
-#     filter_type=1  # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4
-# )
-# REQUEST_FILTER_SETTING = dict(
-#     filter_type=3,  # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4
-#     expire_time=2592000,  # 过期时间1个月
-# )
-#
-# # 报警 支持钉钉、飞书、企业微信、邮件
-# # 钉钉报警
-# DINGDING_WARNING_URL = ""  # 钉钉机器人api
-# DINGDING_WARNING_PHONE = ""  # 被@的群成员手机号，支持列表，可指定多个。
-# DINGDING_WARNING_USER_ID = ""  # 被@的群成员userId，支持列表，可指定多个
-# DINGDING_WARNING_ALL = False  # 是否提示所有人， 默认为False
-# DINGDING_WARNING_SECRET = None  # 加签密钥
-# # 飞书报警
-# # https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f
-# FEISHU_WARNING_URL = ""  # 飞书机器人api
-# FEISHU_WARNING_USER = None  # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}]
-# FEISHU_WARNING_ALL = False  # 是否提示所有人， 默认为False
-# # 邮件报警
-# EMAIL_SENDER = ""  # 发件人
-# EMAIL_PASSWORD = ""  # 授权码
-# EMAIL_RECEIVER = ""  # 收件人 支持列表，可指定多个
-# EMAIL_SMTPSERVER = "smtp.163.com"  # 邮件服务器 默认为163邮箱
-# # 企业微信报警
-# WECHAT_WARNING_URL = ""  # 企业微信机器人api
-# WECHAT_WARNING_PHONE = ""  # 报警人 将会在群内@此人, 支持列表，可指定多人
-# WECHAT_WARNING_ALL = False  # 是否提示所有人， 默认为False
-# # 时间间隔
-# WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏; 0表示不去重
-# WARNING_LEVEL = "DEBUG"  # 报警级别， DEBUG / INFO / ERROR
-# WARNING_FAILED_COUNT = 1000  # 任务失败数 超过WARNING_FAILED_COUNT则报警
-#
-# LOG_NAME = os.path.basename(os.getcwd())
-# LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径
-# LOG_LEVEL = "DEBUG"
-# LOG_COLOR = True  # 是否带有颜色
-# LOG_IS_WRITE_TO_CONSOLE = True  # 是否打印到控制台
-# LOG_IS_WRITE_TO_FILE = False  # 是否写文件
-# LOG_MODE = "w"  # 写文件的模式
-# LOG_MAX_BYTES = 10 * 1024 * 1024  # 每个日志文件的最大字节数
-# LOG_BACKUP_COUNT = 20  # 日志文件保留数量
-# LOG_ENCODING = "utf8"  # 日志文件编码
-# OTHERS_LOG_LEVAL = "ERROR"  # 第三方库的log等级
-#
-# # 切换工作路径为当前项目路径
-# project_path = os.path.abspath(os.path.dirname(__file__))
-# os.chdir(project_path)  # 切换工作路经
-# sys.path.insert(0, project_path)
-# print("当前工作路径为 " + os.getcwd())

crawlo/templates/spider_template.tmpl

@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary:
----------
-@author: {USER}
-"""
-
-import crawlo
-
-
-class ${spider_name}(crawlo.Spider):
-    # 自定义数据库，若项目中有setting.py文件，此自定义可删除
-    __custom_setting__ = dict(
-        REDISDB_IP_PORTS="localhost:6379", REDISDB_USER_PASS="", REDISDB_DB=0
-    )
-
-    def start_requests(self):
-        yield feapder.Request("https://spidertools.cn")
-
-    def parse(self, request, response):
-        # 提取网站title
-        print(response.xpath("//title/text()").extract_first())
-        # 提取网站描述
-        print(response.xpath("//meta[@name='description']/@content").extract_first())
-        print("网站地址: ", response.url)
-
-
-if __name__ == "__main__":
-    ${spider_name}(redis_key="xxx:xxx").start()

crawlo-1.0.4.dist-info/RECORD

@@ -1,79 +0,0 @@
-crawlo/__init__.py,sha256=XOWXajnhT2HVql5cycwGkQ0MS85bpQnFdM7tl0Fusik,327
-crawlo/__version__.py,sha256=acuR_XSJzp4OrQ5T8-Ac5gYe48mUwObuwjRmisFmZ7k,22
-crawlo/crawler.py,sha256=rqKjMLDU6qlm2D2gIhkezF5jFOCz0TgYyq-nS7MEFMU,9237
-crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
-crawlo/exceptions.py,sha256=7dtEJBxb9yvmMJe6MQyDB0LuV9que1J_jQN4QYeyO4g,916
-crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
-crawlo/subscriber.py,sha256=3d4eYtkSgPj-18-mTZM6RQLSil-ux13FUcmfFxr3sMk,3730
-crawlo/task_manager.py,sha256=AS7Xu_8Q_eb3jg9QSkK_wv6W1rRXaI6WjDp8p6h9ltU,721
-crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=OcGsY2ikDNXK4j9VqB0bUHs2v1TyWvSJu2mD1W9CbGc,5872
-crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=3pnm5L241aEHnJJArnbCnirooo4wWFgAmnP1tMB049k,1891
-crawlo/downloader/__init__.py,sha256=ukrDBULCaoDWoMLCO3XcQhDoasF0oUzj0PHnJ_ACJaE,2306
-crawlo/downloader/aiohttp_downloader.py,sha256=EVCfbaCUJlTa1ZG32NhxKyi8FVFh-NoK0u57ct1MWos,9844
-crawlo/downloader/httpx_downloader.py,sha256=UQ7u3U_Iy8u1i2q0fDBakfu3C6EBN-T5Q0US6u-Um84,7002
-crawlo/downloader/playwright_downloader.py,sha256=q-Yy-hAaS7attXeAr7HAWAuFO2u1EOp_NdHnUPoDRRA,6566
-crawlo/extension/__init__.py,sha256=LPy9XyCu089k6L6oVENIi_imr75AEuY8QTtSJjRioiw,1139
-crawlo/extension/log_interval.py,sha256=S-hSoiz9GdmgHrac4vDQ52fleoBcH-kzdPUD8YRAons,1922
-crawlo/extension/log_stats.py,sha256=WeSnOoSKB8pI_xmcGdh906XnF1xwo6fgJnf_prElwwI,1742
-crawlo/filters/__init__.py,sha256=BCZl86BHiTfDGRe_b1TlNSr6pfNbMKTu0Uq0j4gX_1Q,977
-crawlo/filters/aioredis_filter.py,sha256=MJT74BeVZTjdExKEzdrWKc7WPXFss1k-txc7E54H77E,5522
-crawlo/filters/memory_filter.py,sha256=bs2WUe7CdHiXgr344vzDqMfBv1b3RwXJMnwxpDb64Pw,6639
-crawlo/items/__init__.py,sha256=JUw4wZX50DidJuCMLkP41ik_wTKum2b8iDxm7EbRRds,2063
-crawlo/items/items.py,sha256=00TdAYChF5Rbbgm6a6d-GCxkx4gXP-rA-_Q7u33BuFI,3990
-crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
-crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
-crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
-crawlo/middleware/middleware_manager.py,sha256=kMqnSf4kltDkRf8PV0Xs9Ple9z-oKKQrMS0Q0_-4vNQ,6489
-crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
-crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
-crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
-crawlo/middleware/retry.py,sha256=BV-rYm3WVp8Hcrxc0JUGYfYAwvOWfXTWtatp3S5K9oU,3375
-crawlo/network/__init__.py,sha256=VaD0GmsgDYJ8UMgrtjeOc1Wc7lDGee1uAF3neRpyug0,123
-crawlo/network/request.py,sha256=eyju3BddPU8eNoueY48eqju8I96r9RasmNXXSaUU8dg,7086
-crawlo/network/response.py,sha256=6TO8hvkcgtVVPjQhXI3ywx5E_lV0eGrTBceEi60t55c,6034
-crawlo/pipelines/__init__.py,sha256=Hk-M6X0VCGLp6OEdgnhXGhGhKS5TjKf6dkg8bU9pvUE,260
-crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
-crawlo/pipelines/mongo_pipeline.py,sha256=lv-Zn_mWdE_jVy7Nh30Lzqm3YhtLRV5rMy-m4rBWYe0,4442
-crawlo/pipelines/mysql_batch_pipline.py,sha256=g111iuPTRyKr0q4PHTJYIfsYAFf8CCuyYY6DDLSpMO0,4889
-crawlo/pipelines/mysql_pipeline.py,sha256=ZlRWwZLewG9SBLBZ1wWNZ8yAj5xWWitb7BKRSrqEWtI,7857
-crawlo/pipelines/pipeline_manager.py,sha256=JIoX5D-oDfUT7VJrb5m355wi43SChb4nNb09z_0F4_g,2118
-crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=zNMVMo_9s1DGr1TiPzwZjSmxuD4qj_JT_oCCmkoMfjs,2579
-crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
-crawlo/spider/__init__.py,sha256=pP_TChnozpHeuS87Bs-Sj31hb0R7glYN3K6BsRw4FOA,905
-crawlo/templates/item_template.tmpl,sha256=bo0cjaFOT1jMrtLjXs6z7Mhwev-s3037suD4BL2_ji4,351
-crawlo/templates/spider_template.tmpl,sha256=dDMOa_17uWKB3FopYrDYLMYhHGaYevm1hI9AVeY6QAg,805
-crawlo/templates/project_template/main.py,sha256=RbGWsdtpOTPMD-jL80sDqu8C-TgI9qrrwJZ8NeK0PZM,594
-crawlo/templates/project_template/setting.py,sha256=NjP9KuhL3pBtRQfC4djBFq4CvBR4H1_OSVDfWMZITh0,9206
-crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
-crawlo/utils/concurrency_manager.py,sha256=nBrHlrKqGENINDA6zrbpK0jCbcjWqobI10vYy9Sg3wU,5106
-crawlo/utils/date_tools.py,sha256=9OVJB66_0BvRq-lwUE4JYcd6J5RGADW3lcKEWS6lCi0,5319
-crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
-crawlo/utils/log.py,sha256=vNHCIw9owCZ4voNM_hni7fOEyt9kKoOhIrjvl76lKQg,989
-crawlo/utils/pqueue.py,sha256=4Ymkm38fRFqEcSJeD_ULkuBaCk6QdYvJdnYvtJjh-Tk,5386
-crawlo/utils/project.py,sha256=qRErB6Ru81-PpSnT9g2ZPyfbWCwZ8hygpMAWhwIhC_M,1485
-crawlo/utils/request.py,sha256=ftnyr6f--StdcYju5FaU_khRARaxMMktJS8wROA1Fe0,4119
-crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
-crawlo/utils/tools.py,sha256=isRvzTMU3n1FWMhzTAt-7TVrHmH8JPUwgWyVel71Wj0,9462
-crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
-tests/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
-tests/baidu_spider/__init__.py,sha256=xlj0-TBQBhcKglllla_bQbufNiv10UFE0KsWMLvzFz4,123
-tests/baidu_spider/demo.py,sha256=MTEHkm7U4Kyx5QULCgR6to391xn4XPay6fmuV1c1uRc,24278
-tests/baidu_spider/items.py,sha256=vkbdnCw4tjYLmCL4oDIUxDNCgpYNqZHEG-6lVN3qfvI,443
-tests/baidu_spider/middleware.py,sha256=I71ZMmWTiDBFq4t2zfTE7IIXCqwaaeQ1DvKGW70q2Yg,1397
-tests/baidu_spider/pipeline.py,sha256=TUK_LnrU818UYmCn2_gKeNaTZjaj9qjrlndRLsR4wf0,1437
-tests/baidu_spider/request_fingerprints.txt,sha256=TJAuFJZZ_uvYExfruA9bEsIiArz86vxe95QoF2lbnfE,585
-tests/baidu_spider/run.py,sha256=YVe9qwn-2XBRRoZdUnwPRrWlBO5YAmKnyLRI3RpfogE,646
-tests/baidu_spider/settings.py,sha256=EenFOFgupwnn7HIySKSHBgP9--qxxkiWgIi2NDltXRw,2811
-tests/baidu_spider/spiders/__init__.py,sha256=eJ_ih4GiGfwQzPILeouy1Hnc4BrPz0KNPYlLHYvrvoc,123
-tests/baidu_spider/spiders/bai_du.py,sha256=pw4WccbmBR07CuSqCgm_7x9SH63FDJS_sXSaN5Ew5Tw,1589
-tests/baidu_spider/spiders/sina.py,sha256=BKQGJiCS8aiZ2f27C99WcK90QQJwgUY-vS4fUaQSdIQ,2456
-crawlo-1.0.4.dist-info/METADATA,sha256=dzEuRJVuBVSeKTQeEvOXRhfRcyjhcZqJFlPWivAZ9UE,1743
-crawlo-1.0.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.4.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
-crawlo-1.0.4.dist-info/top_level.txt,sha256=bKtfejkszFTNHm7Z6aqtt0AUG8DdeNeL4AoZsg4XdZY,13
-crawlo-1.0.4.dist-info/RECORD,,

crawlo-1.0.4.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-crawlo = crawlo.commands.cmdline:execute

tests/baidu_spider/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:05
-# @Author  :   oscar
-# @Desc    :   None
-"""