crawlo 1.2.7__py3-none-any.whl → 1.2.9__py3-none-any.whl

tests/verify_distributed.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+分布式采集功能验证脚本
+验证Crawlo框架的分布式采集功能是否正常工作
+"""
+
+import redis
+import json
+import os
+import sys
+
+# 添加项目根目录到 Python 路径
+project_root = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, project_root)
+
+
+def verify_distributed_functionality():
+    """验证分布式采集功能"""
+    print("=== Crawlo分布式采集功能验证 ===\n")
+
+    # 1. 连接Redis
+    try:
+        r = redis.Redis(host='localhost', port=6379, db=2, decode_responses=False)
+        r.ping()
+        print("✓ Redis连接成功")
+    except Exception as e:
+        print(f"✗ Redis连接失败: {e}")
+        return False
+
+    # 2. 检查项目配置
+    try:
+        with open('../examples/ofweek_distributed/crawlo.cfg', 'r') as f:
+            config_content = f.read()
+            if 'ofweek_distributed.settings' in config_content:
+                print("✓ 项目配置文件正确")
+            else:
+                print("✗ 项目配置文件不正确")
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取配置文件: {e}")
+        return False
+
+    # 3. 检查设置文件
+    try:
+        with open('../examples/ofweek_distributed/ofweek_distributed/settings.py', 'r') as f:
+            settings_content = f.read()
+            checks = [
+                ('RUN_MODE = \'distributed\'', '运行模式设置为分布式'),
+                ('QUEUE_TYPE = \'redis\'', '队列类型设置为Redis'),
+                ('FILTER_CLASS = \'crawlo.filters.aioredis_filter.AioRedisFilter\'', '过滤器设置为Redis过滤器'),
+                ('REDIS_HOST = \'127.0.0.1\'', 'Redis主机配置正确'),
+            ]
+
+            all_passed = True
+            for check, description in checks:
+                if check in settings_content:
+                    print(f"✓ {description}")
+                else:
+                    print(f"✗ {description}")
+                    all_passed = False
+
+            if not all_passed:
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取设置文件: {e}")
+        return False
+
+    # 4. 检查Redis中的数据
+    try:
+        # 检查请求去重指纹
+        request_fingerprints = r.scard("crawlo:ofweek_distributed:filter:fingerprint")
+        print(f"✓ 请求去重指纹数量: {request_fingerprints}")
+
+        # 检查数据项去重指纹
+        item_fingerprints = r.scard("crawlo:ofweek_distributed:item:fingerprint")
+        print(f"✓ 数据项去重指纹数量: {item_fingerprints}")
+
+        # 检查请求队列
+        queue_size = r.zcard("crawlo:ofweek_distributed:queue:requests")
+        print(f"✓ 请求队列大小: {queue_size}")
+
+        # 验证数据是否存在
+        if request_fingerprints > 0 and item_fingerprints > 0:
+            print("✓ Redis中存在分布式采集数据")
+        else:
+            print("⚠ Redis中分布式采集数据为空")
+
+    except Exception as e:
+        print(f"✗ Redis数据检查失败: {e}")
+        return False
+
+    # 5. 检查输出文件
+    try:
+        import glob
+        json_files = glob.glob("output/*.json")
+        if json_files:
+            latest_file = max(json_files, key=os.path.getctime)
+            file_size = os.path.getsize(latest_file)
+            print(f"✓ 输出文件存在: {latest_file} ({file_size} bytes)")
+        else:
+            print("⚠ 未找到输出文件")
+    except Exception as e:
+        print(f"✗ 输出文件检查失败: {e}")
+
+    print("\n=== 验证结果 ===")
+    print("✓ Crawlo分布式采集功能正常工作!")
+    print("  - Redis连接正常")
+    print("  - 分布式配置正确")
+    print("  - Redis数据存储正常")
+    print("  - 采集任务执行正常")
+
+    return True
+
+
+if __name__ == '__main__':
+    verify_distributed_functionality()

crawlo/cleaners/__init__.py

@@ -1,61 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据清洗工具包
-"""
-
-from .text_cleaner import (
-    TextCleaner,
-    remove_html_tags,
-    decode_html_entities,
-    remove_extra_whitespace,
-    remove_special_chars,
-    normalize_unicode,
-    clean_text,
-    extract_numbers,
-    extract_emails,
-    extract_urls
-)
-
-from .data_formatter import (
-    DataFormatter,
-    format_number,
-    format_currency,
-    format_percentage,
-    format_phone_number,
-    format_chinese_id_card,
-    capitalize_words
-)
-
-from .encoding_converter import (
-    EncodingConverter,
-    detect_encoding,
-    to_utf8,
-    convert_encoding
-)
-
-__all__ = [
-    "TextCleaner",
-    "DataFormatter",
-    "EncodingConverter",
-    "remove_html_tags",
-    "decode_html_entities",
-    "remove_extra_whitespace",
-    "remove_special_chars",
-    "normalize_unicode",
-    "clean_text",
-    "extract_numbers",
-    "extract_emails",
-    "extract_urls",
-    "format_number",
-    "format_currency",
-    "format_percentage",
-    "format_phone_number",
-    "format_chinese_id_card",
-    "capitalize_words",
-    "detect_encoding",
-    "to_utf8",
-    "convert_encoding"
-]

crawlo/utils/date_tools.py

@@ -1,290 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-05-17 10:20
-# @Author  : crawl-coder
-# @Desc    : 智能时间工具库（专为爬虫场景设计）
-"""
-import dateparser
-from typing import Optional, Union, Literal
-from datetime import datetime, timedelta, timezone
-from dateutil.relativedelta import relativedelta
-import pytz
-from pytz import timezone as pytz_timezone
-
-# 支持的单位类型
-TimeUnit = Literal["seconds", "minutes", "hours", "days"]
-# 时间输入类型
-TimeType = Union[str, datetime]
-# 时区类型
-TimezoneType = Union[str, timezone, pytz_timezone]
-
-# 常见时间格式列表（作为 dateparser 的后备方案）
-COMMON_FORMATS = [
-    "%Y-%m-%d %H:%M:%S",
-    "%Y/%m/%d %H:%M:%S",
-    "%d-%m-%Y %H:%M:%S",
-    "%d/%m/%Y %H:%M:%S",
-    "%Y-%m-%d",
-    "%Y/%m/%d",
-    "%d-%m-%Y",
-    "%d/%m/%Y",
-    "%b %d, %Y",
-    "%B %d, %Y",
-    "%Y年%m月%d日",
-    "%Y年%m月%d日 %H时%M分%S秒",
-    "%a %b %d %H:%M:%S %Y",
-    "%a, %d %b %Y %H:%M:%S",
-    "%Y-%m-%dT%H:%M:%S.%f",
-    "%Y-%m-%dT%H:%M:%S",
-]
-
-
-class TimeUtils:
-    """
-    时间处理工具类，提供日期解析、格式化、计算等一站式服务。
-    特别适用于爬虫中处理多语言、多格式、相对时间的场景。
-    """
-
-    @staticmethod
-    def _try_strptime(time_str: str) -> Optional[datetime]:
-        """尝试使用预定义格式解析，作为 dateparser 的后备"""
-        for fmt in COMMON_FORMATS:
-            try:
-                return datetime.strptime(time_str.strip(), fmt)
-            except ValueError:
-                continue
-        return None
-
-    @classmethod
-    def parse(cls, time_input: TimeType, *, default: Optional[datetime] = None) -> Optional[datetime]:
-        """
-        智能解析时间输入（字符串或 datetime）。
-
-        :param time_input: 时间字符串（支持各种语言、格式、相对时间）或 datetime 对象
-        :param default: 解析失败时返回的默认值
-        :return: 解析成功返回 datetime，失败返回 default
-        """
-        if isinstance(time_input, datetime):
-            return time_input
-
-        if not isinstance(time_input, str) or not time_input.strip():
-            return default
-
-        # 1. 优先使用 dateparser（支持多语言和相对时间）
-        try:
-            parsed = dateparser.parse(time_input.strip())
-            if parsed:
-                return parsed
-        except Exception:
-            pass  # 忽略异常，尝试后备方案
-
-        # 2. 尝试使用常见格式解析
-        try:
-            parsed = cls._try_strptime(time_input)
-            if parsed:
-                return parsed
-        except Exception:
-            pass
-
-        return default
-
-    @classmethod
-    def format(cls, dt: TimeType, fmt: str = "%Y-%m-%d %H:%M:%S") -> Optional[str]:
-        """
-        格式化时间。
-
-        :param dt: datetime 对象或可解析的字符串
-        :param fmt: 输出格式
-        :return: 格式化后的字符串，失败返回 None
-        """
-        if isinstance(dt, str):
-            dt = cls.parse(dt)
-            if dt is None:
-                return None
-        try:
-            return dt.strftime(fmt)
-        except Exception:
-            return None
-
-    @classmethod
-    def to_timestamp(cls, dt: TimeType) -> Optional[float]:
-        """转换为时间戳（秒级）"""
-        if isinstance(dt, str):
-            dt = cls.parse(dt)
-            if dt is None:
-                return None
-        try:
-            return dt.timestamp()
-        except Exception:
-            return None
-
-    @classmethod
-    def from_timestamp(cls, ts: float) -> Optional[datetime]:
-        """从时间戳创建 datetime"""
-        try:
-            return datetime.fromtimestamp(ts)
-        except Exception:
-            return None
-
-    @classmethod
-    def diff(cls, start: TimeType, end: TimeType, unit: TimeUnit = "seconds") -> Optional[int]:
-        """
-        计算两个时间的差值（自动解析字符串）。
-
-        :param start: 起始时间
-        :param end: 结束时间
-        :param unit: 单位 ('seconds', 'minutes', 'hours', 'days')
-        :return: 差值（绝对值），失败返回 None
-        """
-        start_dt = cls.parse(start)
-        end_dt = cls.parse(end)
-        if not start_dt or not end_dt:
-            return None
-
-        delta = abs((end_dt - start_dt).total_seconds())
-
-        unit_map = {
-            "seconds": 1,
-            "minutes": 60,
-            "hours": 3600,
-            "days": 86400,
-        }
-        return int(delta // unit_map.get(unit, 1))
-
-    @classmethod
-    def add_days(cls, dt: TimeType, days: int) -> Optional[datetime]:
-        """日期加减（天）"""
-        dt = cls.parse(dt)
-        if dt is None:
-            return None
-        return dt + timedelta(days=days)
-
-    @classmethod
-    def add_months(cls, dt: TimeType, months: int) -> Optional[datetime]:
-        """日期加减（月）"""
-        dt = cls.parse(dt)
-        if dt is None:
-            return None
-        return dt + relativedelta(months=months)
-
-    @classmethod
-    def days_between(cls, dt1: TimeType, dt2: TimeType) -> Optional[int]:
-        """计算两个日期之间的天数差"""
-        return cls.diff(dt1, dt2, "days")
-
-    @classmethod
-    def is_leap_year(cls, year: int) -> bool:
-        """判断是否是闰年"""
-        return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)
-
-    @classmethod
-    def now(cls, fmt: Optional[str] = None) -> Union[datetime, str]:
-        """
-        获取当前时间。
-
-        :param fmt: 如果提供，则返回格式化字符串；否则返回 datetime 对象。
-        :return: datetime 或 str
-        """
-        dt = datetime.now()
-        if fmt is not None:
-            return dt.strftime(fmt)
-        return dt
-
-    @classmethod
-    def iso_format(cls, dt: TimeType) -> Optional[str]:
-        """返回 ISO 8601 格式字符串"""
-        dt = cls.parse(dt)
-        if dt is None:
-            return None
-        return dt.isoformat()
-
-    @classmethod
-    def to_timezone(cls, dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
-        """将时间转换为指定时区"""
-        dt = cls.parse(dt)
-        if dt is None:
-            return None
-        
-        try:
-            if isinstance(tz, str):
-                tz = pytz_timezone(tz)
-            return dt.astimezone(tz)
-        except Exception:
-            return None
-
-    @classmethod
-    def to_utc(cls, dt: TimeType) -> Optional[datetime]:
-        """将时间转换为 UTC 时区"""
-        return cls.to_timezone(dt, pytz.UTC)
-
-    @classmethod
-    def to_local(cls, dt: TimeType) -> Optional[datetime]:
-        """将时间转换为本地时区"""
-        return cls.to_timezone(dt, pytz.timezone("Asia/Shanghai"))
-
-    @classmethod
-    def from_timestamp_with_tz(cls, ts: float, tz: TimezoneType = None) -> Optional[datetime]:
-        """从时间戳创建 datetime，并可选择指定时区"""
-        try:
-            dt = datetime.fromtimestamp(ts)
-            if tz:
-                if isinstance(tz, str):
-                    tz = pytz_timezone(tz)
-                dt = dt.replace(tzinfo=tz)
-            return dt
-        except Exception:
-            return None
-
-
-# =======================对外接口=======================
-
-def parse_time(time_input: TimeType, default: Optional[datetime] = None) -> Optional[datetime]:
-    """解析时间字符串或对象"""
-    return TimeUtils.parse(time_input, default=default)
-
-
-def format_time(dt: TimeType, fmt: str = "%Y-%m-%d %H:%M:%S") -> Optional[str]:
-    """格式化时间"""
-    return TimeUtils.format(dt, fmt)
-
-
-def time_diff(start: TimeType, end: TimeType, unit: TimeUnit = "seconds") -> Optional[int]:
-    """计算时间差"""
-    return TimeUtils.diff(start, end, unit)
-
-
-def to_timestamp(dt: TimeType) -> Optional[float]:
-    """转时间戳"""
-    return TimeUtils.to_timestamp(dt)
-
-
-def to_datetime(ts: float) -> Optional[datetime]:
-    """从时间戳转 datetime"""
-    return TimeUtils.from_timestamp(ts)
-
-
-def now(fmt: Optional[str] = None) -> Union[datetime, str]:
-    """获取当前时间"""
-    return TimeUtils.now(fmt)
-
-def to_timezone(dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
-    """将时间转换为指定时区"""
-    return TimeUtils.to_timezone(dt, tz)
-
-def to_utc(dt: TimeType) -> Optional[datetime]:
-    """将时间转换为 UTC 时区"""
-    return TimeUtils.to_utc(dt)
-
-def to_local(dt: TimeType) -> Optional[datetime]:
-    """将时间转换为本地时区"""
-    return TimeUtils.to_local(dt)
-
-def from_timestamp_with_tz(ts: float, tz: TimezoneType = None) -> Optional[datetime]:
-    """从时间戳创建 datetime，并可选择指定时区"""
-    return TimeUtils.from_timestamp_with_tz(ts, tz)
-
-
-if __name__ == '__main__':
-    get_current_time = now(fmt="%Y-%m-%d %H:%M:%S")
-    print(get_current_time)

crawlo-1.2.7.dist-info/RECORD

@@ -1,209 +0,0 @@
-crawlo/__init__.py,sha256=jSOsZbDJ_Q5wZV8onSXx5LgNM7Z1q3zCROGdImBDr2I,1373
-crawlo/__version__.py,sha256=8fXL0hX3SI-tAETRl-m4ljrkccNA-ZNsta0Mg8Fv7aI,21
-crawlo/cli.py,sha256=AQnAB5NMI-Ic1VPw_Jjng8L4AI4-wMozOwzE6CfXkZU,2402
-crawlo/config.py,sha256=pQm5q4hFFW8DMpQzvlhrHKvZIzhbrDVa0PMQPbk6ybg,9818
-crawlo/config_validator.py,sha256=GsIgHM8LagVN_l5zWv8twXt7yiDyE6anH81Zjeaa7rQ,10134
-crawlo/crawler.py,sha256=avuaHyTbbUqpQfdodKvZ514h4o5MkE3P3n4dLBJ0WUI,41129
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
-crawlo/mode_manager.py,sha256=T34jxDwNSM3Zmpa-IB-nOxb8HzRROTTYpkmnzxEYrrk,7728
-crawlo/project.py,sha256=bYuAxsxNb-wxELuLDb1RoKd1yWQuTtJFab-mp95_t94,12356
-crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
-crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
-crawlo/task_manager.py,sha256=PScfEB03306Txa0l38AeQ_0WVhKzeWOFyT3bnrkbHW0,849
-crawlo/cleaners/__init__.py,sha256=lxL-ZWDKW-DdobdgKUQ27wNmBiUhGnD0CVG6HWkX3_o,1261
-crawlo/cleaners/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
-crawlo/cleaners/encoding_converter.py,sha256=G3khLlk0uBeTwIutsWxVUeSuyc1GMC1BDNJDwsU9ryg,4238
-crawlo/cleaners/text_cleaner.py,sha256=16e6WqIIb9qANMiK-vCEl4TvgkId19Aa2W1NMLU-jFQ,6707
-crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
-crawlo/commands/check.py,sha256=c2_paopV6S8PXSQoyZWyecYpKjuNe_-QXRs6jSouIo0,23443
-crawlo/commands/genspider.py,sha256=MccEP-JGeY4oafpN9yfKa87hZQIK0cee75QgK3a-kxM,5142
-crawlo/commands/help.py,sha256=ZtmpcLQOjRMYfsEAPgJjjF03d9dgY4VG4gS9gEszjYU,5133
-crawlo/commands/list.py,sha256=8gzQGbSzZ3HVciQZNUHEcJSRZ0ldoPTtXD4YPLMZluQ,5927
-crawlo/commands/run.py,sha256=iFpoBkaAxF4xAhaSAizntpVjf4vheB-d0m6Yc1Csz9E,12559
-crawlo/commands/startproject.py,sha256=1Y9bKR9G98hht2RtIZvSdSYIjIMkKAPIadfNXDZ_PjA,16696
-crawlo/commands/stats.py,sha256=_V5dtoYnTY76Zrt3-zolcfwgJv3v7yjfJuQ0B7t4Aj4,6315
-crawlo/commands/utils.py,sha256=6OVdbP8u7e3DF8daNs7KDA9zBEz6vX0MgGWOiBIZoW8,5273
-crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
-crawlo/core/engine.py,sha256=V5n2GQQfUnRJss_u3GIZtAfW5TySRWiGXgcqO3wsG-A,14682
-crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
-crawlo/core/scheduler.py,sha256=HBY48NSochkzzT5z9RmmNx0swOsILm33lHqRtGGqsYU,12549
-crawlo/data/__init__.py,sha256=UPqgioMdu3imSUmpLWzVlpvoBnEfaPSAT-crCcWd7iw,121
-crawlo/data/user_agents.py,sha256=uTASHTJJ-EZJ_A0W3WKo0KRZcRklOH9codkGTr8Qiiw,4805
-crawlo/downloader/__init__.py,sha256=8-r4_Wc_X64FJtKzNQamwsZsc428creKeFo19VxF33o,8565
-crawlo/downloader/aiohttp_downloader.py,sha256=ibyBCnjPdRJD73xOTgaDfenUTWNU-he9J0KzzHq4qrw,9064
-crawlo/downloader/cffi_downloader.py,sha256=U2CMYQ8ZyKMQNNPvyI2USqI-Dclq2d7ZRgp3LxLjTFs,10966
-crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
-crawlo/downloader/hybrid_downloader.py,sha256=dNnFeegRnyLaOxTWI6XrWKqqVPx80AZBZNgmrcKRVBM,8240
-crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
-crawlo/downloader/selenium_downloader.py,sha256=P8GuhEw6OYVeN3oeksuBLpUJCELXiu0mAR23X6IIOAA,21508
-crawlo/extension/__init__.py,sha256=jOdyLjtf-JqEKN67x2haIhtMhy_5bGSMbdFIdsERU7o,1633
-crawlo/extension/health_check.py,sha256=stDpyP4gOzAdbBlPbSf0rge0QounAhF8CtrGq5fa_7c,5657
-crawlo/extension/log_interval.py,sha256=2R3XVdM1grDN8wh9TTHRB_WmQypCr5YSGvESNDnS16s,2474
-crawlo/extension/log_stats.py,sha256=5CpcTHj0lCtFvhBTI3pG41gkiL_hPtv5FWY87L6EVjM,2989
-crawlo/extension/logging_extension.py,sha256=euecjvj71aK6dElMzYCuNXiNb9jO8qQmL5QspQ1xoz8,1669
-crawlo/extension/memory_monitor.py,sha256=fClPchpCkVjcIiU0AJHCKDd7HEiz5B4KqNqKTRZ2hcU,4394
-crawlo/extension/performance_profiler.py,sha256=BjWD3LOb4VwjQJQvQtWNg7GluEwFquI1CztNfgMzy3c,5032
-crawlo/extension/request_recorder.py,sha256=KA_RmcfscDxP5wPdolO76yKfRj-1jmHhG3jkVGO1pbc,4181
-crawlo/filters/__init__.py,sha256=lX-QOCDTiTRFoiK1qrZ5HABo7LgZfcxScx_lELYEvJk,4395
-crawlo/filters/aioredis_filter.py,sha256=aB1GPCALikvPUWdoACaGsvmnkzseKXxpR7l3gh1glsY,8479
-crawlo/filters/memory_filter.py,sha256=FzGJPhVKfZ8P23kP6de-VSfE8oVMjjpfWzKJIdiMtZU,9529
-crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
-crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
-crawlo/items/fields.py,sha256=l-DIwK6CCpdzNvf6ELz7Ckc7YCghZD9UCXA8vhNn2UE,1852
-crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
-crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
-crawlo/middleware/default_header.py,sha256=PHz_VWRZga1gqkpTGY_lDoG2c_ktt03dDacVKUzNu_I,5348
-crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
-crawlo/middleware/middleware_manager.py,sha256=kHqMk5Pm3tmUfXehw1LExGYKZfMH4C5bPK_yEQSVP6Q,6416
-crawlo/middleware/offsite.py,sha256=FZJwsWuAHfgojLfbuXmnyedGwEbo2BAcGQbmL2jOHxU,4140
-crawlo/middleware/proxy.py,sha256=FUwcXhUm-rH_zDVsIeTwC-qnoHZDogLkc9V7uSYwMvo,15606
-crawlo/middleware/request_ignore.py,sha256=7qdX4zAimjSGwdod_aWUbOTfzLBWZ5KzLVFchGMCxCI,2663
-crawlo/middleware/response_code.py,sha256=0_NbiCzLgJmSuSSF2_jqpypWYy0ES4GV-0iWQPLfYLc,5097
-crawlo/middleware/response_filter.py,sha256=tVGr06bfJBR3xAHI2G5c3WimFsGHu8qoJtDcsVuCATU,4384
-crawlo/middleware/retry.py,sha256=Acfo95B9wF8fQTCQIqluZOS2hHdnknQu_FOHvpGKJp0,4248
-crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
-crawlo/network/request.py,sha256=amZdRvDJvFpx82thtWwv6uo5uv9vtCwbvw3jAc3sCrY,12633
-crawlo/network/response.py,sha256=QwJhL3xJfPVy_gwtGrg61oAgaqCoCmjyj1Ug7Zju7Pg,13060
-crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=n0Ay7MtIEJ8L4Otiha4zRvI9toFUSNFugTNubi-Q3aw,5798
-crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
-crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
-crawlo/pipelines/database_dedup_pipeline.py,sha256=GTfnSk-l_Uuultsaf8QH-vkTLgWgXTLYgafpLG7baKY,8182
-crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=Buh_j5cup_DSI8Lh9RM4nS3YRTIXE9-_Xsnnlah7dbk,3943
-crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
-crawlo/pipelines/mysql_pipeline.py,sha256=Zb8PuJf8PK42dz1aeLjOlLZg26SLr-a_NtQY0hBaBfw,13790
-crawlo/pipelines/pipeline_manager.py,sha256=WzqJVq0F-5HTIzB49uam8wgZfXvytUaUYZWhDNbH5kA,2487
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=ib8TFfnIdhXOvQ8bjhKrumfzGnS9KIte-bdSutg2wO8,6403
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=qTFOuvEXsYEZbm0ULjsOeZo0XtSsZ-SHpx7nFEtmluE,1095
-crawlo/queue/queue_manager.py,sha256=S29mbE5-0wl56NHYmXErqOfIXeBIfbMnn1ljp38dhrg,15459
-crawlo/queue/redis_priority_queue.py,sha256=pX5H8-FIMsSCSFkTCI4JR0EPZ9tgkCBzCuwbMWGKFrU,13423
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=AI2mKQwBM1lcOiAW-5FpKvtZ0txqvLVdBWSiJ6I9eXs,9327
-crawlo/settings/setting_manager.py,sha256=uqgm2uZQp7Wx1hMSf2FnA0dh6f4jzZ13NQbRvIfYwsQ,3829
-crawlo/spider/__init__.py,sha256=xAH6NfE_6K2aY_VSL9DoGjcmMHJDd5Nxr7TG1Y8vQAE,21091
-crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
-crawlo/templates/run.py.tmpl,sha256=tRlFe-7QhVvKmbbeyVm53_Jse0NzZCDjMOfAgAs13BY,1144
-crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
-crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMFoFti_obJk,314
-crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
-crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
-crawlo/templates/project/settings.py.tmpl,sha256=LRqgrAXDOt_nrCbDVFBAaFYquNeAMGb-aQi9aA5Uppg,11006
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=zUNM5Hw8pTNIwud4tGCdoVTi1h2V4C4VDMlxJuAmzz0,6044
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=jOO_jtmc_O1adfRAKnep5uqnRBjvBirBb_SQ9r_OupU,3939
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=ovQW-pyMgC4TSTdObXbvOc8beCnjgCVUT7_TxYLpVQo,5036
-crawlo/templates/project/settings_simple.py.tmpl,sha256=lWnB9KEGpgZLcq3JMjklKYXFp2TJpAWgmEuz2vEIZIo,3850
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
-crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
-crawlo/tools/__init__.py,sha256=6BcoEE7t119VzdwqnULrWnYDNpXu86FHkWgx_WL1-Sk,3853
-crawlo/tools/anti_crawler.py,sha256=LwLC6BkxDSkxc5H1hQ6kY9j7O0PZGAMPZECr7gbqw2M,9431
-crawlo/tools/authenticated_proxy.py,sha256=ULCK0Cc9F2rGhRqu6kzKBdxzK9v2n1CsatSQ_PMxpAg,7272
-crawlo/tools/data_validator.py,sha256=bLWnkpFdclJuqjtSAgMI5nznN4vAuPwE1YaiFWKWenM,5490
-crawlo/tools/date_tools.py,sha256=dNm06OwJYQ7KeVkZoFcfFhMwJPsHlB3JJfSplOA0XfY,635
-crawlo/tools/distributed_coordinator.py,sha256=QM4ECyhTba9jtV7-AkOuT_lDJObeKVeBsRLZZFS_A7Y,12863
-crawlo/tools/retry_mechanism.py,sha256=VbHo9-pT8ADbwyPuKcRfeUo4BtVZQy6yvrSvugoiZRA,8021
-crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
-crawlo/utils/__init__.py,sha256=5unISyKR7pZdj7EwP5tadGYKvyb_W1uoC52YzqSbxis,592
-crawlo/utils/batch_processor.py,sha256=8LNy-K2SrQVUxmGEWxQyYw_j9M-erN4Ie7O4d3zpBvM,9142
-crawlo/utils/controlled_spider_mixin.py,sha256=tv-fyy5OLQr4FKlYSUWisVKmdQq5IR1GflTSwoDRq7g,16909
-crawlo/utils/date_tools.py,sha256=gsgHweZFOzeW-K91wRlBEg_XLKBCXl7UZJfnjI_W-J0,9197
-crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
-crawlo/utils/enhanced_error_handler.py,sha256=fJC__rnYNKTNUHNbgjZtT846HoE31qyGbPft9bwyYLU,14214
-crawlo/utils/env_config.py,sha256=W-VD_WF63DHxsyJysvp1eJwRh3L_pBRl_PitQAY3nQY,4079
-crawlo/utils/error_handler.py,sha256=N6suB8Utcn7tp6WRJ8gKECr0RIAG86dcOXdwOr998OE,4367
-crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
-crawlo/utils/large_scale_config.py,sha256=lsraHTAQx3sMPjTnCBY_SVIpkuIBUb3zD9eFvmccOOM,8440
-crawlo/utils/large_scale_helper.py,sha256=_0VgLvO10G2GeTQeNlL0K5vYjIMlTiX3geigM5Egtls,12477
-crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
-crawlo/utils/performance_monitor.py,sha256=2a3hFLQJnxgfbv_iMhsqPYOuX9BC4xQcicu_e3IVcrI,9861
-crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
-crawlo/utils/redis_connection_pool.py,sha256=SN-H_EZSn1jHfoeLNEhyZ3Fyw7j-SPBgx-oMnZ1zvEw,11417
-crawlo/utils/redis_key_validator.py,sha256=dOL_LVZJhzMZUZ3CmU57Yy0skWABeepI1GCecSzXstw,5825
-crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
-crawlo/utils/request_serializer.py,sha256=2Wd55pQDtBWnkhtxHxjzROQ_-IjqaqbT4qaTMtqO6R8,8754
-crawlo/utils/spider_loader.py,sha256=xNzQb7qhQ7TqZsfFtCLpuVcsGi-USriZosU0YSBr9II,2233
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
-crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
-examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
-tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md,sha256=4W6HlT9Uc3cyu77T9pfbkrMxpAZ-xq_L9MU-GbukLV0,3427
-tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
-tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
-tests/authenticated_proxy_example.py,sha256=b307_RybOtxXVQK0ITboLvHmOTwIN4yTF9aup4dYF7Q,8477
-tests/cleaners_example.py,sha256=tEGf-x8CNlXpKurgmNxTmmPjQa45dRUSBjnOJVGGSeU,5330
-tests/config_validation_demo.py,sha256=ah4cLZnozMdwKzACOoa5R52dWfya282WyulCGjadioM,3218
-tests/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
-tests/date_tools_example.py,sha256=XI3iFEzeo7Nb5YepK8WHytIaBegtxWVSISpqQgpV6M8,5042
-tests/dynamic_loading_example.py,sha256=7LdeQZFevrb-U1_dgr4oX3aYo2Da4HvE_0KIf1fw4Ew,18786
-tests/dynamic_loading_test.py,sha256=dzDW7b66HeDsIYsYgvNRihE3V6b6gEbUGQpp-eJbcIM,3413
-tests/env_config_example.py,sha256=_ZRDh_LR23ZKpy9E--y_KM0QIOiZF5vRT98QTn52TY8,4951
-tests/error_handling_example.py,sha256=sqvSoogDefyDoEBz3AVPG4pe49gRqBxT5BDFkijmYaA,5332
-tests/redis_key_validation_demo.py,sha256=R0ZTJJSeMB-ptVfzwegw7bw83FDDcR2XUNnUZqstSPE,4475
-tests/response_improvements_example.py,sha256=t1cbG3nesp82bqog4_ku1GvQzNbhRyWa5EaKTmOPrSk,5402
-tests/test_advanced_tools.py,sha256=HT_TcwfFzli-CavIJSqQqnCxnBn5FDMX09zL7AJ5tNY,5398
-tests/test_all_redis_key_configs.py,sha256=Uk91dnGwGaGsdukpvbTiXd4NnrsOa-THWD2cwSZkqag,5805
-tests/test_authenticated_proxy.py,sha256=lnvmQwuf0zaZP_E05EzcNFR2VJbwTkLjOmZGNoJKaC4,4339
-tests/test_cleaners.py,sha256=qyd20RNuBHIVHz7X5JjLwlIZedn7yHZ4uB3X78BpaF4,1819
-tests/test_comprehensive.py,sha256=dvRJeeVYc1cgXK9Y171hH9Y847zZpWSAFFH-EI3UepQ,5182
-tests/test_config_consistency.py,sha256=Y7R5SYSiPPFN088GyyBFfLTmKv38uQyElFwjsbLEY1w,2033
-tests/test_config_validator.py,sha256=Ec1h8Mw-fVz1d9JoATIWZb0nTc8pYjhTCSjPm3tvkTQ,6825
-tests/test_crawlo_proxy_integration.py,sha256=O0wK1u73qOuahRHWBXeDuMk5y1qMBMHLVVBYVP3U5SM,5576
-tests/test_date_tools.py,sha256=pcLDyhLrZ_jh-PhPm4CvLZEgNeH9kLMPKN5zacHwuWM,4053
-tests/test_default_header_middleware.py,sha256=v-ei_1EY7cvFSsySrQPXF5-DmyGsq2yzjYhhrwFMOXs,6003
-tests/test_double_crawlo_fix.py,sha256=qkJugMiEM2KQFHZ_Iza7Y0cER8p-Mecr57zjwHdsaSE,8033
-tests/test_double_crawlo_fix_simple.py,sha256=aPQQRex7ShxC_QJyjRkgelbn0Lnl3fFTwsPH5OglpwM,4807
-tests/test_download_delay_middleware.py,sha256=Idc6KzhL3hY3aDKgn1j_v5-mLIHz7dTnV5c4tJVZh5Q,9107
-tests/test_downloader_proxy_compatibility.py,sha256=DU19UnjhS3jdP1lfoF098bptKF4jpaowc4G274SgV8c,8915
-tests/test_dynamic_downloaders_proxy.py,sha256=t_aWpxOHi4h3_fg2ImtIq7IIJ0r3PTHtnXiopPe2ZlM,4450
-tests/test_dynamic_proxy.py,sha256=zi7Ocbhc9GL1zCs0XhmG2NvBBeIZ2d2hPJVh18lH4Y0,3172
-tests/test_dynamic_proxy_config.py,sha256=C_9CEjCJtrr0SxIXCyLDhSIi88ujF7UAT1F-FAphd0w,5853
-tests/test_dynamic_proxy_real.py,sha256=krWnbFIH26mWNPhOfPMmx3ZxJfOreZxMZFGwVb_8-K8,3511
-tests/test_edge_cases.py,sha256=kd5irijfGB0MkL-eukmc-WFm_xfbzQ30tPq-FMJaxsg,10856
-tests/test_enhanced_error_handler.py,sha256=cROcikmLvL11NgjCj-fGfVlKV3sLq7VN4DY_9U54IBU,8705
-tests/test_env_config.py,sha256=Qu1sDeADs69dSr1x0QmEe8nJrMHneE_4JClt-N901e8,4867
-tests/test_error_handler_compatibility.py,sha256=xJ43cmCwfBGh-qBwCGiMDPPlfNDLw4ZrmlrHN9IojkU,4241
-tests/test_final_validation.py,sha256=UNHMOkcOBx9jPdnYuYCF4Cx5GlXakBeHybOP27lpbAg,5078
-tests/test_framework_env_usage.py,sha256=bFb_ptdLeX2obdJWEqEHPWweiWR-wR2BpvEaJMQK7h4,4201
-tests/test_integration.py,sha256=l-G_z_btCtK0W4VvP1yNJFaZIZtKXDTeD1Ns2C-v6Nw,11677
-tests/test_item_dedup_redis_key.py,sha256=GIdyFBm56EbFzTvgnYcimgQFoMgvEZW8z1KFVUu3HJ4,3877
-tests/test_mode_consistency.py,sha256=3iFJAm0aUW_lUbJ2MkEHnAaF7OF6VNVLTiMyWZyO4Bs,1239
-tests/test_offsite_middleware.py,sha256=1DYktO_D-hiLEB6dBnc0iOvnWimqOdE6kimnS8aof_s,7764
-tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
-tests/test_performance.py,sha256=cjmWwIXBsHibgVyjyIt30iiXSDVse2HD6lFb0YlZsAQ,11613
-tests/test_proxy_api.py,sha256=SOBedDX8JTl3iPhluaCyybvA6B6k1poYkN7hY9gNpho,11029
-tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
-tests/test_proxy_middleware.py,sha256=EdQAfwwAJIBxw9JmUFTDEu_pdxapaTlcJr7KcrY6-AY,4021
-tests/test_proxy_middleware_enhanced.py,sha256=QR-p26F63N7MxNjZ2QJUeerh_xdnCDejkrGPIh7Fh4U,7035
-tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
-tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
-tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
-tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-tests/test_queue_manager_double_crawlo.py,sha256=5oPn3xnAXcZ2ZJweLTYEKDM-sQXQ0MZJdtjqD_Zmlrc,6995
-tests/test_queue_manager_redis_key.py,sha256=tvy3qkmB6XNpnJ4SOgjKvxE83hltCdL5Z32CupQ2VZ0,6454
-tests/test_real_scenario_proxy.py,sha256=dhlQcbiGd-zp2L9EJ12q7HfDxuEAyi_SxBSoXH6YyZM,8490
-tests/test_redis_config.py,sha256=Kbl3PURGNM1BUIspakEOA-ZOl2xxTHb_8KbftwjYOsg,921
-tests/test_redis_connection_pool.py,sha256=wLPmJ94jUajpShNrnnl4pTbX9ZIGqCEZgzzecasAC4s,9471
-tests/test_redis_key_naming.py,sha256=44A174UkKdj2ipjJa4N0RWIN8CYun6w08yylg8Yu9ac,6957
-tests/test_redis_key_validator.py,sha256=Yw0EsDQuI3Gzt_s86BubBP2VU_s7fyI-TthEX5J5C-o,4427
-tests/test_redis_queue.py,sha256=5LTc86A5qqF5VbmkvkF2OnLAxlJ7ClfJPw0dODxekFk,6846
-tests/test_request_ignore_middleware.py,sha256=QN81wgG_W_XfXCF9LvJNxCNwbOH6_tZnLIwLDTK2K5Q,6229
-tests/test_request_serialization.py,sha256=Jf7Kr7edL0ENwxh8ABa1W_O3dWyqNlvoSfQM1Mykpys,2359
-tests/test_response_code_middleware.py,sha256=wSe525bm-bk_iWMjPDzUu1LfOQrwJY8_MLKAspq2dzk,12193
-tests/test_response_filter_middleware.py,sha256=YWrGzJ7wmftTjJXcNTtJl3b3EdJsO4oR22ZLWwgErhg,16327
-tests/test_response_improvements.py,sha256=vNqHKyoEoYeEGAUiRzdsff2V6yvJ9QnDwGg7gmN38Ow,6028
-tests/test_retry_middleware.py,sha256=RmSYSf0GagGPGAVi5TXJWc0bZlmAI_hwFr2FYhvuKrk,8097
-tests/test_scheduler.py,sha256=UOzxyHK-ZuxlJ2Yj-j_zb8ntzmtoDUSZ4g2qot_2-Q8,8254
-tests/test_scheduler_config_update.py,sha256=JfzInmv1xkPJ8RpefbHMg9nLs0yqt4l1Md3Uz7JNDgE,4279
-tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
-tests/test_telecom_spider_redis_key.py,sha256=SUCUK_CgHgl1wj59o2a1g4qJpLUyW4reXv-5Bjuk6Ko,7675
-tests/test_template_content.py,sha256=URwjlAzMCdUN0sW_OupUcuSNMxp1OKgW79JOpkLPXnw,2965
-tests/test_template_redis_key.py,sha256=dOFutic8CL3tOzGbYhWbMrYiXZ8R3fhNoF5VKax5Iy0,4946
-tests/test_tools.py,sha256=fgzXL2L7eBV_nGjeMxH8IMhfc0dviQ80XgzZkJp_4dA,5266
-tests/tools_example.py,sha256=uXNS4xXJ-OD_xInAn2zjKLG_nlbgVGXZLoJtfhaG9lI,7926
-crawlo-1.2.7.dist-info/METADATA,sha256=z8I3gU3ejPRJLJ4AFKk7r_5uR7nS_6nv-Qr1JmOgNpA,19949
-crawlo-1.2.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.2.7.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.2.7.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.2.7.dist-info/RECORD,,