qcrawler 0.4.3__py3-none-any.whl

qcrawl/__init__.py

@@ -0,0 +1,31 @@
+"""
+QCrawl - 分布式 Python 爬虫框架
+
+用法:
+    from qcrawl import QSpider, Request, ReportItem, send_seed
+"""
+
+from qcrawl.datalayer.report_item import ReportItem
+from qcrawl.debug import send_seed
+from qcrawl.engine import Engine
+from qcrawl.enums import DataType, ProjectId
+from qcrawl.request import Request
+from qcrawl.request_item import RequestItem
+from qcrawl.response import Response
+from qcrawl.settings import Settings
+from qcrawl.spider import QSpider
+
+__version__ = "0.4.3"
+
+__all__ = [
+    "QSpider",
+    "Request",
+    "RequestItem",
+    "Response",
+    "ReportItem",
+    "Engine",
+    "Settings",
+    "send_seed",
+    "DataType",
+    "ProjectId",
+]

qcrawl/cli.py

@@ -0,0 +1,461 @@
+"""
+QCrawl CLI 启动入口
+
+安装后直接使用:
+    qcrawl -s spiders.ithome.ithome_spider.IthomeSpider --env dev
+
+或以模块方式:
+    python -m qcrawl.cli -s spiders.ithome.ithome_spider.IthomeSpider --env dev
+
+选项:
+    -s, --spider        爬虫类路径（必填），如 spiders.ithome.ithome_spider.IthomeSpider
+    --env               运行环境: dev | prod（默认 dev）
+    --project_id        项目 ID（必填，标识所属项目）
+    --processes         工作进程数（默认 1）
+    --threads           每进程并发线程数（默认 4）
+    --proxy_type        代理模式: off | static | api | redis（默认 off）
+    --proxy_url         代理 API 地址 / Redis 连接串
+    --timeout           请求超时秒数（默认 15）
+    --retry             重试次数（默认 3）
+    --delay             请求间隔秒数（默认 0）
+    --http_client       HTTP 引擎: requests | curl | request-go（默认 requests）
+    --log_level         日志级别（默认 INFO）
+    --redis_url         Redis 连接串（种子队列 + 去重 + 代理配置）
+    --db_url            PostgreSQL 连接串（数据写入）
+    --seeds_file        种子文件路径（JSON/JSONL，启动时批量 LPUSH 到 Redis）
+    --settings          配置文件路径（默认 ./settings.yaml）
+    --local             本地模式（无需 Redis/PG，使用内存种子）
+    --migrate           仅执行表结构迁移（ALTER TABLE）
+"""
+
+import argparse
+import os
+import sys
+
+from loguru import logger
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="QCrawl 爬虫框架启动器")
+    parser.add_argument("-s", "--spider", required=True, help="爬虫类路径")
+    parser.add_argument("--env", default="dev", choices=["dev", "prod"], help="运行环境")
+    parser.add_argument("--project_id", default="", help="项目 ID")
+    parser.add_argument("--processes", type=int, default=None, help="工作进程数")
+    parser.add_argument("--threads", type=int, default=None, help="每进程并发线程数")
+    parser.add_argument("--proxy_type", default=None, help="代理模式")
+    parser.add_argument("--proxy_url", default=None, help="代理地址")
+    parser.add_argument("--timeout", type=int, default=None, help="请求超时秒数")
+    parser.add_argument("--retry", type=int, default=None, help="重试次数")
+    parser.add_argument("--delay", type=float, default=None, help="请求间隔秒数")
+    parser.add_argument("--http_client", default=None, help="HTTP 引擎")
+    parser.add_argument("--log_level", default=None, help="日志级别")
+    parser.add_argument("--redis_url", default=None, help="Redis 连接串")
+    parser.add_argument("--db_url", default=None, help="PostgreSQL 连接串")
+    parser.add_argument("--seeds_file", default=None, help="种子文件路径 (JSON/JSONL)")
+    parser.add_argument("--settings", default="settings.yaml", help="配置文件路径")
+    parser.add_argument("--local", action="store_true", help="本地模式（无需 Redis/PG）")
+    parser.add_argument("--migrate", action="store_true", help="仅执行表结构迁移")
+    return parser.parse_args()
+
+
+def main():
+    # 将当前工作目录加入 sys.path，确保能导入用户的 spiders 模块
+    cwd = os.getcwd()
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    args = parse_args()
+
+    # 1. 确定环境
+    env = args.env
+
+    # 2. 初始化日志
+    from qcrawl.utils.log import setup_logger
+    setup_logger(env=env, level=args.log_level)
+
+    # 3. 加载配置（四级优先级）
+    from qcrawl.settings import Settings
+    settings = Settings(env=env)
+    settings.load_defaults()  # 第 4 级
+    settings.load_yaml(args.settings)  # 第 3 级
+
+    # 4. 动态导入爬虫类
+    from qcrawl.utils.import_helper import import_class
+    spider_class = import_class(args.spider)
+
+    settings.load_spider(spider_class)  # 第 2 级
+
+    # 命令行参数（第 1 级）
+    cli_args = {
+        "project_id": args.project_id,
+        "processes": args.processes,
+        "threads": args.threads,
+        "proxy_type": args.proxy_type,
+        "proxy_url": args.proxy_url,
+        "timeout": args.timeout,
+        "retry_times": args.retry,
+        "delay": args.delay,
+        "http_client": args.http_client,
+        "log_level": args.log_level,
+        "redis_url": args.redis_url,
+        "db_url": args.db_url,
+    }
+    settings.load_cli(cli_args)
+
+    # 5. 获取 pod_ip
+    from qcrawl.utils.network import get_pod_ip
+    pod_ip = get_pod_ip()
+    settings._data["pod_ip"] = pod_ip
+
+    # 6. 判断运行模式
+    num_processes = settings.get("processes", 1)
+
+    if args.migrate:
+        _run_migrate(settings, spider_class)
+        return
+
+    if args.seeds_file:
+        _import_seeds_file(settings, args.seeds_file, spider_class)
+
+    if args.local:
+        # 本地模式：无需 Redis/PG，使用内存种子
+        _run_local(settings, spider_class)
+    elif num_processes > 1:
+        # 多进程模式
+        _run_multiprocess(settings, spider_class, num_processes)
+    else:
+        # 单进程模式（直接在工作进程中运行）
+        _run_worker(settings, spider_class)
+
+
+def _run_local(settings, spider_class):
+    """本地模式：无需 Redis/PG，使用内存种子"""
+    from qcrawl.engine import Engine
+
+    seeds = [{"seed_id": "cli-local-001"}]
+    engine = Engine(settings)
+    engine.add_crawl(spider_class, seeds=seeds)
+
+    spider_name = getattr(spider_class, "name", "") or spider_class.__name__
+    logger.info(
+        "🕷️ QCrawl 本地模式启动: spider={spider}, env={env}, threads={threads}",
+        spider=spider_name,
+        env=settings.get("env"),
+        threads=settings.get("threads"),
+    )
+    engine.start()
+
+
+def _run_multiprocess(settings, spider_class, num_processes: int):
+    """多进程模式：ProcessManager fork/spawn 工作进程"""
+    from qcrawl.process_manager import ProcessManager
+
+    pm = ProcessManager(
+        num_processes=num_processes,
+        worker_func=_worker_entry,
+        worker_args=(settings, spider_class),
+        max_restart=3,
+    )
+
+    logger.info(
+        "🕷️ QCrawl 多进程启动: spider={spider}, processes={procs}, threads={threads}",
+        spider=settings.get("project_id"),
+        procs=num_processes,
+        threads=settings.get("threads"),
+    )
+    pm.start()
+
+
+def _worker_entry(settings, spider_class):
+    """工作进程入口（模块级函数，spawn 模式可 pickle）"""
+    _run_worker(settings, spider_class)
+
+
+def _run_worker(settings, spider_class):
+    """
+    工作进程/单进程入口：
+    1. 连接 Redis（种子队列 + 去重）
+    2. 连接 PG（数据写入）
+    3. 初始化 Engine + Scheduler + Reporter + Metrics
+    4. engine.start()
+    """
+    from qcrawl.engine import Engine
+    from qcrawl.scheduler import Scheduler
+
+    redis_url = settings.get("redis_url", "")
+    db_url = settings.get("db_url", "")
+    project_id = settings.get("project_id", "")
+    pod_ip = settings.get("pod_ip", "")
+
+    # --- 连接 Redis ---
+    redis_client = None
+    scheduler = None
+    if redis_url:
+        try:
+            import redis as redis_lib
+            redis_client = redis_lib.Redis.from_url(
+                redis_url,
+                decode_responses=True,
+                socket_connect_timeout=5,
+                socket_timeout=30,  # 必须大于 BRPOP 阻塞时间(5s)
+            )
+            redis_client.ping()
+            logger.info("Redis 连接成功: {url}", url=redis_url)
+        except Exception as e:
+            logger.error("Redis 连接失败: {err}，退化为本地模式", err=str(e))
+            redis_client = None
+
+    # --- 连接 PG + 初始化 DataLayer ---
+    reporter = None
+    db_engine = None
+    if db_url:
+        try:
+            from qcrawl.datalayer import create_db_engine_from_settings, ensure_partition, ensure_table
+            from qcrawl.datalayer.reporter import Reporter
+
+            db_engine = create_db_engine_from_settings(settings)
+
+            # 获取表名
+            table_name = getattr(spider_class, "table", "") or f"{getattr(spider_class, 'name', 'data')}_data"
+
+            # 自动建表（支持可选分区）
+            partition_by_month = settings.get("partition_by_month", False)
+            ensure_table(db_engine, table_name, partition_by_month=partition_by_month)
+
+            # 分区模式：预建当月+下月分区
+            if partition_by_month:
+                from datetime import datetime, timezone
+                now = datetime.now(timezone.utc)
+                ensure_partition(db_engine, table_name, now.year, now.month)
+                # 下月
+                nm = now.month + 1
+                ny = now.year + (1 if nm > 12 else 0)
+                nm = nm if nm <= 12 else 1
+                ensure_partition(db_engine, table_name, ny, nm)
+
+            # 初始化 Reporter
+            reporter = Reporter(
+                engine=db_engine,
+                table_name=table_name,
+                project_id=project_id,
+                spider_name=getattr(spider_class, "name", "") or spider_class.__name__.lower(),
+                pod_ip=pod_ip,
+                batch_size=settings.get("batch_size", 100),
+                flush_interval=settings.get("flush_interval", 5.0),
+                max_buffer_size=settings.get("max_buffer_size", 10000),
+            )
+            logger.info("PostgreSQL 连接成功，目标表: {table}", table=table_name)
+        except Exception as e:
+            logger.error("PostgreSQL 连接失败: {err}，数据将打印到终端", err=str(e))
+            reporter = None
+
+    # --- 初始化 MinIO ---
+    minio_client = None
+    try:
+        from qcrawl.datalayer.minio_client import MinioClient
+        minio_client = MinioClient.from_settings(settings)
+        if minio_client:
+            logger.info("MinIO 配置已加载")
+    except Exception:
+        pass
+
+    # --- 初始化 Scheduler（需要 Redis + Spider 实例）---
+    spider_name = getattr(spider_class, "name", "") or spider_class.__name__.lower()
+    if redis_client:
+        # 创建临时 spider 实例用于 Scheduler（获取 spider_name）
+        tmp_spider = spider_class()
+        tmp_spider.project_id = project_id
+        tmp_spider.pod_ip = pod_ip
+        scheduler = Scheduler(
+            redis_client=redis_client,
+            spider=tmp_spider,
+            project_id=project_id,
+            pod_ip=pod_ip,
+            dedup_ttl=settings.get("dedup_ttl", 0),
+        )
+
+    # --- 初始化指标采集 ---
+    metrics = None
+    if redis_client:
+        from qcrawl.utils.metrics import MetricsCollector
+        metrics = MetricsCollector(
+            redis_client=redis_client,
+            project_id=project_id,
+            spider_name=spider_name,
+            pod_ip=pod_ip,
+            flush_interval=settings.get("metrics_flush_interval", 5.0),
+        )
+
+    # --- 关联 Reporter 与 Metrics（刷盘成功时通知指标采集器）---
+    if reporter and metrics:
+        reporter._metrics = metrics
+
+    # --- 初始化引擎 ---
+    engine = Engine(settings, scheduler=scheduler)
+    engine.set_reporter(reporter)
+    engine.set_metrics(metrics)
+    engine.set_minio_client(minio_client)
+    engine.set_redis_client(redis_client)
+
+    if scheduler:
+        engine.add_crawl(spider_class)
+        logger.info(
+            "🕷️ QCrawl 启动: spider={spider}, env={env}, threads={threads}, "
+            "project={pid}, mode=distributed",
+            spider=spider_name,
+            env=settings.get("env"),
+            threads=settings.get("threads"),
+            pid=project_id,
+        )
+    else:
+        # 退化为本地模式
+        seeds = [{"seed_id": "cli-local-001"}]
+        engine.add_crawl(spider_class, seeds=seeds)
+        logger.info(
+            "🕷️ QCrawl 启动: spider={spider}, env={env}, threads={threads}, "
+            "project={pid}, mode=local",
+            spider=spider_name,
+            env=settings.get("env"),
+            threads=settings.get("threads"),
+            pid=project_id,
+        )
+
+    engine.start()
+
+
+def _import_seeds_file(settings, seeds_file: str, spider_class):
+    """种子文件导入：读取 JSON/JSONL 文件，批量 LPUSH 到 Redis"""
+    import json
+    from pathlib import Path
+
+    redis_url = settings.get("redis_url", "")
+    if not redis_url:
+        logger.error("种子文件导入需要 Redis: 请指定 --redis_url")
+        return
+
+    file_path = Path(seeds_file)
+    if not file_path.exists():
+        logger.error("种子文件不存在: {path}", path=seeds_file)
+        return
+
+    spider_name = getattr(spider_class, "name", "") or spider_class.__name__.lower()
+    queue_key = f"qcrawl:seeds:{spider_name}"
+
+    try:
+        import redis as redis_lib
+        r = redis_lib.Redis.from_url(redis_url, decode_responses=True)
+        r.ping()
+    except Exception as e:
+        logger.error("种子导入 Redis 连接失败: {err}", err=str(e))
+        return
+
+    # 读取种子
+    seeds = []
+    content = file_path.read_text(encoding="utf-8").strip()
+
+    if file_path.suffix == ".jsonl" or "\n" in content[:200]:
+        # JSONL 格式：每行一个 JSON
+        for line in content.splitlines():
+            line = line.strip()
+            if line:
+                try:
+                    seeds.append(json.loads(line))
+                except json.JSONDecodeError:
+                    logger.warning("跳过无效 JSON 行: {line}", line=line[:80])
+    else:
+        # JSON 格式：单个对象或数组
+        data = json.loads(content)
+        if isinstance(data, list):
+            seeds = data
+        elif isinstance(data, dict):
+            seeds = [data]
+
+    if not seeds:
+        logger.warning("种子文件为空: {path}", path=seeds_file)
+        return
+
+    # 批量 LPUSH
+    pipe = r.pipeline(transaction=False)
+    for seed in seeds:
+        pipe.lpush(queue_key, json.dumps(seed, ensure_ascii=False))
+    pipe.execute()
+
+    logger.info(
+        "✅ 种子导入完成: {count} 条 -> {key}",
+        count=len(seeds),
+        key=queue_key,
+    )
+
+
+def _run_migrate(settings, spider_class):
+    """表结构迁移：ALTER TABLE 统一升级"""
+    db_url = settings.get("db_url", "")
+    if not db_url:
+        logger.error("迁移需要数据库: 请指定 --db_url")
+        return
+
+    table_name = getattr(spider_class, "table", "") or f"{getattr(spider_class, 'name', 'data')}_data"
+
+    try:
+        from qcrawl.datalayer import create_db_engine
+        from sqlalchemy import text
+
+        db_engine = create_db_engine(db_url)
+
+        with db_engine.connect() as conn:
+            # 检查表是否存在
+            result = conn.execute(text(
+                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :t)"
+            ), {"t": table_name})
+            exists = result.scalar()
+
+            if not exists:
+                # 表不存在，直接建表
+                from qcrawl.datalayer import ensure_table
+                ensure_table(db_engine, table_name)
+                logger.info("✅ 表不存在，已创建: {table}", table=table_name)
+                return
+
+            # 表已存在，检查并添加缺失列
+            migrations = [
+                ("s3_addr", "ALTER TABLE {t} ADD COLUMN IF NOT EXISTS s3_addr JSONB"),
+                ("url", "ALTER TABLE {t} ADD COLUMN IF NOT EXISTS url TEXT"),
+                ("spider_name", "ALTER TABLE {t} ADD COLUMN IF NOT EXISTS spider_name VARCHAR(64) NOT NULL DEFAULT ''"),
+                ("crawl_time", "ALTER TABLE {t} ADD COLUMN IF NOT EXISTS crawl_time TIMESTAMPTZ NOT NULL DEFAULT NOW()"),
+            ]
+
+            applied = 0
+            for col_name, sql in migrations:
+                try:
+                    conn.execute(text(sql.format(t=table_name)))
+                    conn.commit()
+                    applied += 1
+                except Exception:
+                    pass  # 列已存在
+
+            # 确保索引存在
+            indexes = [
+                f"CREATE INDEX IF NOT EXISTS idx_{table_name}_project ON {table_name}(project_id, data_type)",
+                f"CREATE INDEX IF NOT EXISTS idx_{table_name}_seed ON {table_name}(seed_id)",
+                f"CREATE INDEX IF NOT EXISTS idx_{table_name}_crawl ON {table_name}(crawl_time)",
+                f"CREATE INDEX IF NOT EXISTS idx_{table_name}_ext ON {table_name} USING GIN(ext)",
+            ]
+            for idx_sql in indexes:
+                try:
+                    conn.execute(text(idx_sql))
+                    conn.commit()
+                except Exception:
+                    pass
+
+            logger.info(
+                "✅ 迁移完成: table={table}, applied={n}",
+                table=table_name,
+                n=applied,
+            )
+
+    except Exception as e:
+        logger.error("迁移失败: {err}", err=str(e))
+
+
+if __name__ == "__main__":
+    main()

qcrawl/context.py

@@ -0,0 +1,53 @@
+"""运行时上下文（线程级）
+
+通过 threading.local 实现请求级 seed_id 的线程安全传递。
+Engine 在处理每个请求时设置上下文，report_data / logger 从中读取。
+"""
+
+from __future__ import annotations
+
+import threading
+
+# 线程局部存储
+_local = threading.local()
+
+
+def set_context(
+    seed_id: str = "",
+    spider_name: str = "",
+    project_id: str = "",
+    pod_ip: str = "",
+):
+    """设置当前线程的运行时上下文（Engine 在请求进入回调前调用）"""
+    _local.seed_id = seed_id
+    _local.spider_name = spider_name
+    _local.project_id = project_id
+    _local.pod_ip = pod_ip
+
+
+def get_seed_id() -> str:
+    """获取当前线程的 seed_id"""
+    return getattr(_local, "seed_id", "")
+
+
+def get_spider_name() -> str:
+    """获取当前线程的 spider_name"""
+    return getattr(_local, "spider_name", "")
+
+
+def get_project_id() -> str:
+    """获取当前线程的 project_id"""
+    return getattr(_local, "project_id", "")
+
+
+def get_pod_ip() -> str:
+    """获取当前线程的 pod_ip"""
+    return getattr(_local, "pod_ip", "")
+
+
+def clear_context():
+    """清除当前线程上下文（请求处理完毕后调用）"""
+    _local.seed_id = ""
+    _local.spider_name = ""
+    _local.project_id = ""
+    _local.pod_ip = ""

qcrawl/datalayer/__init__.py

@@ -0,0 +1,19 @@
+"""数据层"""
+
+from qcrawl.datalayer.database import (
+    create_db_engine,
+    create_db_engine_from_settings,
+    ensure_partition,
+    ensure_table,
+)
+from qcrawl.datalayer.report_item import ReportItem
+from qcrawl.datalayer.reporter import Reporter
+
+__all__ = [
+    "ReportItem",
+    "Reporter",
+    "create_db_engine",
+    "create_db_engine_from_settings",
+    "ensure_partition",
+    "ensure_table",
+]