mineworker 0.4.0__py3-none-any.whl

mineworker/__about__.py

@@ -0,0 +1,3 @@
+"""单一版本号来源（hatchling 从此处读取，`mineworker.__version__` 亦转发于此）。"""
+
+__version__ = "0.4.0"

mineworker/__init__.py

@@ -0,0 +1,81 @@
+"""MineWorker —— 一个上手简单、结构清晰的 Python 爬虫框架（对标 feapder）。
+
+已提供：
+    - `setting`               分层配置（框架默认 ← 项目 setting.py ← 环境变量 MINEWORKER_*）
+    - `get_logger`            基于 loguru 的日志
+    - 异常层级                 MineWorkerError 及其子类
+    - `Request` / `Response`      网络层
+    - `AirSpider` / `BaseParser`  轻量单机运行时（含浏览器渲染、中间件、代理池、指标、告警）
+    - `Spider`                    Redis 分布式（多进程 / 多机 + 断点续爬，需 mineworker[redis]）
+    - `TaskSpider`                从任务源（Redis / DB）持续拉任务来爬
+    - `BatchSpider`               周期性批次采集（MySQL 任务表 + 批次记录 + 进度 / 防丢）
+    - `Item` / `UpdateItem`       结构化数据 + 管道落库 + 去重
+"""
+
+from __future__ import annotations
+
+from mineworker import setting
+from mineworker.__about__ import __version__
+from mineworker.core.base_parser import BaseParser
+from mineworker.core.spiders.air_spider import AirSpider
+from mineworker.core.spiders.batch_spider import BatchSpider
+from mineworker.core.spiders.spider import Spider
+from mineworker.core.spiders.task_spider import TaskSpider
+from mineworker.exceptions import (
+    ConfigError,
+    DedupError,
+    ItemError,
+    MineWorkerError,
+    NotRetryError,
+    PipelineError,
+    RequestError,
+    ResponseError,
+    SpiderError,
+    ValidationError,
+)
+from mineworker.network.item import Item, UpdateItem
+from mineworker.network.request import Request
+from mineworker.network.response import Response
+from mineworker.network.user_pool import (
+    GuestUserPool,
+    LocalUserPool,
+    RedisUserPool,
+    User,
+    UserPool,
+)
+from mineworker.utils.log import get_logger, log
+
+# 应用项目 setting.py 与环境变量覆盖，并按最终配置初始化日志
+setting.reload()
+get_logger()
+
+__all__ = [
+    "AirSpider",
+    "BaseParser",
+    "BatchSpider",
+    "ConfigError",
+    "DedupError",
+    "GuestUserPool",
+    "Item",
+    "ItemError",
+    "LocalUserPool",
+    "MineWorkerError",
+    "NotRetryError",
+    "PipelineError",
+    "RedisUserPool",
+    "Request",
+    "RequestError",
+    "Response",
+    "ResponseError",
+    "Spider",
+    "SpiderError",
+    "TaskSpider",
+    "UpdateItem",
+    "User",
+    "UserPool",
+    "ValidationError",
+    "__version__",
+    "get_logger",
+    "log",
+    "setting",
+]

mineworker/buffer/__init__.py

@@ -0,0 +1,8 @@
+"""RequestBuffer / ItemBuffer（阶段 02–03）：去重、批量入队、管道分发。"""
+
+from __future__ import annotations
+
+from mineworker.buffer.item_buffer import ItemBuffer
+from mineworker.buffer.request_buffer import RequestBuffer
+
+__all__ = ["ItemBuffer", "RequestBuffer"]

mineworker/buffer/item_buffer.py

@@ -0,0 +1,198 @@
+"""``ItemBuffer`` —— 收集 parse 产出的数据，批量去重后交给管道落库。
+
+流程：``put`` 累积 → 定时 / 满量 ``flush`` → 按 (表, 是否 UpdateItem, 管道) 分组
+→ Item 级去重（fingerprint）→ 逐管道 ``save_items`` / ``update_items``
+→ 成功则写去重指纹；失败则 dump 到 ``FAILED_ITEM_PATH``。
+
+给了 ``handler`` 时走调试快路径：直接把原始批次交给 handler，不去重、不落库。
+"""
+
+from __future__ import annotations
+
+import threading
+from collections import defaultdict
+from collections.abc import Callable
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, NamedTuple
+
+from mineworker import setting
+from mineworker.exceptions import ItemError
+from mineworker.network.item import Item, UpdateItem
+from mineworker.utils import stats as sk
+from mineworker.utils import tools
+from mineworker.utils.log import get_logger
+
+if TYPE_CHECKING:
+    from mineworker.dedup import Dedup
+    from mineworker.pipelines.base import BasePipeline
+    from mineworker.utils.stats import Stats
+
+log = get_logger("item_buffer")
+
+ItemHandler = Callable[[list[Any]], None]
+
+
+class _Norm(NamedTuple):
+    table: str
+    is_update: bool
+    data: dict[str, Any]
+    fingerprint: str | None
+    update_keys: tuple[str, ...]
+    pipelines: tuple[str, ...] | None
+
+
+def _normalize(obj: Any) -> _Norm:
+    if isinstance(obj, Item):
+        obj.pre_to_db()
+        is_update = isinstance(obj, UpdateItem)
+        keys = tuple(obj.update_key) if isinstance(obj, UpdateItem) else ()
+        fp = obj.fingerprint if setting.ITEM_FILTER_ENABLE else None
+        pipes = tuple(obj.pipelines) if obj.pipelines else None
+        return _Norm(obj.table_name, is_update, obj.to_dict(), fp, keys, pipes)
+    if isinstance(obj, dict):
+        return _Norm(setting.ITEM_DEFAULT_TABLE, False, obj, None, (), None)
+    raise ItemError(f"无法入库的类型：{type(obj)!r}（需要 Item 或 dict）")
+
+
+class ItemBuffer(threading.Thread):
+    def __init__(
+        self,
+        stats: Stats,
+        *,
+        handler: ItemHandler | None = None,
+        pipelines: list[str] | None = None,
+        dedup: Dedup | None = None,
+    ) -> None:
+        super().__init__(name="item-buffer", daemon=True)
+        self._stats = stats
+        self._handler = handler
+        self._pipeline_paths = pipelines
+        self._pipeline_cache: dict[tuple[str, ...], list[BasePipeline]] = {}
+        self._dedup = dedup
+        self._pending: list[Any] = []
+        self._lock = threading.Lock()
+        self._stop_event = threading.Event()
+
+    # ------------------------------------------------------------------
+    def put(self, item: Any) -> None:
+        with self._lock:
+            self._pending.append(item)
+            size = len(self._pending)
+        if size >= setting.ITEM_MAX_CACHED_COUNT:
+            self.flush()
+
+    def is_empty(self) -> bool:
+        with self._lock:
+            return not self._pending
+
+    def pending_count(self) -> int:
+        with self._lock:
+            return len(self._pending)
+
+    def run(self) -> None:
+        while not self._stop_event.wait(setting.BUFFER_FLUSH_INTERVAL):
+            self.flush()
+        self.flush()
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    def close(self) -> None:
+        for pipelines in self._pipeline_cache.values():
+            for pipeline in pipelines:
+                try:
+                    pipeline.close()
+                except Exception:
+                    log.exception("管道 {} close 异常", type(pipeline).__name__)
+        self._pipeline_cache.clear()
+
+    # ------------------------------------------------------------------
+    def flush(self) -> None:
+        with self._lock:
+            batch = self._pending
+            self._pending = []
+        if not batch:
+            return
+        if self._handler is not None:
+            self._handler(batch)
+            self._stats.incr(sk.ITEM, len(batch))
+            return
+        self._persist(batch)
+
+    def _persist(self, batch: list[Any]) -> None:
+        dedup = self._get_dedup()
+        seen: set[str] = set()
+        groups: dict[tuple[str, bool, tuple[str, ...] | None], list[_Norm]] = defaultdict(list)
+        for obj in batch:
+            norm = _normalize(obj)
+            if norm.fingerprint is not None and dedup is not None:
+                if norm.fingerprint in seen or dedup.get(norm.fingerprint):
+                    self._stats.incr(sk.ITEM_DEDUP_DROPPED)
+                    continue
+                seen.add(norm.fingerprint)
+            groups[(norm.table, norm.is_update, norm.pipelines)].append(norm)
+
+        for (table, is_update, pipe_paths), rows in groups.items():
+            datas = [row.data for row in rows]
+            update_keys = list(rows[0].update_keys)
+            pipelines = self._resolve_pipelines(pipe_paths)
+            ok = all(
+                self._write(pipeline, table, is_update, datas, update_keys)
+                for pipeline in pipelines
+            )
+            if ok:
+                self._stats.incr(sk.ITEM, len(datas))
+                if dedup is not None:
+                    for row in rows:
+                        if row.fingerprint is not None:
+                            dedup.add(row.fingerprint)
+            else:
+                self._dump_failed(table, datas)
+
+    # ------------------------------------------------------------------
+    def _get_dedup(self) -> Dedup | None:
+        if not setting.ITEM_FILTER_ENABLE:
+            return None
+        if self._dedup is None:
+            from mineworker.dedup import get_item_filter
+
+            self._dedup = get_item_filter()
+        return self._dedup
+
+    def _resolve_pipelines(self, paths: tuple[str, ...] | None) -> list[BasePipeline]:
+        key = (
+            paths
+            if paths is not None
+            else tuple(
+                self._pipeline_paths if self._pipeline_paths is not None else setting.ITEM_PIPELINES
+            )
+        )
+        cached = self._pipeline_cache.get(key)
+        if cached is None:
+            cached = [tools.load_object(path)() for path in key]
+            self._pipeline_cache[key] = cached
+        return cached
+
+    def _write(
+        self,
+        pipeline: BasePipeline,
+        table: str,
+        is_update: bool,
+        datas: list[dict[str, Any]],
+        update_keys: list[str],
+    ) -> bool:
+        try:
+            if is_update:
+                return pipeline.update_items(table, datas, update_keys)
+            return pipeline.save_items(table, datas)
+        except Exception:
+            log.exception("管道 {} 写入异常", type(pipeline).__name__)
+            return False
+
+    def _dump_failed(self, table: str, datas: list[dict[str, Any]]) -> None:
+        path = Path(setting.FAILED_ITEM_PATH)
+        with path.open("a", encoding="utf-8") as fh:
+            for data in datas:
+                fh.write(tools.dumps_json({"table": table, "data": data}) + "\n")
+        self._stats.incr(sk.ITEM_FAILED, len(datas))
+        log.error("[{}] {} 条数据写入失败，已 dump 到 {}", table, len(datas), path)

mineworker/buffer/request_buffer.py

@@ -0,0 +1,77 @@
+"""``RequestBuffer`` —— 收集 yield 出的 Request，去重后批量写入任务队列。"""
+
+from __future__ import annotations
+
+import threading
+from typing import TYPE_CHECKING, Any
+
+from mineworker import setting
+from mineworker.dedup import get_request_filter
+from mineworker.utils import stats as stats_keys
+
+if TYPE_CHECKING:
+    from mineworker.dedup import Filter
+    from mineworker.network.request import Request
+    from mineworker.utils.stats import Stats
+
+
+class RequestBuffer(threading.Thread):
+    def __init__(
+        self,
+        task_queue: Any,
+        stats: Stats,
+        *,
+        dedup: Filter | None = None,
+    ) -> None:
+        super().__init__(name="request-buffer", daemon=True)
+        self._queue = task_queue
+        self._stats = stats
+        self._dedup = dedup if dedup is not None else get_request_filter()
+        self._pending: list[Request] = []
+        self._lock = threading.Lock()
+        self._stop_event = threading.Event()
+
+    # ------------------------------------------------------------------
+    def put(self, request: Request) -> None:
+        with self._lock:
+            self._pending.append(request)
+        if len(self._pending) >= setting.REQUEST_BUFFER_MAX_CACHED:
+            self.flush()
+
+    def put_retry(self, request: Request) -> None:
+        """重试请求：跳过去重再次入队。"""
+        request.filter_repeat = False
+        self.put(request)
+
+    def is_empty(self) -> bool:
+        with self._lock:
+            return not self._pending
+
+    def pending_count(self) -> int:
+        with self._lock:
+            return len(self._pending)
+
+    # ------------------------------------------------------------------
+    def flush(self) -> None:
+        with self._lock:
+            batch = self._pending
+            self._pending = []
+        for request in batch:
+            if request.filter_repeat and not self._dedup.add(request.fingerprint):
+                self._stats.incr(stats_keys.DEDUP_DROPPED)
+                continue
+            self._queue.put(request)
+
+    def drain_pending(self) -> list[Request]:
+        with self._lock:
+            batch = self._pending
+            self._pending = []
+        return batch
+
+    def run(self) -> None:
+        while not self._stop_event.wait(setting.BUFFER_FLUSH_INTERVAL):
+            self.flush()
+        self.flush()
+
+    def stop(self) -> None:
+        self._stop_event.set()

mineworker/commands/__init__.py

@@ -0,0 +1,22 @@
+"""命令行（阶段 05）：typer 应用 + create / shell / retry 子命令。
+
+需要 ``pip install mineworker[cli]``（typer + jinja2）。
+"""
+
+from __future__ import annotations
+
+
+def main() -> None:
+    """``mineworker`` 控制台脚本入口。
+
+    脚本随核心包一起装，但 CLI 依赖只在 ``[cli]`` extra 里。缺依赖时给一句照着做就能
+    修好的提示，而不是甩一个 ``ModuleNotFoundError`` 堆栈给刚 ``pip install`` 完的人。
+    """
+    try:
+        from mineworker.commands.cmdline import main as _main
+    except ModuleNotFoundError as exc:  # pragma: no cover - 走到这里说明没装 [cli]
+        raise SystemExit(
+            f'mineworker 命令行需要额外依赖（缺少 "{exc.name}"）。\n'
+            '安装：pip install "mineworker[cli]"'
+        ) from exc
+    _main()

mineworker/commands/cmdline.py

@@ -0,0 +1,103 @@
+"""``mineworker`` 命令行入口（typer）。"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import typer
+
+from mineworker.__about__ import __version__
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help="MineWorker 爬虫框架命令行",
+)
+
+
+def _version(value: bool) -> None:
+    if value:
+        typer.echo(f"mineworker {__version__}")
+        raise typer.Exit
+
+
+@app.callback()
+def _root(
+    version: Annotated[
+        bool,
+        typer.Option("-V", "--version", callback=_version, is_eager=True, help="版本号"),
+    ] = False,
+) -> None:
+    pass
+
+
+@app.command()
+def create(
+    project: Annotated[str | None, typer.Option("-p", "--project", help="生成项目脚手架")] = None,
+    spider: Annotated[str | None, typer.Option("-s", "--spider", help="生成一个 AirSpider")] = None,
+    item: Annotated[str | None, typer.Option("-i", "--item", help="生成一个 Item")] = None,
+    table: Annotated[
+        str | None,
+        typer.Option("--table", help="配合 -i：读该 MySQL 表结构反射字段"),
+    ] = None,
+    mysql: Annotated[
+        str | None,
+        typer.Option("--mysql", help="MySQL 连接串 mysql://user:pass@host/db（默认取 setting）"),
+    ] = None,
+    setting_file: Annotated[bool, typer.Option("--setting", help="生成 setting.py")] = False,
+    force: Annotated[bool, typer.Option("-f", "--force", help="覆盖已存在文件")] = False,
+) -> None:
+    """生成项目 / 爬虫 / Item / 配置文件。"""
+    from mineworker.commands import create as gen
+
+    if project:
+        root = gen.create_project(project, force=force)
+        typer.echo(f"✓ 项目已生成 → cd {root} && python main.py")
+    elif spider:
+        typer.echo(f"✓ {gen.create_spider(spider, force=force)}")
+    elif item:
+        typer.echo(f"✓ {gen.create_item(item, force=force, table=table, mysql=mysql)}")
+    elif setting_file:
+        typer.echo(f"✓ {gen.create_setting(force=force)}")
+    else:
+        typer.echo("指定 -p / -s / -i / --setting 之一", err=True)
+        raise typer.Exit(1)
+
+
+@app.command()
+def shell(
+    url: Annotated[str, typer.Argument(help="要抓取的 URL")],
+    render: Annotated[bool, typer.Option("--render", help="用浏览器渲染")] = False,
+) -> None:
+    """抓一个页面并进入交互式 shell（变量 request / response）。"""
+    from mineworker.commands.shell import run_shell
+
+    run_shell(url, render=render)
+
+
+@app.command()
+def retry(
+    requests: Annotated[
+        bool, typer.Option("--requests", help="回放 failed_requests.jsonl")
+    ] = False,
+    items: Annotated[bool, typer.Option("--items", help="回放 failed_items.jsonl")] = False,
+) -> None:
+    """回放 dump 文件里的失败请求 / 数据（默认两者都回放）。"""
+    from mineworker.commands.retry import retry_items, retry_requests
+
+    if not requests and not items:
+        requests = items = True
+    if items:
+        ok, failed = retry_items()
+        typer.echo(f"failed_items：成功 {ok}，仍失败 {failed}")
+    if requests:
+        ok, failed = retry_requests()
+        typer.echo(f"failed_requests：恢复 {ok}，仍失败 {failed}")
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

mineworker/commands/create/__init__.py

@@ -0,0 +1,183 @@
+"""脚手架生成：项目 / 爬虫 / Item / setting（`mineworker create`）。"""
+
+from __future__ import annotations
+
+import importlib.resources
+import re
+from pathlib import Path
+from typing import Any
+
+from jinja2 import Environment, StrictUndefined
+
+from mineworker.utils.log import get_logger
+
+log = get_logger("create")
+
+_env = Environment(  # 生成 .py 文件而非 HTML，无需 autoescape
+    undefined=StrictUndefined,
+    keep_trailing_newline=True,
+    autoescape=False,
+)
+_CAMEL_BOUNDARY = re.compile(r"(?<!^)(?=[A-Z])")
+
+
+def _to_snake(name: str) -> str:
+    name = re.sub(r"[\s-]+", "_", name.strip())
+    return _CAMEL_BOUNDARY.sub("_", name).lower().strip("_")
+
+
+def _to_camel(name: str) -> str:
+    parts = re.split(r"[\s_-]+", name.strip())
+    if len(parts) == 1 and not parts[0].islower():
+        return parts[0][:1].upper() + parts[0][1:]
+    return "".join(p[:1].upper() + p[1:] for p in parts if p)
+
+
+def _render(relpath: str, /, **ctx: Any) -> str:
+    source = (
+        importlib.resources.files("mineworker.templates")
+        .joinpath(relpath)
+        .read_text(encoding="utf-8")
+    )
+    return _env.from_string(source).render(**ctx)
+
+
+def _write(path: Path, content: str, *, force: bool) -> None:
+    if path.exists() and not force:
+        raise FileExistsError(f"{path} 已存在（加 --force 覆盖）")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8")
+    log.info("生成 {}", path)
+
+
+def _spider_names(name: str) -> tuple[str, str]:
+    class_name = _to_camel(name)
+    if not class_name.endswith("Spider"):
+        class_name += "Spider"
+    return class_name, _to_snake(class_name)
+
+
+def _item_names(name: str) -> tuple[str, str, str]:
+    class_name = _to_camel(name)
+    if not class_name.endswith("Item"):
+        class_name += "Item"
+    module = _to_snake(class_name)
+    table = re.sub(r"_item$", "", module) or module
+    return class_name, module, table
+
+
+# ----------------------------------------------------------------------
+def create_spider(name: str, *, target_dir: Path | None = None, force: bool = False) -> Path:
+    class_name, module = _spider_names(name)
+    _, _, table = _item_names(name)
+    path = (target_dir or Path()) / f"{module}.py"
+    _write(
+        path,
+        _render("air_spider.py.jinja", class_name=class_name, table_name=table),
+        force=force,
+    )
+    return path
+
+
+def create_item(
+    name: str,
+    *,
+    target_dir: Path | None = None,
+    force: bool = False,
+    table: str | None = None,
+    mysql: Any = None,
+) -> Path:
+    class_name, module, derived_table = _item_names(name)
+    path = (target_dir or Path()) / f"{module}.py"
+
+    if table:
+        db = _open_mysql(mysql)
+        try:
+            fields, unique = _reflect_table(db, table)
+        finally:
+            if mysql is None or isinstance(mysql, str):
+                db.close()
+        content = _render(
+            "item_fields.py.jinja",
+            class_name=class_name,
+            table_name=table,
+            fields=fields,
+            unique_key=repr(unique),
+        )
+    else:
+        content = _render("item.py.jinja", class_name=class_name, table_name=derived_table)
+
+    _write(path, content, force=force)
+    return path
+
+
+def _open_mysql(mysql: Any) -> Any:
+    from mineworker.db.mysqldb import MysqlDB
+
+    if mysql is None:
+        return MysqlDB()
+    if isinstance(mysql, str):
+        return MysqlDB.from_url(mysql)
+    return mysql  # 已经是一个 db 对象（测试注入）
+
+
+def _reflect_table(db: Any, table: str) -> tuple[list[tuple[str, str]], list[str]]:
+    """读 ``SHOW FULL COLUMNS`` 返回 ``([(字段名, 注释), ...], [主键字段, ...])``。"""
+    rows = db.query(f"SHOW FULL COLUMNS FROM `{table}`")
+    if not rows:
+        raise ValueError(f"表 {table!r} 不存在或没有列")
+    fields = [(r["Field"], str(r.get("Comment") or "").replace("\n", " ").strip()) for r in rows]
+    unique = [r["Field"] for r in rows if str(r.get("Key")) == "PRI"]
+    return fields, unique
+
+
+def create_setting(*, target_dir: Path | None = None, force: bool = False) -> Path:
+    path = (target_dir or Path()) / "setting.py"
+    _write(path, _render("setting.py.jinja", project_name="mineworker"), force=force)
+    return path
+
+
+def create_project(name: str, *, force: bool = False) -> Path:
+    root = Path(name)
+    if root.exists() and any(root.iterdir()) and not force:
+        raise FileExistsError(f"{root} 已存在且非空（加 --force 覆盖）")
+
+    project = _to_snake(name)
+    spider_class, spider_module = _spider_names(name)
+
+    _write(
+        root / "main.py",
+        _render(
+            "project/main.py.jinja",
+            project_name=project,
+            spider_module=spider_module,
+            spider_class=spider_class,
+        ),
+        force=True,
+    )
+    _write(
+        root / "setting.py",
+        _render("setting.py.jinja", project_name=project),
+        force=True,
+    )
+    _write(
+        root / "README.md",
+        _render(
+            "project/README.md.jinja",
+            project_name=project,
+            spider_module=spider_module,
+        ),
+        force=True,
+    )
+    _write(root / "spiders" / "__init__.py", "", force=True)
+    _write(
+        root / "spiders" / f"{spider_module}.py",
+        _render(
+            "project/spider.py.jinja",
+            project_name=project,
+            spider_class=spider_class,
+        ),
+        force=True,
+    )
+    _write(root / "items" / "__init__.py", "", force=True)
+    return root