mineru-parser 0.1.0__py3-none-any.whl

mineru_parser/__init__.py

@@ -0,0 +1,41 @@
+"""MinerU Parser 公共 API。
+
+整个库对外只暴露一个函数 :func:`parse`（SPEC-004 AC-2.1 / AC-2.6）；
+其余导出都是 ``parse`` 的签名或返回值所引用的类型、异常类与版本常量。
+三个组件（client / downloader / extractor）是私有实现，不提供公开导入路径
+（SPEC-004 AC-2.3）。
+"""
+
+from __future__ import annotations
+
+from ._client.models import MinerUTaskStatus
+from ._extractor.limits import ExtractionLimits
+from ._extractor.models import ExtractedImage, ExtractedResult
+from ._telemetry import MetricHook
+from ._version import __version__
+from .errors import MinerUParserError
+from .pipeline import parse
+from .pipeline.errors import (
+    PipelineConfigurationError,
+    PipelineError,
+    PipelineTimeoutError,
+)
+from .pipeline.result import ErrorMessage, ParseResult
+from .pipeline.stages import PipelineStage
+
+__all__ = [
+    "ErrorMessage",
+    "ExtractedImage",
+    "ExtractedResult",
+    "ExtractionLimits",
+    "MetricHook",
+    "MinerUParserError",
+    "MinerUTaskStatus",
+    "ParseResult",
+    "PipelineConfigurationError",
+    "PipelineError",
+    "PipelineStage",
+    "PipelineTimeoutError",
+    "__version__",
+    "parse",
+]

mineru_parser/_client/__init__.py

@@ -0,0 +1,47 @@
+"""MinerU 客户端实现子包（内部实现，不对外）。
+
+本包是 SPEC-001 的私有实现，仅由 :mod:`mineru_parser.pipeline` 内部使用；
+不提供公开导入路径，也不遵循 SemVer。行为契约见 SPEC-001。
+"""
+
+from __future__ import annotations
+
+from ._telemetry import MetricHook
+from .api import MinerUApiClient, MinerUClient
+from .async_result import AsyncResult
+from .errors import (
+    MinerUAuthenticationError,
+    MinerUConfigurationError,
+    MinerUError,
+    MinerUExecutionError,
+    MinerUQuotaError,
+    MinerUTimeoutError,
+    MinerUTransientError,
+)
+from .models import (
+    MinerUProgress,
+    MinerUTaskResult,
+    MinerUTaskState,
+    MinerUTaskStatus,
+)
+from .polling import DEFAULT_POLL_POLICY, PollPolicy
+
+__all__ = [
+    "DEFAULT_POLL_POLICY",
+    "AsyncResult",
+    "MetricHook",
+    "MinerUApiClient",
+    "MinerUAuthenticationError",
+    "MinerUClient",
+    "MinerUConfigurationError",
+    "MinerUError",
+    "MinerUExecutionError",
+    "MinerUProgress",
+    "MinerUQuotaError",
+    "MinerUTaskResult",
+    "MinerUTaskState",
+    "MinerUTaskStatus",
+    "MinerUTimeoutError",
+    "MinerUTransientError",
+    "PollPolicy",
+]

mineru_parser/_client/_telemetry.py

@@ -0,0 +1,7 @@
+"""兼容导出：共享实现位于 :mod:`mineru_parser._telemetry`。"""
+
+from __future__ import annotations
+
+from .._telemetry import MetricHook, Telemetry
+
+__all__ = ["MetricHook", "Telemetry"]

mineru_parser/_client/_time.py

@@ -0,0 +1,13 @@
+"""时间辅助函数。"""
+
+from __future__ import annotations
+
+import time
+
+
+def elapsed_ms(started: float) -> float:
+    """返回自 ``started``（``time.perf_counter()``）以来的毫秒数。"""
+    return round((time.perf_counter() - started) * 1000, 3)
+
+
+__all__ = ["elapsed_ms"]

mineru_parser/_client/_validation.py

@@ -0,0 +1,161 @@
+"""配置参数校验：纯函数，不依赖网络、日志或客户端状态。"""
+
+from __future__ import annotations
+
+import math
+from urllib.parse import urlsplit
+
+from .errors import MinerUConfigurationError
+
+_HTTP_SCHEMES = frozenset({"http", "https"})
+
+
+def is_finite_number(value: object) -> bool:
+    """拒绝 NaN / Infinity：它们能通过 ``<= 0`` 比较，却会在 sleep
+    或 socket 超时处抛出未分类异常。"""
+    return not (isinstance(value, float) and not math.isfinite(value))
+
+
+def is_absolute_http_url(value: object) -> bool:
+    if not isinstance(value, str) or not value.strip():
+        return False
+    try:
+        parsed = urlsplit(value.strip())
+        _ = parsed.port  # 触发非法端口校验
+    except ValueError:
+        return False
+    return parsed.scheme.lower() in _HTTP_SCHEMES and bool(parsed.hostname)
+
+
+def validate_api_key(api_key: str | None) -> str | None:
+    if api_key is None:
+        return None
+    if not isinstance(api_key, str) or not api_key.strip():
+        raise MinerUConfigurationError("MinerU API key must not be blank.")
+    return api_key.strip()
+
+
+def validate_base_url(base_url: str) -> str:
+    if not isinstance(base_url, str) or not base_url.strip():
+        raise MinerUConfigurationError("MinerU base_url must not be empty.")
+    value = base_url.strip().rstrip("/")
+    try:
+        parsed = urlsplit(value)
+        _ = parsed.port  # 触发非法端口校验
+        hostname = parsed.hostname
+    except ValueError as exc:
+        raise MinerUConfigurationError("MinerU base_url is not a valid URL.") from exc
+    if parsed.scheme.lower() not in _HTTP_SCHEMES or not hostname:
+        raise MinerUConfigurationError(
+            "MinerU base_url must be an absolute http(s) URL."
+        )
+    if parsed.query or parsed.fragment:
+        raise MinerUConfigurationError(
+            "MinerU base_url must not contain a query or fragment."
+        )
+    if any(
+        ch.isspace() or ord(ch) < 32 or ord(ch) == 127 or ch == "\\" for ch in hostname
+    ):
+        raise MinerUConfigurationError("MinerU base_url contains an invalid host.")
+    return value
+
+
+def validate_request_timeout(
+    value: float | tuple[float, float],
+) -> float | tuple[float, float]:
+    if isinstance(value, bool):
+        raise MinerUConfigurationError("request_timeout must be positive.")
+    if isinstance(value, (int, float)):
+        if not is_finite_number(value) or value <= 0:
+            raise MinerUConfigurationError("request_timeout must be positive.")
+        return float(value)
+    if isinstance(value, (tuple, list)) and len(value) == 2:
+        connect, read = value
+        for item in (connect, read):
+            if (
+                isinstance(item, bool)
+                or not isinstance(item, (int, float))
+                or not is_finite_number(item)
+                or item <= 0
+            ):
+                raise MinerUConfigurationError(
+                    "request_timeout tuple values must both be positive."
+                )
+        return (float(connect), float(read))
+    raise MinerUConfigurationError(
+        "request_timeout must be a positive number or a (connect, read) tuple."
+    )
+
+
+def validate_positive(value: float, name: str) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not is_finite_number(value)
+    ):
+        raise MinerUConfigurationError(f"{name} must be a positive number.")
+    if value <= 0:
+        raise MinerUConfigurationError(f"{name} must be a positive number.")
+    return float(value)
+
+
+def validate_non_negative(value: float, name: str) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not is_finite_number(value)
+    ):
+        raise MinerUConfigurationError(f"{name} must be zero or positive.")
+    if value < 0:
+        raise MinerUConfigurationError(f"{name} must be zero or positive.")
+    return float(value)
+
+
+def validate_max_errors(value: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise MinerUConfigurationError(
+            "max_consecutive_errors must be an integer >= 1."
+        )
+    return value
+
+
+def validate_task_id(task_id: str) -> str:
+    if not isinstance(task_id, str) or not task_id.strip():
+        raise MinerUConfigurationError("task_id must be a non-empty string.")
+    return task_id.strip()
+
+
+def validate_deadline_epoch(value: float) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not is_finite_number(value)
+    ):
+        raise MinerUConfigurationError("deadline_epoch must be a finite number.")
+    return float(value)
+
+
+def validate_data_id(value: str | None) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value.strip():
+        raise MinerUConfigurationError("data_id must not be blank.")
+    value = value.strip()
+    if len(value) > 128:
+        raise MinerUConfigurationError("data_id must be at most 128 characters.")
+    return value
+
+
+__all__ = [
+    "is_absolute_http_url",
+    "is_finite_number",
+    "validate_api_key",
+    "validate_base_url",
+    "validate_data_id",
+    "validate_deadline_epoch",
+    "validate_max_errors",
+    "validate_non_negative",
+    "validate_positive",
+    "validate_request_timeout",
+    "validate_task_id",
+]

mineru_parser/_client/api.py

@@ -0,0 +1,233 @@
+"""``MinerUApiClient``：配置、凭证与结果句柄工厂。
+
+客户端本身不保存任务状态，也不执行轮询；``submit`` / ``resume`` 返回
+:class:`~mineru_parser._client.async_result.AsyncResult` 句柄，由句柄负责等待。
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import random
+import time
+from collections.abc import Callable
+
+import requests
+from pydantic import ValidationError
+
+from ._telemetry import MetricHook, Telemetry
+from ._time import elapsed_ms
+from ._validation import (
+    validate_api_key,
+    validate_base_url,
+    validate_data_id,
+    validate_request_timeout,
+    validate_task_id,
+)
+from .async_result import AsyncResult
+from .errors import (
+    MinerUAuthenticationError,
+    MinerUConfigurationError,
+    MinerUError,
+)
+from .models import (
+    ExtractionRequest,
+    MinerUTaskStatus,
+    normalize_status,
+)
+from .polling import (
+    DEFAULT_POLL_POLICY,
+    Poller,
+    PollPolicy,
+    PollRuntime,
+)
+from .transport import MINERU_BASE_URL, MinerUTransport
+
+
+class MinerUApiClient:
+    """MinerU API 的同步客户端。
+
+    客户端可以在多个任务之间共享。客户端自行创建并管理底层
+    ``requests.Session``（复用连接池）；也可以注入外部 Session，此时其
+    生命周期由调用方负责，``close()`` 不会关闭它。
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None = None,
+        base_url: str = MINERU_BASE_URL,
+        request_timeout: float | tuple[float, float] = 10.0,
+        poll_policy: PollPolicy | None = None,
+        session: requests.Session | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+        monotonic: Callable[[], float] = time.monotonic,
+        wall_clock: Callable[[], float] = time.time,
+        random_fn: Callable[[], float] = random.random,
+        on_metric: MetricHook | None = None,
+        correlation_id: str | None = None,
+    ) -> None:
+        self._api_key = validate_api_key(api_key)
+        self._base_url = validate_base_url(base_url)
+        self._request_timeout = validate_request_timeout(request_timeout)
+        if poll_policy is None:
+            poll_policy = DEFAULT_POLL_POLICY
+        if not isinstance(poll_policy, PollPolicy):
+            raise MinerUConfigurationError("poll_policy must be a PollPolicy instance.")
+        self._poll_policy = poll_policy
+        self._owns_session = session is None
+        self._session = session if session is not None else requests.Session()
+        self._runtime = PollRuntime(
+            sleep=sleep,
+            monotonic=monotonic,
+            wall_clock=wall_clock,
+            random_fn=random_fn,
+        )
+        self._telemetry = Telemetry(
+            on_metric=on_metric,
+            correlation_id=correlation_id,
+        )
+        self._transport = MinerUTransport(
+            base_url=self._base_url,
+            timeout=self._request_timeout,
+            session=self._session,
+            telemetry=self._telemetry,
+            wall_clock=wall_clock,
+        )
+
+    # --- 生命周期 ---------------------------------------------------------
+
+    def close(self) -> None:
+        """关闭客户端自建的 Session；注入的 Session 由调用方管理。"""
+        if self._owns_session:
+            self._session.close()
+
+    def __enter__(self) -> MinerUApiClient:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    # --- 只读配置 ---------------------------------------------------------
+
+    @property
+    def base_url(self) -> str:
+        return self._base_url
+
+    @property
+    def poll_policy(self) -> PollPolicy:
+        return self._poll_policy
+
+    # --- 提交 / 查询 ------------------------------------------------------
+
+    def submit(
+        self,
+        uri: str,
+        *,
+        model_version: str = "vlm",
+        data_id: str | None = None,
+    ) -> AsyncResult:
+        """提交任务，返回可等待的结果句柄。
+
+        只有“确定请求未发出”的错误或 HTTP 429 抛
+        :class:`MinerUTransientError`；其余错误（包括结果未知、HTTP 4xx/5xx）
+        抛 :class:`MinerUExecutionError`。``data_id`` 仅用于对账，不是幂等键。
+        """
+        api_key = self._resolve_api_key()
+
+        # 请求模型校验必须在 HTTP 异常处理块之外：Pydantic ValidationError
+        # 不能被误映射为 MinerUExecutionError。
+        try:
+            request = ExtractionRequest(
+                uri=uri,
+                model_version=model_version,
+                data_id=data_id,
+            )
+        except ValidationError as exc:
+            raise MinerUConfigurationError(
+                f"Invalid MinerU submit request: {exc}"
+            ) from exc
+
+        started = time.perf_counter()
+        try:
+            provider_task_id = self._transport.submit(request, api_key=api_key)
+        except MinerUError:
+            duration_ms = elapsed_ms(started)
+            self._telemetry.log(
+                "mineru.submit",
+                level=logging.WARNING,
+                data_id=request.data_id,
+                duration_ms=duration_ms,
+                outcome="error",
+            )
+            self._telemetry.metric("mineru.submit.total", 1, {"outcome": "error"})
+            self._telemetry.metric(
+                "mineru.submit.duration_ms",
+                duration_ms,
+                {"outcome": "error"},
+            )
+            raise
+
+        duration_ms = elapsed_ms(started)
+        self._telemetry.log(
+            "mineru.submit",
+            provider_task_id=provider_task_id,
+            data_id=request.data_id,
+            duration_ms=duration_ms,
+            outcome="success",
+        )
+        self._telemetry.metric("mineru.submit.total", 1, {"outcome": "success"})
+        self._telemetry.metric(
+            "mineru.submit.duration_ms",
+            duration_ms,
+            {"outcome": "success"},
+        )
+        return AsyncResult(
+            client=self,
+            task_id=provider_task_id,
+            data_id=request.data_id,
+        )
+
+    def resume(
+        self,
+        task_id: str,
+        *,
+        data_id: str | None = None,
+    ) -> AsyncResult:
+        """从已持久化的 task_id 恢复结果句柄，不发起网络请求。"""
+        return AsyncResult(
+            client=self,
+            task_id=validate_task_id(task_id),
+            data_id=validate_data_id(data_id),
+        )
+
+    # --- 内部 -------------------------------------------------------------
+
+    def _trace(self, task_id: str) -> MinerUTaskStatus:
+        """内部状态查询：仅供 ``AsyncResult.refresh()`` 调用。"""
+        validated = validate_task_id(task_id)
+        api_key = self._resolve_api_key()
+        detail = self._transport.trace(validated, api_key=api_key)
+        return normalize_status(validated, detail)
+
+    def _resolve_api_key(self) -> str:
+        key = self._api_key
+        if key is None:
+            key = os.getenv("MINERU_API_KEY")
+        if key is None or not key.strip():
+            raise MinerUAuthenticationError(
+                "MinerU API key is not configured; set MINERU_API_KEY."
+            )
+        return key.strip()
+
+    def _poller(self, policy: PollPolicy | None = None) -> Poller:
+        return Poller(
+            policy=policy if policy is not None else self._poll_policy,
+            runtime=self._runtime,
+            telemetry=self._telemetry,
+        )
+
+
+MinerUClient = MinerUApiClient
+
+__all__ = ["MetricHook", "MinerUApiClient", "MinerUClient"]