modao-prd-cli 0.1.0__py3-none-any.whl

modao_prd_cli/modao_prd/cli.py

@@ -0,0 +1,205 @@
+"""Command-line interface for extracting public Modao prototype shares."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import click
+
+from . import __version__
+from .browser import (
+    capture_page,
+    default_output_root,
+    doctor,
+    ensure_default_output_ignored,
+    validate_share_url,
+)
+from .evidence import EvidenceBundleWriter
+from .errors import ModaoPrdError
+from .extractor import build_document, summarize_document
+from .writers import write_outputs
+
+
+def _json_mode(ctx: click.Context, command_json: bool) -> bool:
+    return bool(command_json or (ctx.find_root().obj or {}).get("json", False))
+
+
+def _emit(data: Any, *, as_json: bool, message: str | None = None) -> None:
+    if as_json:
+        click.echo(json.dumps(data, ensure_ascii=False, indent=2, default=str))
+        return
+    if message:
+        click.echo(message)
+    if isinstance(data, dict):
+        _print_dict(data)
+    elif isinstance(data, list):
+        for item in data:
+            click.echo(f"- {item}")
+    else:
+        click.echo(str(data))
+
+
+def _print_dict(data: dict[str, Any], indent: int = 0) -> None:
+    prefix = "  " * indent
+    for key, value in data.items():
+        if isinstance(value, dict):
+            click.echo(f"{prefix}{key}:")
+            _print_dict(value, indent + 1)
+        elif isinstance(value, list):
+            click.echo(f"{prefix}{key}:")
+            for item in value:
+                if isinstance(item, dict):
+                    click.echo(f"{prefix}  -")
+                    _print_dict(item, indent + 2)
+                else:
+                    click.echo(f"{prefix}  - {item}")
+        else:
+            click.echo(f"{prefix}{key}: {value}")
+
+
+def _fail(exc: Exception, *, as_json: bool, url: str | None = None) -> None:
+    if isinstance(exc, ModaoPrdError):
+        payload = exc.as_dict()
+    elif isinstance(exc, FileExistsError):
+        payload = ModaoPrdError("output_exists", str(exc), url=url).as_dict()
+    elif isinstance(exc, ValueError):
+        payload = ModaoPrdError("invalid_input", str(exc), url=url).as_dict()
+    else:
+        payload = ModaoPrdError("runtime_error", str(exc), url=url).as_dict()
+    if as_json:
+        click.echo(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        click.echo(f"Error: {payload['error']['message']}", err=True)
+    raise click.exceptions.Exit(1)
+
+
+@click.group(invoke_without_command=True)
+@click.option("--json", "json_output", is_flag=True, help="输出机器可读 JSON。")
+@click.version_option(version=__version__, prog_name="modao-prd-cli")
+@click.pass_context
+def cli(ctx: click.Context, json_output: bool) -> None:
+    """将墨刀公开分享页提取为 Agent 易读取的结构化文档。"""
+
+    ctx.ensure_object(dict)
+    ctx.obj["json"] = json_output
+    if ctx.invoked_subcommand is None:
+        click.echo(ctx.get_help())
+
+
+@cli.command("doctor")
+@click.option("--json", "json_output", is_flag=True, help="输出机器可读 JSON。")
+@click.pass_context
+def doctor_cmd(ctx: click.Context, json_output: bool) -> None:
+    """检查 Python、Playwright 和浏览器运行环境。"""
+
+    _emit(doctor(), as_json=_json_mode(ctx, json_output))
+
+
+@cli.command("inspect")
+@click.argument("url")
+@click.option("--json", "json_output", is_flag=True, help="输出机器可读 JSON。")
+@click.option("--headed", is_flag=True, help="显示浏览器窗口，用于调试页面渲染。")
+@click.option("--timeout", type=click.IntRange(min=1), default=30, show_default=True, help="页面渲染超时时间（秒）。")
+@click.pass_context
+def inspect_cmd(ctx: click.Context, url: str, json_output: bool, headed: bool, timeout: int) -> None:
+    """读取 URL 并输出页面提取摘要，不生成文件。"""
+
+    as_json = _json_mode(ctx, json_output)
+    try:
+        raw = capture_page(url, headed=headed, timeout_seconds=timeout)
+        document = build_document(raw)
+        _emit(summarize_document(document), as_json=as_json, message="墨刀页面检查完成。")
+    except Exception as exc:
+        _fail(exc, as_json=as_json, url=url)
+
+
+@cli.command("export")
+@click.argument("url")
+@click.option("--format", "output_format", type=click.Choice(["json", "markdown", "ndjson", "all"]), default="all", show_default=True, help="导出格式。")
+@click.option("--output", "output_root", type=click.Path(file_okay=False, dir_okay=True, path_type=str), default=None, help="输出根目录；默认是当前项目根目录的 .modao-prd，项目 ID 会作为子目录。")
+@click.option("--headed", is_flag=True, help="显示浏览器窗口，用于调试页面渲染。")
+@click.option("--timeout", type=click.IntRange(min=1), default=30, show_default=True, help="页面渲染超时时间（秒）。")
+@click.option("--force", is_flag=True, help="覆盖已有 document 输出文件。")
+@click.option("--evidence", type=click.Choice(["essential", "full", "none"]), default="essential", show_default=True, help="证据采集级别；essential 适合需求分析，full 适合排障。")
+@click.option("--explore", type=click.Choice(["safe", "none"]), default="safe", show_default=True, help="安全状态探索策略。")
+@click.option("--max-states", type=click.IntRange(min=1), default=50, show_default=True, help="最多采集状态数。")
+@click.option("--max-depth", type=click.IntRange(min=0), default=3, show_default=True, help="安全探索最大深度。")
+@click.option("--max-actions", type=click.IntRange(min=0), default=200, show_default=True, help="最多尝试动作数。")
+@click.option("--max-duration", type=click.IntRange(min=1), default=120, show_default=True, help="整体采集上限（秒）。")
+@click.option("--max-item-mb", type=click.IntRange(min=0), default=10, show_default=True, help="单个证据文件大小上限（MiB）。")
+@click.option("--max-total-mb", type=click.IntRange(min=0), default=200, show_default=True, help="证据包大小上限（MiB）。")
+@click.option("--json", "json_output", is_flag=True, help="输出机器可读 JSON。")
+@click.pass_context
+def export_cmd(
+    ctx: click.Context,
+    url: str,
+    output_format: str,
+    output_root: str | None,
+    headed: bool,
+    timeout: int,
+    force: bool,
+    evidence: str,
+    explore: str,
+    max_states: int,
+    max_depth: int,
+    max_actions: int,
+    max_duration: int,
+    max_item_mb: int,
+    max_total_mb: int,
+    json_output: bool,
+) -> None:
+    """读取墨刀公开分享页并导出 JSON、Markdown 或 NDJSON。"""
+
+    as_json = _json_mode(ctx, json_output)
+    writer: EvidenceBundleWriter | None = None
+    try:
+        share = validate_share_url(url)
+        if output_root is None:
+            root_path = default_output_root()
+            ensure_default_output_ignored(root_path.parent)
+            root = str(root_path)
+        else:
+            root = output_root
+        output_dir = Path(root).expanduser().resolve() / share.project_id
+        if evidence != "none":
+            output_dir.parent.mkdir(parents=True, exist_ok=True)
+            writer = EvidenceBundleWriter(
+                output_dir,
+                max_item=max_item_mb * 1024 * 1024,
+                max_total=max_total_mb * 1024 * 1024,
+                force=force,
+            )
+        raw = capture_page(
+            url,
+            headed=headed,
+            timeout_seconds=timeout,
+            evidence_writer=writer,
+            evidence_mode=evidence,
+            explore=explore,
+            max_states=max_states,
+            max_depth=max_depth,
+            max_actions=max_actions,
+            max_duration_seconds=max_duration,
+            max_item=max_item_mb * 1024 * 1024,
+            max_total=max_total_mb * 1024 * 1024,
+        )
+        document = build_document(raw)
+        result = write_outputs(document, root, output_format, force=force, evidence_writer=writer)
+        writer = None
+        result["summary"] = summarize_document(document)
+        _emit(result, as_json=as_json, message="墨刀文档导出完成。")
+        if document.get("stats", {}).get("quality") == "partial":
+            raise click.exceptions.Exit(2)
+    except Exception as exc:
+        if writer is not None:
+            writer.close()
+        if isinstance(exc, click.exceptions.Exit):
+            raise
+        _fail(exc, as_json=as_json, url=url)
+
+
+def main() -> None:
+    cli(prog_name="modao-prd-cli")

modao_prd_cli/modao_prd/errors.py

@@ -0,0 +1,31 @@
+"""Stable, JSON-safe error types used by the CLI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ModaoPrdError(Exception):
+    """An expected failure with a stable machine-readable type."""
+
+    error_type: str
+    message: str
+    url: str | None = None
+    diagnostics: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        super().__init__(self.message)
+
+    def as_dict(self) -> dict[str, Any]:
+        error: dict[str, Any] = {
+            "type": self.error_type,
+            "message": self.message,
+        }
+        if self.url:
+            error["url"] = self.url
+        if self.diagnostics:
+            error["diagnostics"] = self.diagnostics
+        return {"error": error}
+

modao_prd_cli/modao_prd/evidence.py

@@ -0,0 +1,207 @@
+"""Safe, bounded and atomic evidence bundle writing."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import mimetypes
+import os
+import shutil
+import tempfile
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, BinaryIO
+
+from .models import EvidenceItem
+
+
+class EvidenceBundleWriter:
+    """Stage evidence files and atomically publish a manifest-backed bundle.
+
+    ``output_dir`` is the bundle directory. Files are accepted only as relative
+    paths, and an existing file may be replaced with ``force`` only when the
+    previous manifest identifies it as a generated evidence file.
+    """
+
+    def __init__(
+        self,
+        output_dir: str | Path,
+        *,
+        max_item: int = 10 * 1024 * 1024,
+        max_total: int = 200 * 1024 * 1024,
+        force: bool = False,
+        generated_by: str = "modao-prd-cli",
+    ) -> None:
+        if max_item < 0 or max_total < 0:
+            raise ValueError("size limits must be non-negative")
+        self.output_dir = Path(output_dir).expanduser().resolve()
+        self.max_item = max_item
+        self.max_total = max_total
+        self.force = force
+        self.generated_by = generated_by
+        self._items: list[EvidenceItem] = []
+        self._total = 0
+        self._old_generated = self._read_old_generated()
+        self._temp = Path(tempfile.mkdtemp(prefix=f".{self.output_dir.name}.", dir=str(self.output_dir.parent)))
+        self._closed = False
+
+    def add(
+        self,
+        relative_path: str | Path,
+        content: bytes | bytearray | memoryview | str | Path | BinaryIO,
+        *,
+        media_type: str | None = None,
+        enforce_item_limit: bool = True,
+    ) -> EvidenceItem:
+        """Stage one item and return its manifest record."""
+        if self._closed:
+            raise RuntimeError("bundle writer is already finalized")
+        try:
+            relative = self._validate_relative_path(relative_path)
+            path_key = self._path_key(relative)
+            if path_key == "manifest.json":
+                raise ValueError("manifest.json is reserved")
+            if any(self._path_key(item.path) == path_key for item in self._items):
+                raise ValueError(f"duplicate evidence path: {relative.as_posix()}")
+            destination = self._temp / relative
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            size = 0
+            digest = hashlib.sha256()
+            with destination.open("wb") as target:
+                for chunk in self._chunks(content):
+                    size += len(chunk)
+                    if (enforce_item_limit and size > self.max_item) or self._total + size > self.max_total:
+                        raise ValueError("evidence size limit exceeded")
+                    digest.update(chunk)
+                    target.write(chunk)
+            self._total += size
+            item = EvidenceItem(relative.as_posix(), digest.hexdigest(), size, media_type or self._media_type(relative))
+            self._items.append(item)
+            return item
+        except Exception:
+            # Keep the staged bundle alive so the caller can record a partial
+            # capture and still publish other evidence files. Finalization (or
+            # the caller's explicit close) performs the cleanup.
+            try:
+                if destination.exists():
+                    destination.unlink()
+            except UnboundLocalError:
+                pass
+            raise
+
+    def finalize(self, *, extra: dict[str, Any] | None = None) -> Path:
+        """Publish staged files and an atomically replaced ``manifest.json``."""
+        if self._closed:
+            raise RuntimeError("bundle writer is already finalized")
+        try:
+            self.output_dir.mkdir(parents=True, exist_ok=True)
+            existing_manifest = self.output_dir / "manifest.json"
+            self._assert_inside_output(existing_manifest)
+            existing_files = {self._path_key(path) for path in self._old_generated}
+            targets = []
+            for item in self._items:
+                target = self.output_dir / item.path
+                self._assert_inside_output(target)
+                if target.exists() and not (self.force and self._path_key(item.path) in existing_files):
+                    raise FileExistsError(f"evidence file already exists: {target}")
+                targets.append(target)
+
+            manifest: dict[str, Any] = {
+                "schema_version": "1.0",
+                "generated_by": self.generated_by,
+                "files": [asdict(item) for item in self._items],
+                "total_size": self._total,
+            }
+            if extra:
+                manifest["metadata"] = dict(extra)
+            for item in self._items:
+                target = targets.pop(0)
+                target.parent.mkdir(parents=True, exist_ok=True)
+                os.replace(self._temp / item.path, target)
+            manifest_tmp = self._temp / "manifest.json"
+            manifest_tmp.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+            os.replace(manifest_tmp, existing_manifest)
+            self._remove_stale_generated()
+            return existing_manifest
+        finally:
+            self.close()
+
+    def _read_old_generated(self) -> set[str]:
+        manifest = Path(self.output_dir) / "manifest.json"
+        try:
+            data = json.loads(manifest.read_text(encoding="utf-8"))
+        except (FileNotFoundError, OSError, ValueError):
+            return set()
+        if not isinstance(data, dict) or not isinstance(data.get("files"), list):
+            return set()
+        return {
+            str(entry["path"])
+            for entry in data.get("files", [])
+            if isinstance(entry, dict) and isinstance(entry.get("path"), str)
+        }
+
+    def _remove_stale_generated(self) -> None:
+        """Remove obsolete evidence files previously generated by this tool."""
+
+        if not self.force:
+            return
+        current = {self._path_key(item.path) for item in self._items}
+        for old_path in self._old_generated:
+            if self._path_key(old_path) in current or not old_path.casefold().startswith("evidence/"):
+                continue
+            target = self.output_dir / old_path
+            try:
+                self._assert_inside_output(target)
+                if target.is_file() or target.is_symlink():
+                    target.unlink()
+            except (FileNotFoundError, OSError, ValueError):
+                # A stale file is cleanup-only; never turn a successful bundle
+                # into a failed export because an old artifact disappeared.
+                continue
+
+    @staticmethod
+    def _path_key(path: str | Path) -> str:
+        return Path(path).as_posix().casefold()
+
+    def _assert_inside_output(self, path: Path) -> None:
+        try:
+            path.resolve(strict=False).relative_to(self.output_dir)
+        except ValueError as exc:
+            raise ValueError(f"evidence target escapes output directory: {path}") from exc
+
+    def close(self) -> None:
+        """Discard staged files and make the writer unusable."""
+        if not self._closed:
+            self._closed = True
+            shutil.rmtree(self._temp, ignore_errors=True)
+
+    def __enter__(self) -> "EvidenceBundleWriter":
+        if self._closed:
+            raise RuntimeError("bundle writer is already finalized")
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close()
+
+    @staticmethod
+    def _validate_relative_path(path: str | Path) -> Path:
+        candidate = Path(path)
+        if candidate.is_absolute() or not candidate.parts or ".." in candidate.parts:
+            raise ValueError("evidence path must be relative and cannot contain '..'")
+        return candidate
+
+    @staticmethod
+    def _chunks(content: bytes | bytearray | memoryview | str | Path | BinaryIO):
+        if isinstance(content, Path):
+            with Path(content).open("rb") as source:
+                yield from iter(lambda: source.read(1024 * 1024), b"")
+        elif isinstance(content, str):
+            yield content.encode("utf-8")
+        elif isinstance(content, (bytes, bytearray, memoryview)):
+            yield bytes(content)
+        else:
+            yield from iter(lambda: content.read(1024 * 1024), b"")
+
+    @staticmethod
+    def _media_type(path: Path) -> str:
+        return mimetypes.guess_type(path.name)[0] or "application/octet-stream"