docpull 5.5.0__tar.gz → 5.5.1__tar.gz

{docpull-5.5.0/src/docpull.egg-info → docpull-5.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 5.5.0
+Version: 5.5.1
 Summary: Keep AI agents synced with changing public docs and web sources
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -115,7 +115,7 @@ Dynamic: license-file
 **Keep AI agents synced with changing public docs. Browser-free by default.**
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
-[![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
+[![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
 [![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
 [![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)

{docpull-5.5.0 → docpull-5.5.1}/README.md

@@ -3,7 +3,7 @@
 **Keep AI agents synced with changing public docs. Browser-free by default.**
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
-[![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
+[![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
 [![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
 [![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)

{docpull-5.5.0 → docpull-5.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "5.5.0"
+version = "5.5.1"
 dynamic = []
 description = "Keep AI agents synced with changing public docs and web sources"
 readme = {file = "README.md", content-type = "text/markdown"}

{docpull-5.5.0 → docpull-5.5.1}/src/docpull/__init__.py

@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "5.5.0"
+__version__ = "5.5.1"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .context_packs import (

{docpull-5.5.0 → docpull-5.5.1}/src/docpull/context_packs/search.py

@@ -95,6 +95,14 @@ def _build_local_search_pack(
     output_dir.mkdir(parents=True, exist_ok=True)
     local = search_local_pack(pack_dir, query, required_domains=required_domains, limit=max_results)
     results = _normalize_local_results(local)
+    replay_config = {
+        "query": query,
+        "provider": "local",
+        "pack_dir": str(pack_dir),
+        "include_domains": required_domains,
+        "max_results": max_results,
+        "scrape": scrape,
+    }
     result_path = output_dir / "search.result.json"
     results_path = output_dir / "search.results.ndjson"
     markdown_path = output_dir / "SEARCH.md"
@@ -120,6 +128,7 @@ def _build_local_search_pack(
         "output_dir": str(output_dir),
         "query": query,
         "input": {"pack_dir": str(pack_dir), "scrape": scrape},
+        "replay_config": replay_config,
         "summary": {
             "result_count": len(results),
             "max_results": max_results,
@@ -155,6 +164,7 @@ def _build_local_search_pack(
             "query": query,
             "summary": payload["summary"],
             "request_options": payload["request_options"],
+            "replay_config": replay_config,
             "artifacts": {**artifacts, "pack_metadata": artifact_ref(output_dir, pack_path)},
         },
     )
@@ -219,6 +229,17 @@ def _build_provider_search_pack(
         "max_results": max_results,
         "scrape": scrape,
     }
+    replay_config = {
+        "query": query,
+        "provider": provider,
+        "include_domains": include_domains,
+        "exclude_domains": exclude_domains,
+        "max_results": max_results,
+        "scrape": scrape,
+        "dry_run": dry_run,
+        "budget": budget,
+        "max_estimated_cost": max_estimated_cost,
+    }
     if dry_run or blocked:
         result_path = output_dir / "search.result.json"
         results_path = output_dir / "search.results.ndjson"
@@ -245,6 +266,7 @@ def _build_provider_search_pack(
             "status": "dry_run" if dry_run and not blocked else "blocked_by_budget",
             "output_dir": str(output_dir),
             "query": query,
+            "replay_config": replay_config,
             "summary": {
                 "result_count": 0,
                 "estimated_cost_usd": estimated,
@@ -283,6 +305,7 @@ def _build_provider_search_pack(
                 "query": query,
                 "summary": payload["summary"],
                 "request_options": request_options,
+                "replay_config": replay_config,
                 "artifacts": payload["artifacts"],
             },
         )
@@ -324,6 +347,7 @@ def _build_provider_search_pack(
         "status": "completed",
         "output_dir": str(output_dir),
         "query": query,
+        "replay_config": replay_config,
         "summary": {
             "result_count": _parallel_result_count(pack_path),
             "estimated_cost_usd": estimated,

{docpull-5.5.0 → docpull-5.5.1}/src/docpull/context_packs/visuals.py

@@ -375,11 +375,121 @@ def _run_screenshot_command(
         text=True,
         timeout=45,
     )
+    legacy_error = ""
+    if result.returncode == 0:
+        try:
+            png = _png_from_agent_browser_stdout(result.stdout)
+        except ContextPackError as err:
+            legacy_error = str(err)
+        else:
+            output_path.write_bytes(png)
+            return _screenshot_payload(
+                url=url,
+                output_path=output_path,
+                viewport=viewport,
+                full_page=full_page,
+                png=png,
+                command=[part if part != url else public_url(url) for part in command],
+            )
+    else:
+        legacy_error = (result.stderr or result.stdout).strip()[:500]
+    try:
+        png, command = _run_agent_browser_batch_screenshot(
+            binary=binary,
+            url=url,
+            output_path=output_path,
+            viewport=viewport,
+            full_page=full_page,
+            wait_for=wait_for,
+        )
+    except ContextPackError as err:
+        detail = f"{legacy_error}; batch fallback: {err}" if legacy_error else str(err)
+        raise ContextPackError(
+            f"agent-browser screenshot failed with status {result.returncode}: {detail}"
+        ) from err
+    return _screenshot_payload(
+        url=url,
+        output_path=output_path,
+        viewport=viewport,
+        full_page=full_page,
+        png=png,
+        command=command,
+    )
+
+
+def _run_agent_browser_batch_screenshot(
+    *,
+    binary: str,
+    url: str,
+    output_path: Path,
+    viewport: str,
+    full_page: bool,
+    wait_for: str,
+) -> tuple[bytes, list[str]]:
+    width, height = _viewport_dimensions(viewport)
+    wait_ms = {"domcontentloaded": "500", "load": "1000", "networkidle": "2000"}[wait_for]
+    screenshot_command = ["screenshot", str(output_path)]
+    if full_page:
+        screenshot_command.append("--full")
+    batch = [
+        ["open", url],
+        ["set", "viewport", str(width), str(height)],
+        ["wait", wait_ms],
+        screenshot_command,
+    ]
+    batch_json = json.dumps(batch, separators=(",", ":"))
+    command = [binary, "batch", "--bail", "--json"]
+    result = subprocess.run(  # nosec B603
+        command,
+        input=batch_json,
+        check=False,
+        capture_output=True,
+        text=True,
+        timeout=45,
+    )
     if result.returncode != 0:
         detail = (result.stderr or result.stdout).strip()[:500]
-        raise ContextPackError(f"agent-browser screenshot failed with status {result.returncode}: {detail}")
-    png = _png_from_agent_browser_stdout(result.stdout)
-    output_path.write_bytes(png)
+        raise ContextPackError(
+            f"agent-browser batch screenshot failed with status {result.returncode}: {detail}"
+        )
+    _validate_agent_browser_batch_stdout(result.stdout)
+    if not output_path.exists():
+        raise ContextPackError("agent-browser batch screenshot did not write the expected PNG path.")
+    png = output_path.read_bytes()
+    if not png.startswith(b"\x89PNG\r\n\x1a\n"):
+        raise ContextPackError("agent-browser batch screenshot did not write PNG data.")
+    public_batch = [[public_url(item) if item == url else item for item in step] for step in batch]
+    return png, [*command, json.dumps(public_batch, separators=(",", ":"))]
+
+
+def _viewport_dimensions(viewport: str) -> tuple[int, int]:
+    match = VIEWPORT_RE.fullmatch(viewport)
+    if not match:
+        raise ContextPackError("viewport must use WIDTHxHEIGHT with positive integer dimensions.")
+    return int(match.group("width")), int(match.group("height"))
+
+
+def _validate_agent_browser_batch_stdout(stdout: str) -> None:
+    try:
+        payload = json.loads(stdout)
+    except json.JSONDecodeError as err:
+        raise ContextPackError("agent-browser batch screenshot did not return JSON output.") from err
+    if not isinstance(payload, list):
+        raise ContextPackError("agent-browser batch screenshot JSON was not a command result array.")
+    for item in payload:
+        if not isinstance(item, dict) or item.get("success") is not True:
+            raise ContextPackError("agent-browser batch screenshot reported a failed command.")
+
+
+def _screenshot_payload(
+    *,
+    url: str,
+    output_path: Path,
+    viewport: str,
+    full_page: bool,
+    png: bytes,
+    command: list[str],
+) -> dict[str, Any]:
     return {
         "schema_version": CONTEXT_PACK_SCHEMA_VERSION,
         "url": url,
@@ -389,7 +499,7 @@ def _run_screenshot_command(
         "bytes": len(png),
         "sha256": hashlib.sha256(png).hexdigest(),
         "content_type": "image/png",
-        "command": [part if part != url else public_url(url) for part in command],
+        "command": command,
     }
 
 

{docpull-5.5.0 → docpull-5.5.1/src/docpull.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 5.5.0
+Version: 5.5.1
 Summary: Keep AI agents synced with changing public docs and web sources
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -115,7 +115,7 @@ Dynamic: license-file
 **Keep AI agents synced with changing public docs. Browser-free by default.**
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
-[![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
+[![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
 [![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
 [![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)

{docpull-5.5.0 → docpull-5.5.1}/tests/test_ci_policy.py

@@ -124,6 +124,32 @@ def test_publish_workflow_builds_artifact_before_unlocked_release_gates() -> Non
     assert "needs: [build, release-gates]" in publish_section
 
 
+def test_publish_workflow_creates_github_release_after_pypi_publish() -> None:
+    publish = (WORKFLOW_DIR / "publish.yml").read_text()
+    _, publish_section = publish.split("\n  publish:\n", 1)
+
+    assert "contents: write" in publish_section
+    assert "id-token: write" in publish_section
+    assert "pypa/gh-action-pypi-publish@release/v1" in publish_section
+    assert "name: Create GitHub release" in publish_section
+    assert "if: github.event_name == 'push' && github.ref_type == 'tag'" in publish_section
+    assert "GH_TOKEN: ${{ github.token }}" in publish_section
+    assert 'NOTES_FILE="docs/release-post-v${MINOR_VERSION}.md"' in publish_section
+    assert (
+        'gh release edit "$TAG" --title "$RELEASE_TITLE" --latest --notes-file "$NOTES_FILE"'
+        in publish_section
+    )
+    assert 'gh release edit "$TAG" --title "$RELEASE_TITLE" --latest' in publish_section
+    assert (
+        'gh release create "$TAG" --title "$RELEASE_TITLE" --verify-tag --latest --notes-file "$NOTES_FILE"'
+        in publish_section
+    )
+    assert (
+        'gh release create "$TAG" --title "$RELEASE_TITLE" --verify-tag --latest --generate-notes'
+        in publish_section
+    )
+
+
 def test_security_and_publish_bandit_scan_scripts() -> None:
     security = (WORKFLOW_DIR / "security.yml").read_text()
     publish = (WORKFLOW_DIR / "publish.yml").read_text()

{docpull-5.5.0 → docpull-5.5.1}/tests/test_context_packs.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import base64
 import json
 from pathlib import Path
 from types import SimpleNamespace
@@ -173,8 +174,11 @@ def test_search_pack_local_searches_existing_pack(tmp_path: Path) -> None:
     payload = build_search_pack("cited JSON", pack_dir=pack, output_dir=tmp_path / "search")
 
     assert payload["provider"] == "local"
+    assert payload["replay_config"]["pack_dir"] == str(pack)
     assert payload["summary"]["result_count"] >= 1
     assert payload["artifacts"]["accounting"] == "run.accounting.json"
+    written = json.loads((tmp_path / "search" / "search.pack.json").read_text(encoding="utf-8"))
+    assert written["replay_config"]["provider"] == "local"
     assert (tmp_path / "search" / "search.results.ndjson").exists()
     assert (tmp_path / "search" / "run.accounting.json").exists()
 
@@ -189,7 +193,10 @@ def test_search_pack_provider_dry_run_writes_pack_artifacts(tmp_path: Path) -> N
 
     assert payload["status"] == "dry_run"
     assert payload["output_dir"] == str((tmp_path / "search").resolve())
+    assert payload["replay_config"]["provider"] == "tavily"
     assert payload["artifacts"]["source_policy"] == "source_policy.json"
+    written = json.loads((tmp_path / "search" / "search.pack.json").read_text(encoding="utf-8"))
+    assert written["replay_config"]["dry_run"] is True
     assert (tmp_path / "search" / "search.pack.json").exists()
     assert (tmp_path / "search" / "SEARCH.md").exists()
     assert (tmp_path / "search" / "run.accounting.json").exists()
@@ -203,6 +210,58 @@ def test_screenshot_pack_validates_options_before_renderer_gate(tmp_path: Path)
         capture_screenshot_pack("https://acme.test", output_dir=tmp_path / "shot", wait_for="sleep")
 
 
+def test_screenshot_pack_falls_back_to_agent_browser_batch(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from docpull.context_packs import visuals
+
+    png = base64.b64decode(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR4nGNgYGBgAAAABQABDQottAAAAABJRU5ErkJggg=="
+    )
+    calls: list[dict[str, object]] = []
+
+    def fake_run(command: list[str], **kwargs: object) -> SimpleNamespace:
+        calls.append({"command": command, "kwargs": kwargs})
+        if "--timeout" in command:
+            return SimpleNamespace(
+                returncode=1,
+                stdout='{"error":"Unknown command: --timeout","success":false}',
+                stderr="",
+            )
+        assert command == ["agent-browser", "batch", "--bail", "--json"]
+        batch = json.loads(str(kwargs["input"]))
+        assert batch[0] == ["open", "https://acme.test/"]
+        assert batch[1] == ["set", "viewport", "800", "600"]
+        screenshot_path = Path(batch[-1][1])
+        screenshot_path.write_bytes(png)
+        return SimpleNamespace(
+            returncode=0,
+            stdout=json.dumps(
+                [{"command": step, "success": True, "result": {}} for step in batch[:-1]]
+                + [{"command": batch[-1], "success": True, "result": {"path": str(screenshot_path)}}]
+            ),
+            stderr="",
+        )
+
+    monkeypatch.setenv("DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS", "1")
+    monkeypatch.setattr(visuals.shutil, "which", lambda binary: f"/bin/{binary}")
+    monkeypatch.setattr(visuals.subprocess, "run", fake_run)
+
+    payload = capture_screenshot_pack(
+        "https://acme.test",
+        output_dir=tmp_path / "shot",
+        viewport="800x600",
+        agent_browser_binary="agent-browser",
+    )
+
+    assert payload["status"] == "completed"
+    assert payload["screenshots"][0]["bytes"] == len(png)
+    assert payload["screenshots"][0]["command"][1:4] == ["batch", "--bail", "--json"]
+    assert (tmp_path / "shot" / "screenshots" / "page.png").read_bytes() == png
+    assert len(calls) == 2
+
+
 @pytest.mark.parametrize(
     "argv",
     [