docpull 6.6.0__tar.gz → 6.8.0__tar.gz

{docpull-6.6.0/src/docpull.egg-info → docpull-6.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.6.0
+Version: 6.8.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -30,21 +30,20 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Natural Language :: English
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Typing :: Typed
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: html2text>=2020.1.16
 Requires-Dist: defusedxml>=0.7.1
 Requires-Dist: extruct>=0.15.0
-Requires-Dist: aiohttp>=3.14.3
+Requires-Dist: aiohttp>=3.14.4
 Requires-Dist: idna>=3.15
 Requires-Dist: regex>=2024.11.6
 Requires-Dist: rich>=13.0.0
@@ -61,7 +60,7 @@ Provides-Extra: tokens
 Requires-Dist: tiktoken>=0.7.0; extra == "tokens"
 Provides-Extra: mcp
 Requires-Dist: cryptography>=50.0.0; extra == "mcp"
-Requires-Dist: mcp<2,>=1.28.1; extra == "mcp"
+Requires-Dist: mcp<2,>=1.30.0; extra == "mcp"
 Requires-Dist: pydantic-settings>=2.14.2; extra == "mcp"
 Requires-Dist: pyjwt>=2.15.1; extra == "mcp"
 Requires-Dist: python-multipart>=0.0.27; extra == "mcp"
@@ -71,15 +70,15 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
 Provides-Extra: markitdown
-Requires-Dist: pypdf>=6.20.0; extra == "markitdown"
+Requires-Dist: pypdf>=6.21.1; extra == "markitdown"
 Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
 Provides-Extra: unstructured
-Requires-Dist: pypdf>=6.20.0; extra == "unstructured"
+Requires-Dist: pypdf>=6.21.1; extra == "unstructured"
 Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "unstructured"
 Provides-Extra: pdf
-Requires-Dist: pypdf>=6.20.0; extra == "pdf"
+Requires-Dist: pypdf>=6.21.1; extra == "pdf"
 Provides-Extra: parse
-Requires-Dist: pypdf>=6.20.0; extra == "parse"
+Requires-Dist: pypdf>=6.21.1; extra == "parse"
 Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
 Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "parse"
 Provides-Extra: presidio
@@ -96,14 +95,14 @@ Requires-Dist: url-normalize>=1.4.0; extra == "all"
 Requires-Dist: trafilatura>=1.12.0; extra == "all"
 Requires-Dist: tiktoken>=0.7.0; extra == "all"
 Requires-Dist: cryptography>=50.0.0; extra == "all"
-Requires-Dist: mcp<2,>=1.28.1; extra == "all"
+Requires-Dist: mcp<2,>=1.30.0; extra == "all"
 Requires-Dist: pydantic-settings>=2.14.2; extra == "all"
 Requires-Dist: pyjwt>=2.15.1; extra == "all"
 Requires-Dist: python-multipart>=0.0.27; extra == "all"
 Requires-Dist: starlette>=1.3.1; extra == "all"
 Requires-Dist: uvicorn>=0.30.0; extra == "all"
 Requires-Dist: pyarrow>=16.0.0; extra == "all"
-Requires-Dist: pypdf>=6.20.0; extra == "all"
+Requires-Dist: pypdf>=6.21.1; extra == "all"
 Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
 Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "all"
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
@@ -138,7 +137,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
 agents and retrieval pipelines. Use it when your application needs to know which
 sources it used, whether they changed, and how to rebuild the same context later.
 
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
 
@@ -230,9 +229,7 @@ Claude Code can register the same local server:
 claude mcp add --transport stdio docpull -- docpull mcp
 ```
 
-The Python stdio server is the supported release path. The TypeScript code formerly
-documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
-not part of the package contract.
+The Python stdio server is the supported MCP server.
 
 ## Limits and security boundary
 
@@ -259,6 +256,8 @@ extending acquisition behavior.
 - [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
 - [Public contracts](docs/contracts.md) — Schemas and versioning rules.
 - [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
+- [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
+- [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
 - [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
 - [Changelog](docs/CHANGELOG.md) — Release history.
 
@@ -278,8 +277,11 @@ release cycle.
 See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
 for current lifecycle and distribution links.
 
+Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
+spending, investments, assets, debts, and net worth in one current record.
+
 ## Project policies
 
-[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
+[Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
 [Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
 [MIT License](LICENSE)

{docpull-6.6.0 → docpull-6.8.0}/README.md

@@ -12,7 +12,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
 agents and retrieval pipelines. Use it when your application needs to know which
 sources it used, whether they changed, and how to rebuild the same context later.
 
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
 
@@ -104,9 +104,7 @@ Claude Code can register the same local server:
 claude mcp add --transport stdio docpull -- docpull mcp
 ```
 
-The Python stdio server is the supported release path. The TypeScript code formerly
-documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
-not part of the package contract.
+The Python stdio server is the supported MCP server.
 
 ## Limits and security boundary
 
@@ -133,6 +131,8 @@ extending acquisition behavior.
 - [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
 - [Public contracts](docs/contracts.md) — Schemas and versioning rules.
 - [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
+- [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
+- [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
 - [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
 - [Changelog](docs/CHANGELOG.md) — Release history.
 
@@ -152,8 +152,11 @@ release cycle.
 See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
 for current lifecycle and distribution links.
 
+Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
+spending, investments, assets, debts, and net worth in one current record.
+
 ## Project policies
 
-[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
+[Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
 [Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
 [MIT License](LICENSE)

{docpull-6.6.0 → docpull-6.8.0}/pyproject.toml

@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.6.0"
+version = "6.8.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [
@@ -71,7 +71,6 @@ classifiers = [
 
     # Programming Language
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -86,7 +85,7 @@ dependencies = [
     "html2text>=2020.1.16",
     "defusedxml>=0.7.1",
     "extruct>=0.15.0",
-    "aiohttp>=3.14.3",  # Security floor for the 2026 request-parsing advisories
+    "aiohttp>=3.14.4",  # Security floor for the 2026 request-parsing advisories
     "idna>=3.15",  # Security floor for transitive URL/IDNA handling; Renovate raises as needed
     "regex>=2024.11.6",
     "rich>=13.0.0",
@@ -110,7 +109,7 @@ tokens = [
 ]
 mcp = [
     "cryptography>=50.0.0",
-    "mcp>=1.28.1,<2",
+    "mcp>=1.30.0,<2",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.15.1",
     "python-multipart>=0.0.27",
@@ -123,18 +122,18 @@ parquet = [
     "pyarrow>=16.0.0",
 ]
 markitdown = [
-    "pypdf>=6.20.0",
+    "pypdf>=6.21.1",
     "markitdown[docx,pdf]>=0.1.0",
 ]
 unstructured = [
-    "pypdf>=6.20.0",
+    "pypdf>=6.21.1",
     "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
 ]
 pdf = [
-    "pypdf>=6.20.0",
+    "pypdf>=6.21.1",
 ]
 parse = [
-    "pypdf>=6.20.0",
+    "pypdf>=6.21.1",
     "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
 ]
@@ -156,14 +155,14 @@ all = [
     "trafilatura>=1.12.0",
     "tiktoken>=0.7.0",
     "cryptography>=50.0.0",
-    "mcp>=1.28.1,<2",
+    "mcp>=1.30.0,<2",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.15.1",
     "python-multipart>=0.0.27",
     "starlette>=1.3.1",
     "uvicorn>=0.30.0",
     "pyarrow>=16.0.0",
-    "pypdf>=6.20.0",
+    "pypdf>=6.21.1",
     "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
     "presidio-analyzer>=2.2.0",
@@ -213,19 +212,22 @@ constraint-dependencies = [
     "multidict>=6.9.1",
     "accelerate>=1.15.0",
     "soupsieve>=2.9",
-    "virtualenv>=21.7.13",
+    "virtualenv>=21.14.9",
 ]
 
 [tool.ruff]
 line-length = 110
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM"]
-ignore = ["A003"]  # Allow 'type' and 'format' as field names in data models
+ignore = [
+    "A003",  # Allow 'type' and 'format' as field names in data models
+    "UP042",  # StrEnum changes str() output for existing str, Enum classes
+]
 
 [tool.mypy]
-python_version = "3.10"
+python_version = "3.11"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{docpull-6.6.0 → docpull-6.8.0}/src/docpull/__init__.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any
 
 from .surface import PUBLIC_SDK_EXPORTS
 
-__version__ = "6.6.0"
+__version__ = "6.8.0"
 
 _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
     "query_context": (".context_query", "query_context"),

docpull-6.8.0/src/docpull/_jsonutil.py

@@ -0,0 +1,48 @@
+"""Shared JSON file and value helpers for internal modules."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+_SLUG_RE = re.compile(r"[^a-zA-Z0-9]+")
+
+
+def read_json(path: Path, *, error: type[Exception], required: bool = True, default: Any = None) -> Any:
+    """Load JSON from ``path``; a missing optional file returns ``default``."""
+    if not path.exists():
+        if required:
+            raise error(f"Missing required file: {path}")
+        return default
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as err:
+        raise error(f"Invalid JSON in {path}: {err}") from err
+
+
+def write_json(path: Path, payload: Any, *, sort_keys: bool = False) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    text = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=sort_keys)
+    path.write_text(text + "\n", encoding="utf-8")
+
+
+def dict_value(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def list_value(value: Any) -> list[Any]:
+    return value if isinstance(value, list) else []
+
+
+def string_list(value: Any) -> list[str]:
+    """Return the non-blank, stripped items of a list; anything else is empty."""
+    if not isinstance(value, list):
+        return []
+    return [str(item).strip() for item in value if str(item).strip()]
+
+
+def slugify(value: str, *, fallback: str) -> str:
+    slug = _SLUG_RE.sub("-", value).strip("-").lower()
+    return slug[:80].strip("-") or fallback

{docpull-6.6.0 → docpull-6.8.0}/src/docpull/accounting.py

@@ -114,39 +114,6 @@ def parse_budget_value(value: str) -> float:
     return parsed
 
 
-def extract_budget_flags(argv: list[str] | None) -> tuple[list[str] | None, float | None, bool]:
-    """Strip cross-command ``--budget`` and ``--explain-route`` flags.
-
-    This supports subcommands where adding a parent argparse option would force
-    users to place the flag before the subcommand.
-    """
-    if argv is None:
-        return None, None, False
-    cleaned: list[str] = []
-    budget: float | None = None
-    explain_route = False
-    index = 0
-    while index < len(argv):
-        item = argv[index]
-        if item == "--explain-route":
-            explain_route = True
-            index += 1
-            continue
-        if item == "--budget":
-            if index + 1 >= len(argv):
-                raise ValueError("--budget requires a value")
-            budget = parse_budget_value(argv[index + 1])
-            index += 2
-            continue
-        if item.startswith("--budget="):
-            budget = parse_budget_value(item.split("=", 1)[1])
-            index += 1
-            continue
-        cleaned.append(item)
-        index += 1
-    return cleaned, budget, explain_route
-
-
 def paid_action_blocked(
     budget_limit_usd: float | None,
     *,

{docpull-6.6.0 → docpull-6.8.0}/src/docpull/cache/frontier.py

@@ -89,6 +89,7 @@ class FrontierStore:
         self.created_at: str | None = None
         self.updated_at: str | None = None
         self._journal_needs_separator = False
+        self._loaded_from_disk = False
         self._load()
 
     def _load(self) -> None:
@@ -100,6 +101,7 @@ class FrontierStore:
             else:
                 self._load_snapshot(data)
         self._replay_journal()
+        self._loaded_from_disk = bool(self.entries)
 
     def _load_snapshot(self, data: object) -> None:
         if not isinstance(data, dict) or data.get("schema_version") != FRONTIER_SCHEMA_VERSION:
@@ -197,7 +199,11 @@ class FrontierStore:
             self.save()
 
     def initialize(self, *, start_url: str, run_fingerprint: dict[str, object]) -> None:
-        if self.start_url != start_url or self.run_fingerprint != run_fingerprint:
+        stale = self.start_url != start_url or self.run_fingerprint != run_fingerprint
+        # A never-initialized store holding entries from this process (streaming workers
+        # finish pages before the first snapshot) keeps them; entries read from disk are
+        # from an unknown crawl and are dropped.
+        if stale and (self.start_url is not None or self._loaded_from_disk):
             self.entries.clear()
             self.created_at = utc_now_iso()
         self.start_url = start_url
@@ -259,3 +265,4 @@ class FrontierStore:
         self.created_at = None
         self.updated_at = None
         self._journal_needs_separator = False
+        self._loaded_from_disk = False

{docpull-6.6.0 → docpull-6.8.0}/src/docpull/cli.py

@@ -1687,6 +1687,22 @@ def main(argv: list[str] | None = None) -> int:
         from .graph import run_graph_cli
 
         return run_graph_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "registry":
+        from .registry import run_registry_cli
+
+        return run_registry_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "research":
+        from .research import run_research_cli
+
+        return run_research_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "index":
+        from .discovery_index import run_index_cli
+
+        return run_index_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "search":
+        from .web_search import run_search_cli
+
+        return run_search_cli(raw_argv[1:])
     if raw_argv and raw_argv[0] == "serve":
         from .server import run_serve_cli