docpull 6.3.0__tar.gz → 6.5.0__tar.gz

{docpull-6.3.0/src/docpull.egg-info → docpull-6.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.3.0
+Version: 6.5.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -61,7 +61,7 @@ Provides-Extra: tokens
 Requires-Dist: tiktoken>=0.7.0; extra == "tokens"
 Provides-Extra: mcp
 Requires-Dist: cryptography>=48.0.1; extra == "mcp"
-Requires-Dist: mcp>=1.0.0; extra == "mcp"
+Requires-Dist: mcp>=1.28.1; extra == "mcp"
 Requires-Dist: pydantic-settings>=2.14.2; extra == "mcp"
 Requires-Dist: pyjwt>=2.13.0; extra == "mcp"
 Requires-Dist: python-multipart>=0.0.27; extra == "mcp"
@@ -84,6 +84,8 @@ Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
 Provides-Extra: presidio
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
+Provides-Extra: provenance
+Requires-Dist: cryptography>=48.0.1; extra == "provenance"
 Provides-Extra: e2b
 Requires-Dist: e2b>=2.0.0; extra == "e2b"
 Provides-Extra: llm
@@ -94,7 +96,7 @@ Requires-Dist: url-normalize>=1.4.0; extra == "all"
 Requires-Dist: trafilatura>=1.12.0; extra == "all"
 Requires-Dist: tiktoken>=0.7.0; extra == "all"
 Requires-Dist: cryptography>=48.0.1; extra == "all"
-Requires-Dist: mcp>=1.0.0; extra == "all"
+Requires-Dist: mcp>=1.28.1; extra == "all"
 Requires-Dist: pydantic-settings>=2.14.2; extra == "all"
 Requires-Dist: pyjwt>=2.13.0; extra == "all"
 Requires-Dist: python-multipart>=0.0.27; extra == "all"
@@ -180,8 +182,10 @@ through their typed lanes and do not use discovery.
 
 The original `docpull URL ...` workflow still works: fetch public or explicitly
 authorized static/server-rendered web pages and write clean Markdown, NDJSON,
-SQLite, or OKF outputs. Project mode adds the persistent evidence lifecycle on
-top: sources, runs, diffs, exports, evals, accounting, and local auditability.
+SQLite, or Open Knowledge Format (OKF) outputs. OKF is a portable bundle format
+that keeps content, indexes, and manifests together for reuse across tools.
+Project mode adds the persistent evidence lifecycle on top: sources, runs,
+diffs, exports, evals, accounting, and local auditability.
 
 DocPull is local-first: direct fetching, sitemap/link discovery, extraction,
 indexing, pack intelligence, and opt-in `agent-browser` rendering can run with
@@ -307,6 +311,7 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull dataset-pack 'https://data.example.org/rows.json?$limit=100' -o packs/dataset
 docpull relationship-pack example.com --subject "Example Brand" -o packs/relationships
+docpull website-pack example.com -o packs/website
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
 docpull brand-pack example.com -o packs/brand
@@ -320,6 +325,9 @@ docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
 docpull export packs/docs --format cursor-rules -o .cursor/rules --skill-name docs
 ```
 
+The canonical baseline-aware website acquisition contract is documented in
+[Website snapshots](docs/website-snapshot.md).
+
 Use `docpull ci --prepare` to validate a project or standalone pack in CI.
 
 Install optional extras as needed:
@@ -700,6 +708,13 @@ installed.
 | Hosted extraction APIs | Managed extraction at scale | External dependency and cost |
 | docpull | Local public web-source extraction and context packs | No JavaScript rendering by default |
 
+For a tool-by-tool comparison, including self-hosted browser crawlers and
+hosted extraction services, see [Alternatives](docs/alternatives.md). Measured
+extraction-quality, tokens-per-page, and cost comparisons on a fixed corpus —
+plus local extractor baselines — live in the evaluation lab under
+[`bench/`](bench/README.md), which is an internal, reproducible harness rather
+than a public leaderboard.
+
 ## Python SDK
 
 ```python
@@ -772,7 +787,7 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull wiki-pack`, `docpull website-pack`, `docpull brand-pack`, `docpull product-pack`,
   `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
@@ -794,6 +809,9 @@ part of the package release contract.
 When running with `--proxy`, DNS pinning is delegated to the proxy. Pass
 `--require-pinned-dns` to refuse that configuration.
 
+See [Compliance Posture](docs/compliance.md) for the full identity, robots.txt,
+rate-limit, and AI/TDM opt-out behavior.
+
 ## Troubleshooting
 
 ```bash

{docpull-6.3.0 → docpull-6.5.0}/README.md

@@ -56,8 +56,10 @@ through their typed lanes and do not use discovery.
 
 The original `docpull URL ...` workflow still works: fetch public or explicitly
 authorized static/server-rendered web pages and write clean Markdown, NDJSON,
-SQLite, or OKF outputs. Project mode adds the persistent evidence lifecycle on
-top: sources, runs, diffs, exports, evals, accounting, and local auditability.
+SQLite, or Open Knowledge Format (OKF) outputs. OKF is a portable bundle format
+that keeps content, indexes, and manifests together for reuse across tools.
+Project mode adds the persistent evidence lifecycle on top: sources, runs,
+diffs, exports, evals, accounting, and local auditability.
 
 DocPull is local-first: direct fetching, sitemap/link discovery, extraction,
 indexing, pack intelligence, and opt-in `agent-browser` rendering can run with
@@ -183,6 +185,7 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull dataset-pack 'https://data.example.org/rows.json?$limit=100' -o packs/dataset
 docpull relationship-pack example.com --subject "Example Brand" -o packs/relationships
+docpull website-pack example.com -o packs/website
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
 docpull brand-pack example.com -o packs/brand
@@ -196,6 +199,9 @@ docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
 docpull export packs/docs --format cursor-rules -o .cursor/rules --skill-name docs
 ```
 
+The canonical baseline-aware website acquisition contract is documented in
+[Website snapshots](docs/website-snapshot.md).
+
 Use `docpull ci --prepare` to validate a project or standalone pack in CI.
 
 Install optional extras as needed:
@@ -576,6 +582,13 @@ installed.
 | Hosted extraction APIs | Managed extraction at scale | External dependency and cost |
 | docpull | Local public web-source extraction and context packs | No JavaScript rendering by default |
 
+For a tool-by-tool comparison, including self-hosted browser crawlers and
+hosted extraction services, see [Alternatives](docs/alternatives.md). Measured
+extraction-quality, tokens-per-page, and cost comparisons on a fixed corpus —
+plus local extractor baselines — live in the evaluation lab under
+[`bench/`](bench/README.md), which is an internal, reproducible harness rather
+than a public leaderboard.
+
 ## Python SDK
 
 ```python
@@ -648,7 +661,7 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull wiki-pack`, `docpull website-pack`, `docpull brand-pack`, `docpull product-pack`,
   `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
@@ -670,6 +683,9 @@ part of the package release contract.
 When running with `--proxy`, DNS pinning is delegated to the proxy. Pass
 `--require-pinned-dns` to refuse that configuration.
 
+See [Compliance Posture](docs/compliance.md) for the full identity, robots.txt,
+rate-limit, and AI/TDM opt-out behavior.
+
 ## Troubleshooting
 
 ```bash

{docpull-6.3.0 → docpull-6.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.3.0"
+version = "6.5.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -110,7 +110,7 @@ tokens = [
 ]
 mcp = [
     "cryptography>=48.0.1",
-    "mcp>=1.0.0",
+    "mcp>=1.28.1",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.13.0",
     "python-multipart>=0.0.27",
@@ -141,6 +141,9 @@ parse = [
 presidio = [
     "presidio-analyzer>=2.2.0",
 ]
+provenance = [
+    "cryptography>=48.0.1",
+]
 e2b = [
     "e2b>=2.0.0",
 ]
@@ -153,7 +156,7 @@ all = [
     "trafilatura>=1.12.0",
     "tiktoken>=0.7.0",
     "cryptography>=48.0.1",
-    "mcp>=1.0.0",
+    "mcp>=1.28.1",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.13.0",
     "python-multipart>=0.0.27",

{docpull-6.3.0 → docpull-6.5.0}/src/docpull/__init__.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any
 
 from .surface import PUBLIC_SDK_EXPORTS
 
-__version__ = "6.3.0"
+__version__ = "6.5.0"
 
 _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
     **{
@@ -41,9 +41,11 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
             "build_repo_pack",
             "build_standards_pack",
             "build_styleguide_pack",
+            "build_website_pack",
             "build_transcript_pack",
             "build_wiki_pack",
             "capture_screenshot_pack",
+            "validate_website_snapshot_pack",
         )
     },
     **{
@@ -59,6 +61,8 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
             "SourceAuthority",
             "WorkflowRequest",
             "WorkflowResult",
+            "WebsiteSnapshot",
+            "WebsiteSnapshotDocument",
             "bundled_schema_path",
             "write_contract_schemas",
         )
@@ -198,8 +202,10 @@ if TYPE_CHECKING:
         build_standards_pack,
         build_styleguide_pack,
         build_transcript_pack,
+        build_website_pack,
         build_wiki_pack,
         capture_screenshot_pack,
+        validate_website_snapshot_pack,
     )
     from .contracts import (
         ArtifactManifest,
@@ -210,6 +216,8 @@ if TYPE_CHECKING:
         RelationshipCandidate,
         RelationshipPack,
         SourceAuthority,
+        WebsiteSnapshot,
+        WebsiteSnapshotDocument,
         WorkflowRequest,
         WorkflowResult,
         bundled_schema_path,

{docpull-6.3.0 → docpull-6.5.0}/src/docpull/accounting.py

@@ -232,6 +232,11 @@ def default_route_steps(
         RouteStep("direct_http", "available", detail="fetch public HTTPS source directly"),
         RouteStep("sitemap_link_discovery", "available", detail="use sitemap and static links"),
         RouteStep("embedded_data_extraction", "available", detail="use framework/static page data"),
+        RouteStep(
+            "archive_fallback",
+            "available",
+            detail="replay Wayback CDX and Common Crawl index snapshots at zero cost",
+        ),
     ]
     steps.append(
         RouteStep(