pagedigest 0.1.0__tar.gz

pagedigest-0.1.0/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: pagedigest
+Version: 0.1.0
+Summary: Minimal pagedigest consumer reference implementation
+License-Expression: MIT
+Project-URL: Homepage, https://pagedigest.org
+Project-URL: Repository, https://github.com/maxwellsantoro/pagedigest
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: jsonschema>=4.23.0; extra == "dev"
+Requires-Dist: ruff>=0.8.0; extra == "dev"
+
+# Python Consumer (Minimal Reference)
+
+## Install (from repo)
+
+```bash
+cd implementations/python-consumer
+uv sync
+uv pip install -e .
+```
+
+Requires Python ≥3.9. Runtime dependency: `requests`.
+
+## API
+
+- `fetch` — fetch and validate manifest; graceful fallback on errors
+- `diff` — compare manifest to cached `site_rev` / per-URL `rev`
+- `audit` — identity-encoding digest check (streams the body with a size cap)
+- `check_site` — `fetch` + `diff` + optional sampled audit plan
+- `verify_live` — fetch a manifest and sample live identity-encoded responses for digest verification
+- `identity_digest` — stream-hash a `stream=True` response body, capped at `max_bytes`, for custom audit pipelines
+- `validate_manifest`, `resolve_url_key`, `manifest_url` — validation and URL helpers
+- `format_state_header`, `parse_state_header` — strict optional `PageDigest-State` helpers
+
+## CLI
+
+```bash
+pagedigest verify-live https://example.com --sample-size 25
+```
+
+`verify-live` exits `2` on digest mismatches so it can act as a deployment gate.
+Redirects, network errors, and body-size caps are reported as inconclusive.
+
+## Example
+
+```python
+from pagedigest import check_site
+
+decision = check_site(
+    "https://example.com",
+    cached_site_rev=12,
+    cached_revs={"/": 3, "/about": 1},
+    sample_audit_rate=0.01,
+)
+```
+
+After a successful manifest check, an integration may make its observed state
+visible on subsequent page requests:
+
+```python
+from pagedigest import format_state_header
+
+headers = {
+    "PageDigest-State": format_state_header(
+        decision["manifest"]["site_rev"],
+        "/.well-known/pagedigest.json",
+    )
+}
+```
+
+This is a corroborating observation signal, not authentication. See
+[SPEC.md §5.4](../../SPEC.md#54-optional-cooperation-request-header).
+
+Conformance fixtures: `tests/test_vectors.py` exercises `../../test-vectors/`.
+
+## Persistent cache example
+
+```bash
+uv run python examples/cache_persistence.py https://example.com ./pagedigest-cache.json
+```
+
+The example stores `site_rev`, per-URL `rev`, `ETag`, and `Last-Modified`
+between runs. It prints page fetch decisions so crawler/indexer integrations can
+replace the `print` calls with their own fetch pipeline.

pagedigest-0.1.0/README.md

@@ -0,0 +1,73 @@
+# Python Consumer (Minimal Reference)
+
+## Install (from repo)
+
+```bash
+cd implementations/python-consumer
+uv sync
+uv pip install -e .
+```
+
+Requires Python ≥3.9. Runtime dependency: `requests`.
+
+## API
+
+- `fetch` — fetch and validate manifest; graceful fallback on errors
+- `diff` — compare manifest to cached `site_rev` / per-URL `rev`
+- `audit` — identity-encoding digest check (streams the body with a size cap)
+- `check_site` — `fetch` + `diff` + optional sampled audit plan
+- `verify_live` — fetch a manifest and sample live identity-encoded responses for digest verification
+- `identity_digest` — stream-hash a `stream=True` response body, capped at `max_bytes`, for custom audit pipelines
+- `validate_manifest`, `resolve_url_key`, `manifest_url` — validation and URL helpers
+- `format_state_header`, `parse_state_header` — strict optional `PageDigest-State` helpers
+
+## CLI
+
+```bash
+pagedigest verify-live https://example.com --sample-size 25
+```
+
+`verify-live` exits `2` on digest mismatches so it can act as a deployment gate.
+Redirects, network errors, and body-size caps are reported as inconclusive.
+
+## Example
+
+```python
+from pagedigest import check_site
+
+decision = check_site(
+    "https://example.com",
+    cached_site_rev=12,
+    cached_revs={"/": 3, "/about": 1},
+    sample_audit_rate=0.01,
+)
+```
+
+After a successful manifest check, an integration may make its observed state
+visible on subsequent page requests:
+
+```python
+from pagedigest import format_state_header
+
+headers = {
+    "PageDigest-State": format_state_header(
+        decision["manifest"]["site_rev"],
+        "/.well-known/pagedigest.json",
+    )
+}
+```
+
+This is a corroborating observation signal, not authentication. See
+[SPEC.md §5.4](../../SPEC.md#54-optional-cooperation-request-header).
+
+Conformance fixtures: `tests/test_vectors.py` exercises `../../test-vectors/`.
+
+## Persistent cache example
+
+```bash
+uv run python examples/cache_persistence.py https://example.com ./pagedigest-cache.json
+```
+
+The example stores `site_rev`, per-URL `rev`, `ETag`, and `Last-Modified`
+between runs. It prints page fetch decisions so crawler/indexer integrations can
+replace the `print` calls with their own fetch pipeline.

pagedigest-0.1.0/pagedigest/__init__.py

@@ -0,0 +1,31 @@
+from .core import (
+    audit,
+    check_site,
+    diff,
+    fetch,
+    fetch_manifest_url,
+    format_state_header,
+    identity_digest,
+    live_manifest_url,
+    manifest_url,
+    parse_state_header,
+    resolve_url_key,
+    validate_manifest,
+    verify_live,
+)
+
+__all__ = [
+    "fetch",
+    "fetch_manifest_url",
+    "format_state_header",
+    "diff",
+    "audit",
+    "check_site",
+    "identity_digest",
+    "live_manifest_url",
+    "manifest_url",
+    "parse_state_header",
+    "resolve_url_key",
+    "validate_manifest",
+    "verify_live",
+]

pagedigest-0.1.0/pagedigest/cli.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from typing import TextIO
+
+from .core import DEFAULT_MAX_AUDIT_BYTES, DEFAULT_MAX_MANIFEST_BYTES, verify_live
+
+
+def _add_verify_live_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("base_url", help="Site base URL, e.g. https://example.com")
+    parser.add_argument(
+        "--manifest-url",
+        help="Override manifest URL (defaults to /.well-known/pagedigest.json)",
+    )
+    parser.add_argument(
+        "--sample-size",
+        type=int,
+        default=25,
+        help="Number of digest entries to sample",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for deterministic sampling",
+    )
+    parser.add_argument("--timeout", type=int, default=15, help="HTTP timeout seconds")
+    parser.add_argument(
+        "--max-bytes",
+        type=int,
+        default=DEFAULT_MAX_AUDIT_BYTES,
+        help="Abort identity fetches larger than this many bytes",
+    )
+    parser.add_argument(
+        "--manifest-max-bytes",
+        type=int,
+        default=DEFAULT_MAX_MANIFEST_BYTES,
+        help="Abort manifest fetches larger than this many bytes",
+    )
+
+
+def _run_verify_live(args: argparse.Namespace, stdout: TextIO, stderr: TextIO) -> int:
+    if args.sample_size < 0:
+        print("sample-size must be non-negative", file=stderr)
+        return 1
+    if args.timeout <= 0:
+        print("timeout must be positive", file=stderr)
+        return 1
+    if args.max_bytes <= 0:
+        print("max-bytes must be positive", file=stderr)
+        return 1
+    if args.manifest_max_bytes <= 0:
+        print("manifest-max-bytes must be positive", file=stderr)
+        return 1
+
+    result = verify_live(
+        args.base_url,
+        manifest_url_override=args.manifest_url,
+        sample_size=args.sample_size,
+        seed=args.seed,
+        timeout=args.timeout,
+        max_bytes=args.max_bytes,
+        manifest_max_bytes=args.manifest_max_bytes,
+    )
+
+    print(f"manifest: {result.get('manifest_url')}", file=stdout)
+    if not result.get("ok"):
+        error = result.get("error") or "unknown-error"
+        status_code = result.get("status_code")
+        suffix = f":{status_code}" if status_code is not None else ""
+        print(f"error: {error}{suffix}", file=stderr)
+        return 1
+
+    print(f"sampled: {result['sampled']}", file=stdout)
+    print(f"match: {result['match_count']}", file=stdout)
+    print(f"mismatch: {result['mismatch_count']}", file=stdout)
+    print(f"inconclusive: {result['inconclusive_count']}", file=stdout)
+
+    for item in result["results"]:
+        if item["status"] != "match":
+            print(f"- {item['status']}: {item['url']} ({item['detail']})", file=stdout)
+
+    if result["mismatch_count"] > 0:
+        return 2
+    return 0
+
+
+def verify_live_main(
+    argv: Sequence[str] | None = None, stdout: TextIO | None = None, stderr: TextIO | None = None
+) -> int:
+    parser = argparse.ArgumentParser(description="Verify pagedigest digest values over the wire")
+    _add_verify_live_args(parser)
+    return _run_verify_live(parser.parse_args(argv), stdout or sys.stdout, stderr or sys.stderr)
+
+
+def main(argv: Sequence[str] | None = None, stdout: TextIO | None = None, stderr: TextIO | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="pagedigest")
+    subcommands = parser.add_subparsers(dest="command")
+    verify_parser = subcommands.add_parser("verify-live", help="Verify manifest digests against live responses")
+    _add_verify_live_args(verify_parser)
+
+    args = parser.parse_args(argv)
+    if args.command == "verify-live":
+        return _run_verify_live(args, stdout or sys.stdout, stderr or sys.stderr)
+
+    parser.print_help(stderr or sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())