docpull 6.0.1__tar.gz → 6.2.0__tar.gz

{docpull-6.0.1/src/docpull.egg-info → docpull-6.2.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.0.1
+Version: 6.2.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
 License-Expression: MIT
-Project-URL: Homepage, https://docpull.raintree.technology
-Project-URL: Documentation, https://docpull.raintree.technology
+Project-URL: Homepage, https://github.com/raintree-technology/docpull
+Project-URL: Documentation, https://github.com/raintree-technology/docpull#readme
 Project-URL: Repository, https://github.com/raintree-technology/docpull
 Project-URL: Source Code, https://github.com/raintree-technology/docpull
 Project-URL: Bug Tracker, https://github.com/raintree-technology/docpull/issues
@@ -71,11 +71,16 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
 Provides-Extra: markitdown
-Requires-Dist: markitdown>=0.1.0; extra == "markitdown"
+Requires-Dist: pypdf>=6.14.2; extra == "markitdown"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
 Provides-Extra: unstructured
+Requires-Dist: pypdf>=6.14.2; extra == "unstructured"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "unstructured"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=6.14.2; extra == "pdf"
 Provides-Extra: parse
-Requires-Dist: markitdown>=0.1.0; extra == "parse"
+Requires-Dist: pypdf>=6.14.2; extra == "parse"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
 Provides-Extra: presidio
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
@@ -96,7 +101,8 @@ Requires-Dist: python-multipart>=0.0.27; extra == "all"
 Requires-Dist: starlette>=1.3.1; extra == "all"
 Requires-Dist: uvicorn>=0.30.0; extra == "all"
 Requires-Dist: pyarrow>=16.0.0; extra == "all"
-Requires-Dist: markitdown>=0.1.0; extra == "all"
+Requires-Dist: pypdf>=6.14.2; extra == "all"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "all"
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
 Requires-Dist: e2b>=2.0.0; extra == "all"
@@ -133,6 +139,11 @@ web sources an agent depends on, sync them into cited context packs, diff what
 changed, and export reproducible context for Cursor, Claude, OpenAI,
 LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
+Architecturally, DocPull is the evidence and acquisition engine: it acquires,
+versions, cites, hashes, and replays evidence. Downstream products own
+scheduling, review, approved claims, legal conclusions, and notifications. See
+the [architecture decision](docs/adr/0001-evidence-acquisition-engine.md).
+
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
 
@@ -181,6 +192,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -282,6 +299,11 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
+docpull brand-pack example.com -o packs/brand
+docpull product-pack https://example.com/pricing -o packs/product
+docpull styleguide-pack example.com -o packs/styleguide
+docpull image-pack example.com -o packs/visuals
+docpull policy-pack example.com -o packs/policies
 docpull pack prepare packs/docs --eval-grade
 docpull pack validate packs/docs --level eval
 docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
@@ -556,6 +578,11 @@ special handling for common web, documentation, and API surfaces.
 | Local datasets | `docpull dataset-pack` emits bounded schema, exact row counts where streamable, column, null-count, and sample summaries |
 | Transcripts | `docpull transcript-pack` emits timestamped segment records from VTT, SRT, text, JSON, or direct transcript URLs |
 | Wikimedia / Wikipedia | `docpull wiki-pack` emits MediaWiki REST page metadata, license/revision metadata, and section-level records |
+| Brand evidence | `docpull brand-pack` emits cited identity, firmographic, social, logo, and color observations |
+| Product and pricing | `docpull product-pack` emits plans, currencies, intervals, trials, feature gates, and page-text provenance |
+| Styleguide | `docpull styleguide-pack` emits cited CSS variables, colors, fonts, spacing, and component samples |
+| Visual assets | `docpull image-pack` emits bounded image candidates and optional validated local assets |
+| Policy documents | `docpull policy-pack` discovers policy/security pages and emits effective dates, stable clauses, and textual change candidates without legal conclusions |
 | Docusaurus / Sphinx / MkDocs | Extracts static article or document regions |
 | VitePress / VuePress / Astro Starlight | Extracts static content regions |
 | GitBook / ReadMe.io | Extracts available article or content regions |
@@ -592,10 +619,32 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
 
+Evidence-pack workflows also write `workflow.request.json`,
+`workflow.result.json`, and `artifact.manifest.json`. Export their Draft 2020-12
+schemas with `docpull contracts export -o schemas/`. The complete inventory and
+compatibility rules are in [Public Contracts](docs/contracts.md).
+
+For tracker imports, run `docpull pack intelligence-bundle PACK`. The canonical
+output is `intelligence.bundle.v1.json`; `company_brain.bundle.json` remains a
+compatibility alias. See the
+[competitor-tracker integration contract](docs/competitor-tracker-integration.md).
+
 ## Profiles
 
 ```bash
@@ -712,7 +761,8 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`,
+  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
   `docpull serve`, `docpull share`, `docpull render`, `docpull auth check`,
@@ -754,7 +804,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.1 → docpull-6.2.0}/README.md

@@ -15,6 +15,11 @@ web sources an agent depends on, sync them into cited context packs, diff what
 changed, and export reproducible context for Cursor, Claude, OpenAI,
 LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
+Architecturally, DocPull is the evidence and acquisition engine: it acquires,
+versions, cites, hashes, and replays evidence. Downstream products own
+scheduling, review, approved claims, legal conclusions, and notifications. See
+the [architecture decision](docs/adr/0001-evidence-acquisition-engine.md).
+
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
 
@@ -63,6 +68,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -164,6 +175,11 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
+docpull brand-pack example.com -o packs/brand
+docpull product-pack https://example.com/pricing -o packs/product
+docpull styleguide-pack example.com -o packs/styleguide
+docpull image-pack example.com -o packs/visuals
+docpull policy-pack example.com -o packs/policies
 docpull pack prepare packs/docs --eval-grade
 docpull pack validate packs/docs --level eval
 docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
@@ -438,6 +454,11 @@ special handling for common web, documentation, and API surfaces.
 | Local datasets | `docpull dataset-pack` emits bounded schema, exact row counts where streamable, column, null-count, and sample summaries |
 | Transcripts | `docpull transcript-pack` emits timestamped segment records from VTT, SRT, text, JSON, or direct transcript URLs |
 | Wikimedia / Wikipedia | `docpull wiki-pack` emits MediaWiki REST page metadata, license/revision metadata, and section-level records |
+| Brand evidence | `docpull brand-pack` emits cited identity, firmographic, social, logo, and color observations |
+| Product and pricing | `docpull product-pack` emits plans, currencies, intervals, trials, feature gates, and page-text provenance |
+| Styleguide | `docpull styleguide-pack` emits cited CSS variables, colors, fonts, spacing, and component samples |
+| Visual assets | `docpull image-pack` emits bounded image candidates and optional validated local assets |
+| Policy documents | `docpull policy-pack` discovers policy/security pages and emits effective dates, stable clauses, and textual change candidates without legal conclusions |
 | Docusaurus / Sphinx / MkDocs | Extracts static article or document regions |
 | VitePress / VuePress / Astro Starlight | Extracts static content regions |
 | GitBook / ReadMe.io | Extracts available article or content regions |
@@ -474,10 +495,32 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
 
+Evidence-pack workflows also write `workflow.request.json`,
+`workflow.result.json`, and `artifact.manifest.json`. Export their Draft 2020-12
+schemas with `docpull contracts export -o schemas/`. The complete inventory and
+compatibility rules are in [Public Contracts](docs/contracts.md).
+
+For tracker imports, run `docpull pack intelligence-bundle PACK`. The canonical
+output is `intelligence.bundle.v1.json`; `company_brain.bundle.json` remains a
+compatibility alias. See the
+[competitor-tracker integration contract](docs/competitor-tracker-integration.md).
+
 ## Profiles
 
 ```bash
@@ -594,7 +637,8 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`,
+  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
   `docpull serve`, `docpull share`, `docpull render`, `docpull auth check`,
@@ -636,7 +680,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.1 → docpull-6.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.0.1"
+version = "6.2.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -123,13 +123,19 @@ parquet = [
     "pyarrow>=16.0.0",
 ]
 markitdown = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
 ]
 unstructured = [
+    "pypdf>=6.14.2",
     "unstructured[all-docs]>=0.16.0",
 ]
+pdf = [
+    "pypdf>=6.14.2",
+]
 parse = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
 ]
 presidio = [
@@ -154,7 +160,8 @@ all = [
     "starlette>=1.3.1",
     "uvicorn>=0.30.0",
     "pyarrow>=16.0.0",
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
     "presidio-analyzer>=2.2.0",
     "e2b>=2.0.0",
@@ -179,8 +186,8 @@ dev = [
 docpull = "docpull.cli:main"
 
 [project.urls]
-Homepage = "https://docpull.raintree.technology"
-Documentation = "https://docpull.raintree.technology"
+Homepage = "https://github.com/raintree-technology/docpull"
+Documentation = "https://github.com/raintree-technology/docpull#readme"
 Repository = "https://github.com/raintree-technology/docpull"
 "Source Code" = "https://github.com/raintree-technology/docpull"
 "Bug Tracker" = "https://github.com/raintree-technology/docpull/issues"
@@ -195,7 +202,7 @@ where = ["src"]
 include = ["docpull*"]
 
 [tool.setuptools.package-data]
-docpull = ["py.typed", "fixtures/*.json"]
+docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"]
 
 [tool.ruff]
 line-length = 110

{docpull-6.0.1 → docpull-6.2.0}/src/docpull/__init__.py

@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "6.0.1"
+__version__ = "6.2.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .context_ci import CIThresholds, ContextCIError, run_context_ci
@@ -26,15 +26,32 @@ from .context_packs import (
     async_build_standards_pack,
     async_build_transcript_pack,
     async_build_wiki_pack,
+    build_brand_pack,
     build_dataset_pack,
     build_feed_pack,
+    build_image_pack,
     build_openapi_pack,
     build_package_pack,
     build_paper_pack,
+    build_policy_pack,
+    build_product_pack,
     build_repo_pack,
     build_standards_pack,
+    build_styleguide_pack,
     build_transcript_pack,
     build_wiki_pack,
+    capture_screenshot_pack,
+)
+from .contracts import (
+    ArtifactManifest,
+    ChangeEvent,
+    EvidenceSpan,
+    IntelligenceBundle,
+    SourceAuthority,
+    WorkflowRequest,
+    WorkflowResult,
+    bundled_schema_path,
+    write_contract_schemas,
 )
 from .conversion.chunking import Chunk, TokenCounter, chunk_markdown
 from .core.fetcher import Fetcher, fetch_blocking, fetch_one
@@ -69,6 +86,8 @@ from .output_contract import validate_pack_contract
 from .pack_reader import LocalPack, PackReadError, PackSource, load_pack
 from .pack_tools import (
     build_citation_map,
+    build_company_brain_bundle,
+    build_intelligence_bundle,
     build_research_brief,
     diff_packs,
     extract_pack_entities,
@@ -100,6 +119,7 @@ from .rendering import (
 from .server import PackASGIApp, PackServerError, create_pack_app
 from .share import ReportHTTPServer, ShareError, create_report_server, render_report_document
 from .surface import PUBLIC_SDK_EXPORTS
+from .workflows import async_run_workflow, create_workflow_request, run_workflow
 
 __all__ = [
     "__version__",
@@ -110,20 +130,38 @@ __all__ = [
     "async_build_standards_pack",
     "async_build_transcript_pack",
     "async_build_wiki_pack",
+    "async_run_workflow",
+    "ArtifactManifest",
+    "ChangeEvent",
+    "EvidenceSpan",
+    "IntelligenceBundle",
+    "SourceAuthority",
+    "WorkflowRequest",
+    "WorkflowResult",
+    "bundled_schema_path",
+    "write_contract_schemas",
+    "create_workflow_request",
+    "run_workflow",
     "Fetcher",
     "fetch_blocking",
     "fetch_one",
     "refresh_pack",
     "audit_pack",
     "build_dataset_pack",
+    "build_brand_pack",
     "build_feed_pack",
     "build_openapi_pack",
     "build_package_pack",
     "build_paper_pack",
+    "build_policy_pack",
+    "build_product_pack",
     "build_repo_pack",
     "build_standards_pack",
+    "build_styleguide_pack",
     "build_transcript_pack",
     "build_wiki_pack",
+    "build_image_pack",
+    "capture_screenshot_pack",
     "parse_documents",
     "parse_one_document",
     "DocumentParseError",
@@ -132,6 +170,8 @@ __all__ = [
     "score_pack_sources",
     "diff_packs",
     "build_citation_map",
+    "build_intelligence_bundle",
+    "build_company_brain_bundle",
     "extract_pack_entities",
     "search_pack",
     "build_research_brief",

{docpull-6.0.1 → docpull-6.2.0}/src/docpull/benchmark.py

@@ -200,8 +200,8 @@ TOOL_DOC_TARGETS: tuple[_BenchmarkTarget, ...] = (
     _BenchmarkTarget(
         id="docpull_docs",
         label="DocPull docs",
-        url="https://docpull.raintree.technology",
-        include_domains=("docpull.raintree.technology",),
+        url="https://github.com/raintree-technology/docpull",
+        include_domains=("github.com",),
         objective="Build an agent context pack for DocPull documentation",
         queries=("DocPull documentation CLI provider benchmark docs",),
         freshness_terms=("changelog", "release", "benchmark"),

docpull-6.2.0/src/docpull/change_events.py

@@ -0,0 +1,209 @@
+"""Versioned, scheduler-neutral change-event generation."""
+
+from __future__ import annotations
+
+import difflib
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from .contracts import ChangeEvent, ReplayConfiguration, canonical_sha256, stable_id
+from .evidence import evidence_span
+
+ChangeClassification = Literal["pricing", "positioning", "product", "security", "policy", "other"]
+
+_CLASSIFICATION_TERMS: dict[ChangeClassification, tuple[str, ...]] = {
+    "pricing": ("price", "pricing", "plan", "$", "usd", "billing", "trial"),
+    "positioning": ("positioning", "mission", "leader", "best", "platform for", "built for"),
+    "product": ("feature", "product", "launch", "available", "integration", "api"),
+    "security": ("security", "encryption", "soc 2", "iso 27001", "vulnerability", "breach"),
+    "policy": ("terms", "privacy", "cookie", "dpa", "subprocessor", "refund", "effective date"),
+}
+
+
+def build_change_events(
+    old_records: dict[str, list[dict[str, Any]]],
+    new_records: dict[str, list[dict[str, Any]]],
+    *,
+    workflow: str = "pack-diff",
+) -> list[dict[str, Any]]:
+    """Build idempotent events from already acquired document versions."""
+
+    events: list[dict[str, Any]] = []
+    for url in sorted(set(old_records) | set(new_records)):
+        old_items = old_records.get(url, [])
+        new_items = new_records.get(url, [])
+        old_hashes = sorted(_hashes(old_items))
+        new_hashes = sorted(_hashes(new_items))
+        if old_hashes == new_hashes and _titles(old_items) == _titles(new_items):
+            continue
+        old_primary = old_items[0] if old_items else None
+        new_primary = new_items[0] if new_items else None
+        old_text = _combined_content(old_items)
+        new_text = _combined_content(new_items)
+        classifications = _classifications(old_text, new_text, url)
+        structural = _structural_changes(old_items, new_items)
+        textual = _textual_changes(old_text, new_text)
+        old_evidence = _record_evidence(old_primary, old_text) if old_primary else []
+        new_evidence = _record_evidence(new_primary, new_text) if new_primary else []
+        identity = {
+            "workflow": workflow,
+            "url": url,
+            "old_hashes": old_hashes,
+            "new_hashes": new_hashes,
+        }
+        idempotency_key = canonical_sha256(identity)
+        semantic_candidates = [
+            {
+                "classification": item,
+                "status": "candidate",
+                "confidence": _classification_confidence(item, old_text, new_text, url),
+                "requires_review": True,
+            }
+            for item in classifications
+        ]
+        event = ChangeEvent(
+            event_id=stable_id("change_event", identity),
+            idempotency_key=idempotency_key,
+            workflow=workflow,
+            url=url,
+            old_document_id=_string(old_primary, "document_id"),
+            new_document_id=_string(new_primary, "document_id"),
+            old_hash=canonical_sha256(old_hashes) if old_hashes else None,
+            new_hash=canonical_sha256(new_hashes) if new_hashes else None,
+            old_evidence=old_evidence,
+            new_evidence=new_evidence,
+            structural_changes=structural,
+            textual_changes=textual,
+            semantic_candidates=semantic_candidates,
+            classifications=classifications,
+            replay_configuration=ReplayConfiguration(
+                configuration={
+                    "workflow": workflow,
+                    "old_hashes": old_hashes,
+                    "new_hashes": new_hashes,
+                }
+            ),
+        )
+        events.append(event.model_dump(mode="json", exclude_none=True))
+    return events
+
+
+def write_change_events(path: Path, events: list[dict[str, Any]]) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        "".join(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n" for event in events),
+        encoding="utf-8",
+    )
+    return path
+
+
+def _record_evidence(record: dict[str, Any], content: str) -> list[Any]:
+    if not content:
+        return []
+    excerpt = content[: min(600, len(content))]
+    citation_id = str(record.get("source_citation_id") or "S0")
+    record_citation_id = record.get("record_citation_id")
+    return [
+        evidence_span(
+            url=str(record.get("url") or ""),
+            content=content,
+            exact_text=excerpt,
+            citation_id=citation_id,
+            record_citation_id=str(record_citation_id) if record_citation_id else None,
+        )
+    ]
+
+
+def _structural_changes(
+    old_items: list[dict[str, Any]],
+    new_items: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    changes: list[dict[str, Any]] = []
+    if not old_items:
+        changes.append({"type": "document_added", "new_record_count": len(new_items)})
+    elif not new_items:
+        changes.append({"type": "document_removed", "old_record_count": len(old_items)})
+    if len(old_items) != len(new_items):
+        changes.append(
+            {
+                "type": "record_count_changed",
+                "before": len(old_items),
+                "after": len(new_items),
+            }
+        )
+    if _titles(old_items) != _titles(new_items):
+        changes.append({"type": "title_changed", "before": _titles(old_items), "after": _titles(new_items)})
+    return changes
+
+
+def _textual_changes(old_text: str, new_text: str) -> list[dict[str, Any]]:
+    old_lines = old_text.splitlines()
+    new_lines = new_text.splitlines()
+    matcher = difflib.SequenceMatcher(a=old_lines, b=new_lines, autojunk=False)
+    changes: list[dict[str, Any]] = []
+    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+        if tag == "equal":
+            continue
+        before = "\n".join(old_lines[i1:i2])[:1000]
+        after = "\n".join(new_lines[j1:j2])[:1000]
+        changes.append(
+            {
+                "type": tag,
+                "old_line_start": i1,
+                "old_line_end": i2,
+                "new_line_start": j1,
+                "new_line_end": j2,
+                "before": before,
+                "after": after,
+                "before_sha256": canonical_sha256(before),
+                "after_sha256": canonical_sha256(after),
+            }
+        )
+        if len(changes) >= 100:
+            break
+    return changes
+
+
+def _classifications(old_text: str, new_text: str, url: str) -> list[ChangeClassification]:
+    haystack = f"{url}\n{old_text}\n{new_text}".lower()
+    output = [
+        classification
+        for classification, terms in _CLASSIFICATION_TERMS.items()
+        if any(term in haystack for term in terms)
+    ]
+    return output or ["other"]
+
+
+def _classification_confidence(
+    classification: ChangeClassification,
+    old_text: str,
+    new_text: str,
+    url: str,
+) -> float:
+    terms = _CLASSIFICATION_TERMS.get(classification, ())
+    haystack = f"{url}\n{old_text}\n{new_text}".lower()
+    hits = sum(1 for term in terms if term in haystack)
+    return min(0.95, 0.5 + hits * 0.1) if terms else 0.4
+
+
+def _combined_content(items: list[dict[str, Any]]) -> str:
+    return "\n\n".join(str(item.get("content") or "") for item in items)
+
+
+def _hashes(items: list[dict[str, Any]]) -> set[str]:
+    return {str(item.get("content_hash")) for item in items if item.get("content_hash")}
+
+
+def _titles(items: list[dict[str, Any]]) -> list[str]:
+    return sorted({str(item.get("title") or "") for item in items})
+
+
+def _string(item: dict[str, Any] | None, key: str) -> str | None:
+    if not item or item.get(key) is None:
+        return None
+    value = str(item[key]).strip()
+    return value or None
+
+
+__all__ = ["build_change_events", "write_change_events"]