docpull 6.1.0__tar.gz → 6.2.0__tar.gz

{docpull-6.1.0/src/docpull.egg-info → docpull-6.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.1.0
+Version: 6.2.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -139,6 +139,11 @@ web sources an agent depends on, sync them into cited context packs, diff what
 changed, and export reproducible context for Cursor, Claude, OpenAI,
 LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
+Architecturally, DocPull is the evidence and acquisition engine: it acquires,
+versions, cites, hashes, and replays evidence. Downstream products own
+scheduling, review, approved claims, legal conclusions, and notifications. See
+the [architecture decision](docs/adr/0001-evidence-acquisition-engine.md).
+
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
 
@@ -294,6 +299,11 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
+docpull brand-pack example.com -o packs/brand
+docpull product-pack https://example.com/pricing -o packs/product
+docpull styleguide-pack example.com -o packs/styleguide
+docpull image-pack example.com -o packs/visuals
+docpull policy-pack example.com -o packs/policies
 docpull pack prepare packs/docs --eval-grade
 docpull pack validate packs/docs --level eval
 docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
@@ -568,6 +578,11 @@ special handling for common web, documentation, and API surfaces.
 | Local datasets | `docpull dataset-pack` emits bounded schema, exact row counts where streamable, column, null-count, and sample summaries |
 | Transcripts | `docpull transcript-pack` emits timestamped segment records from VTT, SRT, text, JSON, or direct transcript URLs |
 | Wikimedia / Wikipedia | `docpull wiki-pack` emits MediaWiki REST page metadata, license/revision metadata, and section-level records |
+| Brand evidence | `docpull brand-pack` emits cited identity, firmographic, social, logo, and color observations |
+| Product and pricing | `docpull product-pack` emits plans, currencies, intervals, trials, feature gates, and page-text provenance |
+| Styleguide | `docpull styleguide-pack` emits cited CSS variables, colors, fonts, spacing, and component samples |
+| Visual assets | `docpull image-pack` emits bounded image candidates and optional validated local assets |
+| Policy documents | `docpull policy-pack` discovers policy/security pages and emits effective dates, stable clauses, and textual change candidates without legal conclusions |
 | Docusaurus / Sphinx / MkDocs | Extracts static article or document regions |
 | VitePress / VuePress / Astro Starlight | Extracts static content regions |
 | GitBook / ReadMe.io | Extracts available article or content regions |
@@ -620,6 +635,16 @@ Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
 
+Evidence-pack workflows also write `workflow.request.json`,
+`workflow.result.json`, and `artifact.manifest.json`. Export their Draft 2020-12
+schemas with `docpull contracts export -o schemas/`. The complete inventory and
+compatibility rules are in [Public Contracts](docs/contracts.md).
+
+For tracker imports, run `docpull pack intelligence-bundle PACK`. The canonical
+output is `intelligence.bundle.v1.json`; `company_brain.bundle.json` remains a
+compatibility alias. See the
+[competitor-tracker integration contract](docs/competitor-tracker-integration.md).
+
 ## Profiles
 
 ```bash
@@ -736,7 +761,8 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`,
+  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
   `docpull serve`, `docpull share`, `docpull render`, `docpull auth check`,

{docpull-6.1.0 → docpull-6.2.0}/README.md

@@ -15,6 +15,11 @@ web sources an agent depends on, sync them into cited context packs, diff what
 changed, and export reproducible context for Cursor, Claude, OpenAI,
 LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
+Architecturally, DocPull is the evidence and acquisition engine: it acquires,
+versions, cites, hashes, and replays evidence. Downstream products own
+scheduling, review, approved claims, legal conclusions, and notifications. See
+the [architecture decision](docs/adr/0001-evidence-acquisition-engine.md).
+
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
 
@@ -170,6 +175,11 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
+docpull brand-pack example.com -o packs/brand
+docpull product-pack https://example.com/pricing -o packs/product
+docpull styleguide-pack example.com -o packs/styleguide
+docpull image-pack example.com -o packs/visuals
+docpull policy-pack example.com -o packs/policies
 docpull pack prepare packs/docs --eval-grade
 docpull pack validate packs/docs --level eval
 docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
@@ -444,6 +454,11 @@ special handling for common web, documentation, and API surfaces.
 | Local datasets | `docpull dataset-pack` emits bounded schema, exact row counts where streamable, column, null-count, and sample summaries |
 | Transcripts | `docpull transcript-pack` emits timestamped segment records from VTT, SRT, text, JSON, or direct transcript URLs |
 | Wikimedia / Wikipedia | `docpull wiki-pack` emits MediaWiki REST page metadata, license/revision metadata, and section-level records |
+| Brand evidence | `docpull brand-pack` emits cited identity, firmographic, social, logo, and color observations |
+| Product and pricing | `docpull product-pack` emits plans, currencies, intervals, trials, feature gates, and page-text provenance |
+| Styleguide | `docpull styleguide-pack` emits cited CSS variables, colors, fonts, spacing, and component samples |
+| Visual assets | `docpull image-pack` emits bounded image candidates and optional validated local assets |
+| Policy documents | `docpull policy-pack` discovers policy/security pages and emits effective dates, stable clauses, and textual change candidates without legal conclusions |
 | Docusaurus / Sphinx / MkDocs | Extracts static article or document regions |
 | VitePress / VuePress / Astro Starlight | Extracts static content regions |
 | GitBook / ReadMe.io | Extracts available article or content regions |
@@ -496,6 +511,16 @@ Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
 
+Evidence-pack workflows also write `workflow.request.json`,
+`workflow.result.json`, and `artifact.manifest.json`. Export their Draft 2020-12
+schemas with `docpull contracts export -o schemas/`. The complete inventory and
+compatibility rules are in [Public Contracts](docs/contracts.md).
+
+For tracker imports, run `docpull pack intelligence-bundle PACK`. The canonical
+output is `intelligence.bundle.v1.json`; `company_brain.bundle.json` remains a
+compatibility alias. See the
+[competitor-tracker integration contract](docs/competitor-tracker-integration.md).
+
 ## Profiles
 
 ```bash
@@ -612,7 +637,8 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`,
+  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,
   `docpull serve`, `docpull share`, `docpull render`, `docpull auth check`,

{docpull-6.1.0 → docpull-6.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.1.0"
+version = "6.2.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -202,7 +202,7 @@ where = ["src"]
 include = ["docpull*"]
 
 [tool.setuptools.package-data]
-docpull = ["py.typed", "fixtures/*.json"]
+docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"]
 
 [tool.ruff]
 line-length = 110

{docpull-6.1.0 → docpull-6.2.0}/src/docpull/__init__.py

@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "6.1.0"
+__version__ = "6.2.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .context_ci import CIThresholds, ContextCIError, run_context_ci
@@ -26,15 +26,32 @@ from .context_packs import (
     async_build_standards_pack,
     async_build_transcript_pack,
     async_build_wiki_pack,
+    build_brand_pack,
     build_dataset_pack,
     build_feed_pack,
+    build_image_pack,
     build_openapi_pack,
     build_package_pack,
     build_paper_pack,
+    build_policy_pack,
+    build_product_pack,
     build_repo_pack,
     build_standards_pack,
+    build_styleguide_pack,
     build_transcript_pack,
     build_wiki_pack,
+    capture_screenshot_pack,
+)
+from .contracts import (
+    ArtifactManifest,
+    ChangeEvent,
+    EvidenceSpan,
+    IntelligenceBundle,
+    SourceAuthority,
+    WorkflowRequest,
+    WorkflowResult,
+    bundled_schema_path,
+    write_contract_schemas,
 )
 from .conversion.chunking import Chunk, TokenCounter, chunk_markdown
 from .core.fetcher import Fetcher, fetch_blocking, fetch_one
@@ -69,6 +86,8 @@ from .output_contract import validate_pack_contract
 from .pack_reader import LocalPack, PackReadError, PackSource, load_pack
 from .pack_tools import (
     build_citation_map,
+    build_company_brain_bundle,
+    build_intelligence_bundle,
     build_research_brief,
     diff_packs,
     extract_pack_entities,
@@ -100,6 +119,7 @@ from .rendering import (
 from .server import PackASGIApp, PackServerError, create_pack_app
 from .share import ReportHTTPServer, ShareError, create_report_server, render_report_document
 from .surface import PUBLIC_SDK_EXPORTS
+from .workflows import async_run_workflow, create_workflow_request, run_workflow
 
 __all__ = [
     "__version__",
@@ -110,20 +130,38 @@ __all__ = [
     "async_build_standards_pack",
     "async_build_transcript_pack",
     "async_build_wiki_pack",
+    "async_run_workflow",
+    "ArtifactManifest",
+    "ChangeEvent",
+    "EvidenceSpan",
+    "IntelligenceBundle",
+    "SourceAuthority",
+    "WorkflowRequest",
+    "WorkflowResult",
+    "bundled_schema_path",
+    "write_contract_schemas",
+    "create_workflow_request",
+    "run_workflow",
     "Fetcher",
     "fetch_blocking",
     "fetch_one",
     "refresh_pack",
     "audit_pack",
     "build_dataset_pack",
+    "build_brand_pack",
     "build_feed_pack",
     "build_openapi_pack",
     "build_package_pack",
     "build_paper_pack",
+    "build_policy_pack",
+    "build_product_pack",
     "build_repo_pack",
     "build_standards_pack",
+    "build_styleguide_pack",
     "build_transcript_pack",
     "build_wiki_pack",
+    "build_image_pack",
+    "capture_screenshot_pack",
     "parse_documents",
     "parse_one_document",
     "DocumentParseError",
@@ -132,6 +170,8 @@ __all__ = [
     "score_pack_sources",
     "diff_packs",
     "build_citation_map",
+    "build_intelligence_bundle",
+    "build_company_brain_bundle",
     "extract_pack_entities",
     "search_pack",
     "build_research_brief",

docpull-6.2.0/src/docpull/change_events.py

@@ -0,0 +1,209 @@
+"""Versioned, scheduler-neutral change-event generation."""
+
+from __future__ import annotations
+
+import difflib
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from .contracts import ChangeEvent, ReplayConfiguration, canonical_sha256, stable_id
+from .evidence import evidence_span
+
+ChangeClassification = Literal["pricing", "positioning", "product", "security", "policy", "other"]
+
+_CLASSIFICATION_TERMS: dict[ChangeClassification, tuple[str, ...]] = {
+    "pricing": ("price", "pricing", "plan", "$", "usd", "billing", "trial"),
+    "positioning": ("positioning", "mission", "leader", "best", "platform for", "built for"),
+    "product": ("feature", "product", "launch", "available", "integration", "api"),
+    "security": ("security", "encryption", "soc 2", "iso 27001", "vulnerability", "breach"),
+    "policy": ("terms", "privacy", "cookie", "dpa", "subprocessor", "refund", "effective date"),
+}
+
+
+def build_change_events(
+    old_records: dict[str, list[dict[str, Any]]],
+    new_records: dict[str, list[dict[str, Any]]],
+    *,
+    workflow: str = "pack-diff",
+) -> list[dict[str, Any]]:
+    """Build idempotent events from already acquired document versions."""
+
+    events: list[dict[str, Any]] = []
+    for url in sorted(set(old_records) | set(new_records)):
+        old_items = old_records.get(url, [])
+        new_items = new_records.get(url, [])
+        old_hashes = sorted(_hashes(old_items))
+        new_hashes = sorted(_hashes(new_items))
+        if old_hashes == new_hashes and _titles(old_items) == _titles(new_items):
+            continue
+        old_primary = old_items[0] if old_items else None
+        new_primary = new_items[0] if new_items else None
+        old_text = _combined_content(old_items)
+        new_text = _combined_content(new_items)
+        classifications = _classifications(old_text, new_text, url)
+        structural = _structural_changes(old_items, new_items)
+        textual = _textual_changes(old_text, new_text)
+        old_evidence = _record_evidence(old_primary, old_text) if old_primary else []
+        new_evidence = _record_evidence(new_primary, new_text) if new_primary else []
+        identity = {
+            "workflow": workflow,
+            "url": url,
+            "old_hashes": old_hashes,
+            "new_hashes": new_hashes,
+        }
+        idempotency_key = canonical_sha256(identity)
+        semantic_candidates = [
+            {
+                "classification": item,
+                "status": "candidate",
+                "confidence": _classification_confidence(item, old_text, new_text, url),
+                "requires_review": True,
+            }
+            for item in classifications
+        ]
+        event = ChangeEvent(
+            event_id=stable_id("change_event", identity),
+            idempotency_key=idempotency_key,
+            workflow=workflow,
+            url=url,
+            old_document_id=_string(old_primary, "document_id"),
+            new_document_id=_string(new_primary, "document_id"),
+            old_hash=canonical_sha256(old_hashes) if old_hashes else None,
+            new_hash=canonical_sha256(new_hashes) if new_hashes else None,
+            old_evidence=old_evidence,
+            new_evidence=new_evidence,
+            structural_changes=structural,
+            textual_changes=textual,
+            semantic_candidates=semantic_candidates,
+            classifications=classifications,
+            replay_configuration=ReplayConfiguration(
+                configuration={
+                    "workflow": workflow,
+                    "old_hashes": old_hashes,
+                    "new_hashes": new_hashes,
+                }
+            ),
+        )
+        events.append(event.model_dump(mode="json", exclude_none=True))
+    return events
+
+
+def write_change_events(path: Path, events: list[dict[str, Any]]) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        "".join(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n" for event in events),
+        encoding="utf-8",
+    )
+    return path
+
+
+def _record_evidence(record: dict[str, Any], content: str) -> list[Any]:
+    if not content:
+        return []
+    excerpt = content[: min(600, len(content))]
+    citation_id = str(record.get("source_citation_id") or "S0")
+    record_citation_id = record.get("record_citation_id")
+    return [
+        evidence_span(
+            url=str(record.get("url") or ""),
+            content=content,
+            exact_text=excerpt,
+            citation_id=citation_id,
+            record_citation_id=str(record_citation_id) if record_citation_id else None,
+        )
+    ]
+
+
+def _structural_changes(
+    old_items: list[dict[str, Any]],
+    new_items: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    changes: list[dict[str, Any]] = []
+    if not old_items:
+        changes.append({"type": "document_added", "new_record_count": len(new_items)})
+    elif not new_items:
+        changes.append({"type": "document_removed", "old_record_count": len(old_items)})
+    if len(old_items) != len(new_items):
+        changes.append(
+            {
+                "type": "record_count_changed",
+                "before": len(old_items),
+                "after": len(new_items),
+            }
+        )
+    if _titles(old_items) != _titles(new_items):
+        changes.append({"type": "title_changed", "before": _titles(old_items), "after": _titles(new_items)})
+    return changes
+
+
+def _textual_changes(old_text: str, new_text: str) -> list[dict[str, Any]]:
+    old_lines = old_text.splitlines()
+    new_lines = new_text.splitlines()
+    matcher = difflib.SequenceMatcher(a=old_lines, b=new_lines, autojunk=False)
+    changes: list[dict[str, Any]] = []
+    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+        if tag == "equal":
+            continue
+        before = "\n".join(old_lines[i1:i2])[:1000]
+        after = "\n".join(new_lines[j1:j2])[:1000]
+        changes.append(
+            {
+                "type": tag,
+                "old_line_start": i1,
+                "old_line_end": i2,
+                "new_line_start": j1,
+                "new_line_end": j2,
+                "before": before,
+                "after": after,
+                "before_sha256": canonical_sha256(before),
+                "after_sha256": canonical_sha256(after),
+            }
+        )
+        if len(changes) >= 100:
+            break
+    return changes
+
+
+def _classifications(old_text: str, new_text: str, url: str) -> list[ChangeClassification]:
+    haystack = f"{url}\n{old_text}\n{new_text}".lower()
+    output = [
+        classification
+        for classification, terms in _CLASSIFICATION_TERMS.items()
+        if any(term in haystack for term in terms)
+    ]
+    return output or ["other"]
+
+
+def _classification_confidence(
+    classification: ChangeClassification,
+    old_text: str,
+    new_text: str,
+    url: str,
+) -> float:
+    terms = _CLASSIFICATION_TERMS.get(classification, ())
+    haystack = f"{url}\n{old_text}\n{new_text}".lower()
+    hits = sum(1 for term in terms if term in haystack)
+    return min(0.95, 0.5 + hits * 0.1) if terms else 0.4
+
+
+def _combined_content(items: list[dict[str, Any]]) -> str:
+    return "\n\n".join(str(item.get("content") or "") for item in items)
+
+
+def _hashes(items: list[dict[str, Any]]) -> set[str]:
+    return {str(item.get("content_hash")) for item in items if item.get("content_hash")}
+
+
+def _titles(items: list[dict[str, Any]]) -> list[str]:
+    return sorted({str(item.get("title") or "") for item in items})
+
+
+def _string(item: dict[str, Any] | None, key: str) -> str | None:
+    if not item or item.get(key) is None:
+        return None
+    value = str(item[key]).strip()
+    return value or None
+
+
+__all__ = ["build_change_events", "write_change_events"]

{docpull-6.1.0 → docpull-6.2.0}/src/docpull/cli.py

@@ -1489,6 +1489,10 @@ def main(argv: list[str] | None = None) -> int:
         from .pack_tools import run_pack_cli
 
         return run_pack_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "contracts":
+        from .contracts_cli import run_contracts_cli
+
+        return run_contracts_cli(raw_argv[1:])
     if raw_argv and raw_argv[0] == "graph":
         from .graph import run_graph_cli
 
@@ -1525,6 +1529,32 @@ def main(argv: list[str] | None = None) -> int:
         from .monitor import run_monitor_cli
 
         return run_monitor_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] in {
+        "brand-pack",
+        "product-pack",
+        "styleguide-pack",
+        "image-pack",
+        "screenshot-pack",
+        "policy-pack",
+    }:
+        from .context_packs.workflow_cli import (
+            run_brand_pack_cli,
+            run_image_pack_cli,
+            run_policy_pack_cli,
+            run_product_pack_cli,
+            run_screenshot_pack_cli,
+            run_styleguide_pack_cli,
+        )
+
+        workflow_runners = {
+            "brand-pack": run_brand_pack_cli,
+            "product-pack": run_product_pack_cli,
+            "styleguide-pack": run_styleguide_pack_cli,
+            "image-pack": run_image_pack_cli,
+            "screenshot-pack": run_screenshot_pack_cli,
+            "policy-pack": run_policy_pack_cli,
+        }
+        return workflow_runners[raw_argv[0]](raw_argv[1:])
     if raw_argv and raw_argv[0] == "openapi-pack":
         from .context_packs.cli import run_openapi_pack_cli
 

{docpull-6.1.0 → docpull-6.2.0}/src/docpull/context_packs/__init__.py

@@ -3,14 +3,19 @@
 from __future__ import annotations
 
 from ..surface import PUBLIC_CONTEXT_PACK_EXPORTS
+from .brand import build_brand_pack
 from .dataset import async_build_dataset_pack, build_dataset_pack
 from .feed import build_feed_pack
 from .openapi import build_openapi_pack
 from .package import async_build_package_pack, build_package_pack
 from .paper import async_build_paper_pack, build_paper_pack
+from .policy_pack import build_policy_pack
+from .product import build_product_pack
 from .repo import async_build_repo_pack, build_repo_pack
 from .standards import async_build_standards_pack, build_standards_pack
+from .styleguide import build_styleguide_pack
 from .transcript import async_build_transcript_pack, build_transcript_pack
+from .visuals import build_image_pack, capture_screenshot_pack
 from .wiki import async_build_wiki_pack, build_wiki_pack
 
 __all__ = [
@@ -21,15 +26,21 @@ __all__ = [
     "async_build_standards_pack",
     "async_build_transcript_pack",
     "async_build_wiki_pack",
+    "build_brand_pack",
     "build_dataset_pack",
     "build_feed_pack",
     "build_openapi_pack",
     "build_package_pack",
     "build_paper_pack",
+    "build_policy_pack",
+    "build_product_pack",
     "build_repo_pack",
     "build_standards_pack",
+    "build_styleguide_pack",
     "build_transcript_pack",
     "build_wiki_pack",
+    "build_image_pack",
+    "capture_screenshot_pack",
 ]
 
 assert tuple(__all__) == PUBLIC_CONTEXT_PACK_EXPORTS

{docpull-6.1.0 → docpull-6.2.0}/src/docpull/context_packs/_legacy_cli.py

@@ -1,8 +1,7 @@
-"""Private CLI adapters for legacy context-pack experiments.
+"""Compatibility import path for the original context-pack CLI adapters.
 
-These functions are intentionally not dispatched from ``docpull``. They remain
-only for internal compatibility tests and private workflows that still exercise
-the old experimental pack builders.
+The commands are public again through :mod:`docpull.context_packs.workflow_cli`;
+this module remains importable so pre-6.2 integrations do not break.
 """
 
 from __future__ import annotations