docpull 6.3.0__tar.gz → 6.4.0__tar.gz

{docpull-6.3.0/src/docpull.egg-info → docpull-6.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.3.0
+Version: 6.4.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -307,6 +307,7 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull dataset-pack 'https://data.example.org/rows.json?$limit=100' -o packs/dataset
 docpull relationship-pack example.com --subject "Example Brand" -o packs/relationships
+docpull website-pack example.com -o packs/website
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
 docpull brand-pack example.com -o packs/brand
@@ -320,6 +321,9 @@ docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
 docpull export packs/docs --format cursor-rules -o .cursor/rules --skill-name docs
 ```
 
+The canonical baseline-aware website acquisition contract is documented in
+[Website snapshots](docs/website-snapshot.md).
+
 Use `docpull ci --prepare` to validate a project or standalone pack in CI.
 
 Install optional extras as needed:
@@ -772,7 +776,7 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull wiki-pack`, `docpull website-pack`, `docpull brand-pack`, `docpull product-pack`,
   `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,

{docpull-6.3.0 → docpull-6.4.0}/README.md

@@ -183,6 +183,7 @@ docpull standards-pack rfc:9110 -o packs/standard
 docpull dataset-pack ./metrics.csv -o packs/dataset
 docpull dataset-pack 'https://data.example.org/rows.json?$limit=100' -o packs/dataset
 docpull relationship-pack example.com --subject "Example Brand" -o packs/relationships
+docpull website-pack example.com -o packs/website
 docpull transcript-pack ./meeting.vtt -o packs/transcript
 docpull wiki-pack wiki:Web_scraping -o packs/wiki
 docpull brand-pack example.com -o packs/brand
@@ -196,6 +197,9 @@ docpull export packs/docs --format openai-vector-jsonl -o exports/openai.jsonl
 docpull export packs/docs --format cursor-rules -o .cursor/rules --skill-name docs
 ```
 
+The canonical baseline-aware website acquisition contract is documented in
+[Website snapshots](docs/website-snapshot.md).
+
 Use `docpull ci --prepare` to validate a project or standalone pack in CI.
 
 Install optional extras as needed:
@@ -648,7 +652,7 @@ part of the package release contract.
   `docpull parse`, `docpull openapi-pack`, `docpull feed-pack`,
   `docpull paper-pack`, `docpull repo-pack`, `docpull package-pack`,
   `docpull standards-pack`, `docpull dataset-pack`, `docpull transcript-pack`,
-  `docpull wiki-pack`, `docpull brand-pack`, `docpull product-pack`,
+  `docpull wiki-pack`, `docpull website-pack`, `docpull brand-pack`, `docpull product-pack`,
   `docpull styleguide-pack`, `docpull image-pack`, `docpull policy-pack`,
   `docpull pack validate`,
   `docpull pack audit`, `docpull export`,

{docpull-6.3.0 → docpull-6.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.3.0"
+version = "6.4.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/__init__.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any
 
 from .surface import PUBLIC_SDK_EXPORTS
 
-__version__ = "6.3.0"
+__version__ = "6.4.0"
 
 _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
     **{
@@ -41,9 +41,11 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
             "build_repo_pack",
             "build_standards_pack",
             "build_styleguide_pack",
+            "build_website_pack",
             "build_transcript_pack",
             "build_wiki_pack",
             "capture_screenshot_pack",
+            "validate_website_snapshot_pack",
         )
     },
     **{
@@ -59,6 +61,8 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
             "SourceAuthority",
             "WorkflowRequest",
             "WorkflowResult",
+            "WebsiteSnapshot",
+            "WebsiteSnapshotDocument",
             "bundled_schema_path",
             "write_contract_schemas",
         )
@@ -198,8 +202,10 @@ if TYPE_CHECKING:
         build_standards_pack,
         build_styleguide_pack,
         build_transcript_pack,
+        build_website_pack,
         build_wiki_pack,
         capture_screenshot_pack,
+        validate_website_snapshot_pack,
     )
     from .contracts import (
         ArtifactManifest,
@@ -210,6 +216,8 @@ if TYPE_CHECKING:
         RelationshipCandidate,
         RelationshipPack,
         SourceAuthority,
+        WebsiteSnapshot,
+        WebsiteSnapshotDocument,
         WorkflowRequest,
         WorkflowResult,
         bundled_schema_path,

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/cli.py

@@ -1661,6 +1661,7 @@ def main(argv: list[str] | None = None) -> int:
         return run_monitor_cli(raw_argv[1:])
     if raw_argv and raw_argv[0] in {
         "brand-pack",
+        "website-pack",
         "product-pack",
         "styleguide-pack",
         "image-pack",
@@ -1676,9 +1677,11 @@ def main(argv: list[str] | None = None) -> int:
             run_relationship_pack_cli,
             run_screenshot_pack_cli,
             run_styleguide_pack_cli,
+            run_website_pack_cli,
         )
 
         workflow_runners = {
+            "website-pack": run_website_pack_cli,
             "brand-pack": run_brand_pack_cli,
             "product-pack": run_product_pack_cli,
             "styleguide-pack": run_styleguide_pack_cli,

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/context_packs/__init__.py

@@ -25,6 +25,8 @@ _LAZY_EXPORTS = {
     **{name: (".transcript", name) for name in ("async_build_transcript_pack", "build_transcript_pack")},
     **{name: (".wiki", name) for name in ("async_build_wiki_pack", "build_wiki_pack")},
     "capture_screenshot_pack": (".visuals", "capture_screenshot_pack"),
+    "build_website_pack": (".website", "build_website_pack"),
+    "validate_website_snapshot_pack": (".website", "validate_website_snapshot_pack"),
 }
 
 __all__ = list(PUBLIC_CONTEXT_PACK_EXPORTS)
@@ -59,6 +61,7 @@ if TYPE_CHECKING:
     from .styleguide import build_styleguide_pack
     from .transcript import async_build_transcript_pack, build_transcript_pack
     from .visuals import build_image_pack, capture_screenshot_pack
+    from .website import build_website_pack, validate_website_snapshot_pack
     from .wiki import async_build_wiki_pack, build_wiki_pack
 
 

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/context_packs/brand.py

@@ -29,6 +29,7 @@ from .common import (
     homepage_url_for_domain,
     is_free_or_disposable_email_domain,
     likely_internal_pages,
+    pages_from_local_pack,
     public_url,
     quote_markdown,
     resolve_url,
@@ -92,7 +93,14 @@ def build_brand_pack(
     max_pages: int = MAX_BRAND_PAGES,
 ) -> dict[str, Any]:
     """Build a local brand profile from public website evidence."""
-    domain = _resolve_brand_domain(domain_or_url, email=email, allow_free_email=allow_free_email)
+    local_pages = pages_from_local_pack(domain_or_url)
+    domain = (
+        domain_from_input(local_pages[0].url)
+        if local_pages
+        else _resolve_brand_domain(domain_or_url, email=email, allow_free_email=allow_free_email)
+    )
+    if not domain:
+        raise ContextPackError("Could not resolve a domain from local brand-pack evidence.")
     policy = ensure_policy_for_domain(policy, domain)
     output_dir = output_dir.resolve()
     run = ContextPackRun(
@@ -101,14 +109,20 @@ def build_brand_pack(
         policy=policy,
         input_value=email or domain_or_url,
     )
-    home_url = homepage_url_for_domain(domain)
-    home_pages = fetch_pages_blocking([home_url], run=run, max_pages=1)
-    if not home_pages:
-        raise ContextPackError(f"Could not fetch homepage for {domain}.")
-    candidate_urls = likely_internal_pages(home_pages[0], domain, max_pages=max_pages)
-    pages = fetch_pages_blocking(candidate_urls, run=run, max_pages=max_pages)
-    if not pages:
-        pages = home_pages
+    if local_pages is not None:
+        pages = local_pages[:max_pages]
+        if not pages:
+            raise ContextPackError("Local brand-pack input contains no readable documents.")
+        download_assets = False
+    else:
+        home_url = homepage_url_for_domain(domain)
+        home_pages = fetch_pages_blocking([home_url], run=run, max_pages=1)
+        if not home_pages:
+            raise ContextPackError(f"Could not fetch homepage for {domain}.")
+        candidate_urls = likely_internal_pages(home_pages[0], domain, max_pages=max_pages)
+        pages = fetch_pages_blocking(candidate_urls, run=run, max_pages=max_pages)
+        if not pages:
+            pages = home_pages
 
     profile = _extract_brand_profile(pages, domain=domain, name_hint=name, ticker=ticker)
     assets = _extract_brand_assets(

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/context_packs/common.py

@@ -40,6 +40,7 @@ from ..models.document import DocumentRecord
 from ..models.events import SkipReason
 from ..models.run import RunIdentity
 from ..output_contract import OUTPUT_CONTRACT_SCHEMA_VERSION, write_raw_contract_sidecars
+from ..pack_reader import PackReadError, load_pack, resolve_pack_path
 from ..pack_tools import build_citation_map
 from ..policy import PolicyConfig, policy_domain_matches
 from ..security.download_policy import SafeDownloadPolicy, UnsafeDownloadError, content_type_base
@@ -118,6 +119,8 @@ class PageSnapshot:
     metadata: dict[str, Any]
     extraction: dict[str, Any]
     source_type: str | None = None
+    etag: str | None = None
+    last_modified: str | None = None
 
 
 @dataclass(frozen=True)
@@ -214,6 +217,9 @@ class ContextPackRun:
     http_request_count: int = 0
     cache_hit_count: int = 0
     progress_events: list[dict[str, Any]] = field(default_factory=list)
+    fetch_output_dir: Path | None = None
+    cache_dir: Path | None = None
+    unchanged_urls: set[str] = field(default_factory=set)
 
     def __post_init__(self) -> None:
         self.progress("run", "started", message=f"Started {self.workflow}", timestamp=self.started_at)
@@ -444,10 +450,17 @@ async def fetch_pages(
     run.progress("acquisition", "started", total=len(selected), message="Acquiring source pages")
     config = DocpullConfig(url=selected[0], profile=ProfileName.CUSTOM)
     config.network.log_retry_warnings = False
+    if run.fetch_output_dir is not None:
+        config.output.directory = run.fetch_output_dir
+    if run.cache_dir is not None:
+        config.cache.enabled = True
+        config.cache.directory = run.cache_dir
+        config.cache.ttl_days = None
+        config.cache.skip_unchanged = True
     snapshots: list[PageSnapshot] = []
     async with Fetcher(config) as fetcher:
         for index, url in enumerate(selected, start=1):
-            ctx = await fetcher.fetch_one(url, save=False)
+            ctx = await fetcher.fetch_one(url, save=run.fetch_output_dir is not None)
             run.http_request_count += 1
             if ctx.error:
                 run.errors.append(
@@ -462,6 +475,10 @@ async def fetch_pages(
                 )
                 continue
             if ctx.should_skip:
+                if ctx.skip_code == SkipReason.CACHE_UNCHANGED:
+                    run.cache_hit_count += 1
+                    run.unchanged_urls.add(public_url(url))
+                    continue
                 if ctx.skip_code in {
                     SkipReason.HTTP_ERROR,
                     SkipReason.ROBOTS_DISALLOWED,
@@ -503,6 +520,8 @@ async def fetch_pages(
                     metadata=dict(ctx.metadata or {}),
                     extraction=dict(ctx.extraction_info or {}),
                     source_type=ctx.source_type,
+                    etag=getattr(ctx, "etag", None),
+                    last_modified=getattr(ctx, "last_modified", None),
                 )
             )
             run.progress(
@@ -531,6 +550,60 @@ def fetch_pages_blocking(
     return asyncio.run(fetch_pages(urls, run=run, max_pages=max_pages))
 
 
+def pages_from_local_pack(value: str | Path) -> list[PageSnapshot] | None:
+    """Load an existing DocPull pack without permitting any network fallback."""
+
+    path = Path(value).expanduser()
+    if not path.exists():
+        return None
+    try:
+        pack = load_pack(path)
+    except PackReadError as err:
+        raise ContextPackError(f"Could not read local DocPull pack {path}: {err}") from err
+    raw_by_identity: dict[tuple[str, str], str] = {}
+    website_metadata_by_identity: dict[tuple[str, str], dict[str, Any]] = {}
+    records_path = path.resolve() / "documents.ndjson"
+    if records_path.is_file():
+        for line in records_path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(row, dict):
+                continue
+            identity = (str(row.get("document_id") or ""), str(row.get("content_hash") or ""))
+            website_metadata_by_identity[identity] = {
+                key: row[key]
+                for key in ("page_role", "entity_reference", "authority", "representations")
+                if row.get(key) is not None
+            }
+            representations = row.get("representations")
+            raw = representations.get("raw") if isinstance(representations, dict) else None
+            raw_path = resolve_pack_path(path.resolve(), raw.get("path") if isinstance(raw, dict) else None)
+            if raw_path and raw_path.is_file():
+                raw_by_identity[identity] = raw_path.read_text(encoding="utf-8")
+    pages: list[PageSnapshot] = []
+    for record in pack.documents:
+        identity = (record.document_id, record.content_hash)
+        pages.append(
+            PageSnapshot(
+                url=public_url(record.url),
+                title=record.title,
+                html=raw_by_identity.get(identity, ""),
+                markdown=record.content,
+                metadata={
+                    **dict(record.metadata),
+                    **website_metadata_by_identity.get(identity, {}),
+                },
+                extraction={**record.extraction, "local_pack": str(path.resolve())},
+                source_type=record.source_type or "local-pack",
+            )
+        )
+    return pages
+
+
 def soup_for(page: PageSnapshot) -> BeautifulSoup:
     return BeautifulSoup(page.html or page.markdown, "html.parser")
 

{docpull-6.3.0 → docpull-6.4.0}/src/docpull/context_packs/policy_pack.py

@@ -24,6 +24,7 @@ from .common import (
     extract_links,
     fetch_pages_blocking,
     homepage_url_for_domain,
+    pages_from_local_pack,
     public_url,
     quote_markdown,
     same_policy_domain,
@@ -73,7 +74,8 @@ def build_policy_pack(
 ) -> dict[str, Any]:
     """Discover policy pages and emit neutral, clause-level evidence records."""
 
-    domain = domain_from_input(domain_or_url)
+    local_pages = pages_from_local_pack(domain_or_url)
+    domain = domain_from_input(local_pages[0].url) if local_pages else domain_from_input(domain_or_url)
     if not domain:
         raise ContextPackError("Could not resolve a domain from policy-pack input.")
     effective_policy = ensure_policy_for_domain(policy, domain)
@@ -84,14 +86,21 @@ def build_policy_pack(
         input_value=domain_or_url,
     )
     start_url = public_url(domain_or_url if "://" in domain_or_url else homepage_url_for_domain(domain))
-    discovery_pages = fetch_pages_blocking([start_url], run=run, max_pages=1)
+    discovery_pages = (
+        local_pages[:max_pages]
+        if local_pages is not None
+        else fetch_pages_blocking([start_url], run=run, max_pages=1)
+    )
     if not discovery_pages:
-        raise ContextPackError(f"Could not fetch policy discovery target: {start_url}")
+        raise ContextPackError(f"Could not load policy discovery target: {start_url}")
 
-    urls = _policy_urls(discovery_pages[0], domain=domain, max_pages=max_pages)
-    if _policy_target_hint(discovery_pages[0]) and start_url not in urls:
-        urls.insert(0, start_url)
-    pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages) if urls else []
+    if local_pages is not None:
+        pages = local_pages[:max_pages]
+    else:
+        urls = _policy_urls(discovery_pages[0], domain=domain, max_pages=max_pages)
+        if _policy_target_hint(discovery_pages[0]) and start_url not in urls:
+            urls.insert(0, start_url)
+        pages = fetch_pages_blocking(urls, run=run, max_pages=max_pages) if urls else []
     if not pages:
         run.warn(
             "policy_documents_not_found",