sift-engine 0.2.0__tar.gz → 0.3.0__tar.gz

{sift_engine-0.2.0/sift_engine.egg-info → sift_engine-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sift-engine
-Version: 0.2.0
+Version: 0.3.0
 Summary: Sift — deterministic website indexing for grep-first LLM agents
 Author: Deval Shah
 License-Expression: Apache-2.0
@@ -21,7 +21,9 @@ Requires-Dist: lxml>=5.0
 Requires-Dist: selectolax>=0.3
 Requires-Dist: mcp>=1.0
 Requires-Dist: pypdf>=4.0
+Requires-Dist: pdfplumber>=0.11
 Requires-Dist: jsonschema>=4
+Requires-Dist: protego>=0.3
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
@@ -62,8 +64,8 @@ Your agent's built-in web-fetch gives you:
 
 ## ✅ With sift
 
-- **Always fresh** — you control the crawl; conditional GETs refetch only what changed
-- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot
+- **Always fresh** — you control the crawl; conditional GETs refetch only what changed, and `changed_since` lets your agent pull just the delta instead of re-reading the corpus
+- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot, read any past snapshot (`as_of`), or emit a self-contained `prove` inclusion proof a third party verifies offline (`python -m sift.verify_proof`) without trusting the server — optionally anchored by an RFC-3161 timestamp so the date is witnessed by a third party, not self-asserted
 - **Complete & grep-native** — the whole site on disk, not a handful of retrieved snippets
 - **Self-hosted** — any `http(s)` site you can reach, public or internal; your data stays yours
 
@@ -109,7 +111,7 @@ sift --version && which sift-mcp  # confirm it's on PATH
 Download the skill so you (the agent) know how to build, operate, and query an index.
 Read .claude/skills/sift/SKILL.md after downloading — it is the source of truth for the rest.
 ```bash
-SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.2.0/.claude/skills/sift
+SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.3.0/.claude/skills/sift
 mkdir -p .claude/skills/sift/reference
 curl -fsSL $SKILL/SKILL.md -o .claude/skills/sift/SKILL.md
 for f in cli config mcp-tools; do curl -fsSL $SKILL/reference/$f.md -o .claude/skills/sift/reference/$f.md; done
@@ -131,12 +133,22 @@ sift run    --root ./sift-index --config sift.toml --limit 25 --coverage-base pl
 sift verify --root ./sift-index --skip-signature
 ```
 Drop `--limit 25 --coverage-base planned` for a full crawl once extraction looks good.
+Preview exactly which URLs a crawl would touch — and how many — *without fetching a
+byte*: `sift discover --from-domain TARGET_SITE` (writes nothing; emits JSON).
+No `sitemap.xml` (or an incomplete one)? After a run, `sift seed --from-frontier`
+extracts in-scope links from the fetched pages and crawls one hop deeper each pass.
 Hardened / bot-blocked host (Cloudflare/Akamai/Imperva) → add `--impersonate-fallback`
 (free, TLS-fingerprint impersonation; `pip install 'sift-engine[impersonate]'`).
 JS-rendered SPA → `pip install 'sift-engine[browser]' && python -m playwright install chromium`,
 then add `[browser]\nenabled = true` (it joins the ladder as a free render tier).
+JS-rendered SPA that has an official API (eCFR, CVE.org, FederalRegister …) → a
+profile's `api_url(url)` fetches the robots-allowed API instead of the shell —
+cleaner and cheaper than a browser, and the citation stays the human page (see
+the bundled `sift.sites.cve:CVEProfile`).
 Still blocked (JS-challenge edges) → `--firecrawl-fallback` (paid; needs FIRECRAWL_API_KEY).
 These compose into one escalation ladder: native → impersonate → browser → Firecrawl.
+sift respects `robots.txt` `Disallow` at seed by default; set `[crawl]` →
+`respect_robots = false` only for sources you have permission to index.
 
 4 — WIRE THE READ-ONLY MCP SERVER
 Use the ABSOLUTE path to ./sift-index. Add this to the project's .mcp.json (Claude Code / Cursor / Codex):
@@ -204,7 +216,7 @@ Your agent reads the published snapshot **read-only** over MCP: `snapshot_status
 
 > **Open core, Apache-2.0.** This repo is the full open-source engine (pipeline + MCP server) and runs standalone. A hosted platform built on it is in development.
 
-**Status — v0.2.0**; tests green on Python 3.11–3.13. Adds the tiered fetch transport (native → curl_cffi → browser → Firecrawl) for hardened and JS-rendered sites. Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
+**Status — v0.3.0**; tests green on Python 3.11–3.13. Adds proof-carrying answers (`prove` / `verify-proof` + an RFC-3161 timestamp anchor), time-travel reads (`changed_since`, `diff_md`, `as_of` over retained snapshots), API acquisition/content transports (`api_url` source + `json` extract), robots + SSRF + bot-challenge hardening, digital-PDF tables, a recursive link frontier, and `sift discover` (read-only pre-crawl preview). Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
 
 ## License
 

{sift_engine-0.2.0 → sift_engine-0.3.0}/README.md

@@ -24,8 +24,8 @@ Your agent's built-in web-fetch gives you:
 
 ## ✅ With sift
 
-- **Always fresh** — you control the crawl; conditional GETs refetch only what changed
-- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot
+- **Always fresh** — you control the crawl; conditional GETs refetch only what changed, and `changed_since` lets your agent pull just the delta instead of re-reading the corpus
+- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot, read any past snapshot (`as_of`), or emit a self-contained `prove` inclusion proof a third party verifies offline (`python -m sift.verify_proof`) without trusting the server — optionally anchored by an RFC-3161 timestamp so the date is witnessed by a third party, not self-asserted
 - **Complete & grep-native** — the whole site on disk, not a handful of retrieved snippets
 - **Self-hosted** — any `http(s)` site you can reach, public or internal; your data stays yours
 
@@ -71,7 +71,7 @@ sift --version && which sift-mcp  # confirm it's on PATH
 Download the skill so you (the agent) know how to build, operate, and query an index.
 Read .claude/skills/sift/SKILL.md after downloading — it is the source of truth for the rest.
 ```bash
-SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.2.0/.claude/skills/sift
+SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.3.0/.claude/skills/sift
 mkdir -p .claude/skills/sift/reference
 curl -fsSL $SKILL/SKILL.md -o .claude/skills/sift/SKILL.md
 for f in cli config mcp-tools; do curl -fsSL $SKILL/reference/$f.md -o .claude/skills/sift/reference/$f.md; done
@@ -93,12 +93,22 @@ sift run    --root ./sift-index --config sift.toml --limit 25 --coverage-base pl
 sift verify --root ./sift-index --skip-signature
 ```
 Drop `--limit 25 --coverage-base planned` for a full crawl once extraction looks good.
+Preview exactly which URLs a crawl would touch — and how many — *without fetching a
+byte*: `sift discover --from-domain TARGET_SITE` (writes nothing; emits JSON).
+No `sitemap.xml` (or an incomplete one)? After a run, `sift seed --from-frontier`
+extracts in-scope links from the fetched pages and crawls one hop deeper each pass.
 Hardened / bot-blocked host (Cloudflare/Akamai/Imperva) → add `--impersonate-fallback`
 (free, TLS-fingerprint impersonation; `pip install 'sift-engine[impersonate]'`).
 JS-rendered SPA → `pip install 'sift-engine[browser]' && python -m playwright install chromium`,
 then add `[browser]\nenabled = true` (it joins the ladder as a free render tier).
+JS-rendered SPA that has an official API (eCFR, CVE.org, FederalRegister …) → a
+profile's `api_url(url)` fetches the robots-allowed API instead of the shell —
+cleaner and cheaper than a browser, and the citation stays the human page (see
+the bundled `sift.sites.cve:CVEProfile`).
 Still blocked (JS-challenge edges) → `--firecrawl-fallback` (paid; needs FIRECRAWL_API_KEY).
 These compose into one escalation ladder: native → impersonate → browser → Firecrawl.
+sift respects `robots.txt` `Disallow` at seed by default; set `[crawl]` →
+`respect_robots = false` only for sources you have permission to index.
 
 4 — WIRE THE READ-ONLY MCP SERVER
 Use the ABSOLUTE path to ./sift-index. Add this to the project's .mcp.json (Claude Code / Cursor / Codex):
@@ -166,7 +176,7 @@ Your agent reads the published snapshot **read-only** over MCP: `snapshot_status
 
 > **Open core, Apache-2.0.** This repo is the full open-source engine (pipeline + MCP server) and runs standalone. A hosted platform built on it is in development.
 
-**Status — v0.2.0**; tests green on Python 3.11–3.13. Adds the tiered fetch transport (native → curl_cffi → browser → Firecrawl) for hardened and JS-rendered sites. Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
+**Status — v0.3.0**; tests green on Python 3.11–3.13. Adds proof-carrying answers (`prove` / `verify-proof` + an RFC-3161 timestamp anchor), time-travel reads (`changed_since`, `diff_md`, `as_of` over retained snapshots), API acquisition/content transports (`api_url` source + `json` extract), robots + SSRF + bot-challenge hardening, digital-PDF tables, a recursive link frontier, and `sift discover` (read-only pre-crawl preview). Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
 
 ## License
 

{sift_engine-0.2.0 → sift_engine-0.3.0}/evals/bench/per_stage/plan.py

@@ -57,9 +57,8 @@ def eval_plan_decision_correctness() -> PlanDecisionCorrectnessResult:
     from sift.config import IndexConfig, TierConfig
     from sift.classify import CLASSIFIER_VERSION
     from sift.extract import EXTRACTOR_VERSION
-    from sift.normalize import NORMALIZER_VERSION
+    from sift.normalize import normalizer_version
     from sift import decide as decide_mod
-    from sift import sites as sites_mod
     from sift import paths
 
     # Set up an in-memory-ish manifest in a temp dir
@@ -151,7 +150,7 @@ def eval_plan_decision_correctness() -> PlanDecisionCorrectnessResult:
         plan_phase(
             conn, plan_path, now=now,
             extractor_version=EXTRACTOR_VERSION,
-            normalizer_version=NORMALIZER_VERSION,
+            normalizer_version=normalizer_version(),
             profile=GenericProfile(),
             cfg=cfg,
         )

{sift_engine-0.2.0 → sift_engine-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sift-engine"
-version = "0.2.0"
+version = "0.3.0"
 description = "Sift — deterministic website indexing for grep-first LLM agents"
 readme = "README.md"
 license = "Apache-2.0"
@@ -24,7 +24,9 @@ dependencies = [
     "selectolax>=0.3",
     "mcp>=1.0",
     "pypdf>=4.0",
+    "pdfplumber>=0.11",
     "jsonschema>=4",
+    "protego>=0.3",
 ]
 
 [project.optional-dependencies]

{sift_engine-0.2.0 → sift_engine-0.3.0}/sift/__init__.py

@@ -16,7 +16,7 @@ bumping a version triggers re-derivation from cached raw, not refetch.
 See README.md for the full design.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"
 # Provenance stamp on manifest rows — a *behavioral* version (like
 # EXTRACTOR_VERSION), bumped only when crawl/fetch behavior changes, not on
 # every package release. this is the initial public release; crawl behavior is unchanged, so this stays.