sift-engine 0.1.0__tar.gz → 0.3.0__tar.gz

sift_engine-0.3.0/PKG-INFO

@@ -0,0 +1,223 @@
+Metadata-Version: 2.4
+Name: sift-engine
+Version: 0.3.0
+Summary: Sift — deterministic website indexing for grep-first LLM agents
+Author: Deval Shah
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/dvlshah/sift
+Project-URL: Repository, https://github.com/dvlshah/sift
+Project-URL: Issues, https://github.com/dvlshah/sift/issues
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx>=0.27
+Requires-Dist: trafilatura>=1.12
+Requires-Dist: aiolimiter>=1.1
+Requires-Dist: click>=8.1
+Requires-Dist: lxml>=5.0
+Requires-Dist: selectolax>=0.3
+Requires-Dist: mcp>=1.0
+Requires-Dist: pypdf>=4.0
+Requires-Dist: pdfplumber>=0.11
+Requires-Dist: jsonschema>=4
+Requires-Dist: protego>=0.3
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
+Requires-Dist: respx>=0.21; extra == "dev"
+Provides-Extra: evals
+Requires-Dist: anthropic>=0.40; extra == "evals"
+Requires-Dist: pydantic>=2.0; extra == "evals"
+Requires-Dist: jsonschema>=4; extra == "evals"
+Provides-Extra: browser
+Requires-Dist: playwright>=1.40; extra == "browser"
+Provides-Extra: impersonate
+Requires-Dist: curl_cffi>=0.7; extra == "impersonate"
+Dynamic: license-file
+
+<div align="center">
+
+<img src="assets/sift.gif" alt="sift" width="480">
+
+**Give your coding agent a complete, always-fresh copy of any website — with proof of every answer.**
+
+[![Tests](https://github.com/dvlshah/sift/actions/workflows/tests.yml/badge.svg)](https://github.com/dvlshah/sift/actions/workflows/tests.yml)
+[![PyPI](https://img.shields.io/pypi/v/sift-engine?color=blue)](https://pypi.org/project/sift-engine/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/downloads/)
+[![MCP](https://img.shields.io/badge/MCP-stdio-purple)](https://modelcontextprotocol.io/)
+
+</div>
+
+sift crawls a site into local markdown + structured facts your agent greps, reads, and cites — files on disk, not vectors, read over MCP. Every page is **content-hashed and dated**, so answers trace back to the exact source and snapshot. Re-run anytime; only changed pages are refetched. Self-hosted.
+
+## ❌ Without sift
+
+Your agent's built-in web-fetch gives you:
+
+- **Stale** answers — the provider's crawler cached that page weeks ago
+- **Partial** answers — just the one page it happened to land on
+- **Unprovable** answers — no record of what the source said, or when
+
+## ✅ With sift
+
+- **Always fresh** — you control the crawl; conditional GETs refetch only what changed, and `changed_since` lets your agent pull just the delta instead of re-reading the corpus
+- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot, read any past snapshot (`as_of`), or emit a self-contained `prove` inclusion proof a third party verifies offline (`python -m sift.verify_proof`) without trusting the server — optionally anchored by an RFC-3161 timestamp so the date is witnessed by a third party, not self-asserted
+- **Complete & grep-native** — the whole site on disk, not a handful of retrieved snippets
+- **Self-hosted** — any `http(s)` site you can reach, public or internal; your data stays yours
+
+---
+
+## Get started
+
+Requires Python 3.11+.
+
+### 🤖 Let your coding agent set it up — recommended
+
+Paste one prompt into **Claude Code, Cursor, Codex, or any MCP-aware agent**. It explains sift, asks which site to index, then installs the engine, builds an index, wires up MCP, and shows you how to query it — end to end.
+
+<details>
+<summary><b>📋 Copy the one-paste setup prompt</b></summary>
+
+````text
+Set up **sift** in this project — a deterministic, content-hashed website indexer that
+serves a verifiable markdown corpus to AI agents over MCP (https://github.com/dvlshah/sift).
+Do the steps in order. Needs Python 3.11+.
+
+0 — EXPLAIN, THEN ASK ME (do this BEFORE installing anything)
+First, explain sift to me in 4–6 plain-English lines — assume I've never heard of it:
+  • WHAT it is: it crawls a whole website/docs site into local markdown files (plus structured
+    facts) that you, the agent, can grep / read / cite — files on disk, not a vector database.
+  • CORE FEATURES: (1) complete — the full site, not the few pages a live web-fetch happens to
+    land on; (2) verifiable — every page is content-hashed + dated, so any answer can be proved
+    back to the exact source and snapshot; (3) always-current — re-running the crawl refetches
+    only what changed; (4) read over MCP — you query it with provenance, read-only by default.
+  • WHY IT'S NEEDED: a one-off scrape gives you 3 pages with no proof of what they said or when.
+    sift keeps an agent correct about an evolving body of docs AND able to prove what it cited.
+    (If someone only needs one page once, sift is the wrong tool — a plain fetch is fine.)
+Then ASK ME: "Which website or docs site do you want to index locally?" — wait for my answer
+and use it as TARGET_SITE everywhere below. Do not run any commands until I answer.
+
+1 — INSTALL THE ENGINE
+```bash
+pip install sift-engine          # adds the `sift`, `sift-mcp`, `sift-evals` commands
+sift --version && which sift-mcp  # confirm it's on PATH
+```
+
+2 — INSTALL THE SIFT SKILL INTO THIS REPO
+Download the skill so you (the agent) know how to build, operate, and query an index.
+Read .claude/skills/sift/SKILL.md after downloading — it is the source of truth for the rest.
+```bash
+SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.3.0/.claude/skills/sift
+mkdir -p .claude/skills/sift/reference
+curl -fsSL $SKILL/SKILL.md -o .claude/skills/sift/SKILL.md
+for f in cli config mcp-tools; do curl -fsSL $SKILL/reference/$f.md -o .claude/skills/sift/reference/$f.md; done
+```
+
+3 — BUILD A SMALL STARTER INDEX (using the TARGET_SITE I gave you in step 0)
+Write a sift.toml (generic profile + host allow-list = the host of TARGET_SITE), then build a
+capped, publishable smoke-test index:
+```bash
+cat > sift.toml <<'TOML'
+[site]
+profile = "sift.sites.generic:GenericProfile"
+[seed]
+host_allow = ["HOST_OF_TARGET_SITE"]      # e.g. docs.example.com — derive from TARGET_SITE
+TOML
+sift init   --root ./sift-index
+sift seed   --root ./sift-index --config sift.toml --from-domain TARGET_SITE
+sift run    --root ./sift-index --config sift.toml --limit 25 --coverage-base planned
+sift verify --root ./sift-index --skip-signature
+```
+Drop `--limit 25 --coverage-base planned` for a full crawl once extraction looks good.
+Preview exactly which URLs a crawl would touch — and how many — *without fetching a
+byte*: `sift discover --from-domain TARGET_SITE` (writes nothing; emits JSON).
+No `sitemap.xml` (or an incomplete one)? After a run, `sift seed --from-frontier`
+extracts in-scope links from the fetched pages and crawls one hop deeper each pass.
+Hardened / bot-blocked host (Cloudflare/Akamai/Imperva) → add `--impersonate-fallback`
+(free, TLS-fingerprint impersonation; `pip install 'sift-engine[impersonate]'`).
+JS-rendered SPA → `pip install 'sift-engine[browser]' && python -m playwright install chromium`,
+then add `[browser]\nenabled = true` (it joins the ladder as a free render tier).
+JS-rendered SPA that has an official API (eCFR, CVE.org, FederalRegister …) → a
+profile's `api_url(url)` fetches the robots-allowed API instead of the shell —
+cleaner and cheaper than a browser, and the citation stays the human page (see
+the bundled `sift.sites.cve:CVEProfile`).
+Still blocked (JS-challenge edges) → `--firecrawl-fallback` (paid; needs FIRECRAWL_API_KEY).
+These compose into one escalation ladder: native → impersonate → browser → Firecrawl.
+sift respects `robots.txt` `Disallow` at seed by default; set `[crawl]` →
+`respect_robots = false` only for sources you have permission to index.
+
+4 — WIRE THE READ-ONLY MCP SERVER
+Use the ABSOLUTE path to ./sift-index. Add this to the project's .mcp.json (Claude Code / Cursor / Codex):
+```json
+{ "mcpServers": { "sift": { "command": "sift-mcp", "args": ["--root", "ABSOLUTE/PATH/TO/sift-index"] } } }
+```
+Claude Code shortcut: `claude mcp add sift -- sift-mcp --root "$(pwd)/sift-index"`
+Then restart the MCP client so the `sift` tools load.
+
+5 — SHOW ME THE QUERY LOOP
+Call the `snapshot_status` tool to confirm the index is published, then explain the loop:
+snapshot_status first → grep_corpus to locate → read_md / read_facts to drill in →
+cite source_url + content_hash + fetched_at. Mention that re-running `sift run` refreshes the
+index, and that the /sift skill covers building, operating, and querying in depth.
+
+(TARGET_SITE = the site I name when you ask in step 0, e.g. https://docs.example.com)
+````
+
+</details>
+
+### 🧑 Or do it yourself
+
+```bash
+pip install sift-engine
+
+# the ATO sitemap below uses a bundled profile, so this runs with zero config
+sift init   --root ./index
+sift seed   --root ./index --from-sitemap https://www.ato.gov.au/sitemap.xml
+sift run    --root ./index --limit 25 --coverage-base planned   # smoke-test first
+sift verify --root ./index --skip-signature
+sift-mcp    --root ./index   # serve the index to your agent
+```
+
+Then point your agent at it (use an absolute path):
+
+```json
+{
+  "mcpServers": {
+    "sift": { "command": "sift-mcp", "args": ["--root", "/abs/path/to/index"] }
+  }
+}
+```
+
+Indexing your own site? Add a `sift.toml` (generic profile + host allow-list) — see [Configuration](.claude/skills/sift/reference/config.md).
+
+---
+
+## How it works
+
+```
+seed → plan → fetch → extract → commit → publish
+```
+
+Five idempotent phases. `publish` runs 5 verification gates, then atomically swaps the `current/` snapshot and writes a Merkle root over every page hash. Deterministic: **same input → same `content_hash` → same Merkle root** — so any reader re-verifies a page in O(1), or the whole snapshot end-to-end with `sift verify`.
+
+Your agent reads the published snapshot **read-only** over MCP: `snapshot_status` → `grep_corpus` → `read_md` / `read_facts` → cite source + hash + date.
+
+## Docs
+
+- **[CLI reference](.claude/skills/sift/reference/cli.md)** — every command and flag
+- **[Configuration & site profiles](.claude/skills/sift/reference/config.md)** — `sift.toml` and the `SiteProfile` contract
+- **[MCP tools](.claude/skills/sift/reference/mcp-tools.md)** — parameters, output caps, multi-index mode
+- **[Corpus format & integrity contract](./corpus.contract.md)** — on-disk layout and what each read tool returns
+- **[Contributing](./CONTRIBUTING.md)** · **[Security](./SECURITY.md)**
+
+> **Open core, Apache-2.0.** This repo is the full open-source engine (pipeline + MCP server) and runs standalone. A hosted platform built on it is in development.
+
+**Status — v0.3.0**; tests green on Python 3.11–3.13. Adds proof-carrying answers (`prove` / `verify-proof` + an RFC-3161 timestamp anchor), time-travel reads (`changed_since`, `diff_md`, `as_of` over retained snapshots), API acquisition/content transports (`api_url` source + `json` extract), robots + SSRF + bot-challenge hardening, digital-PDF tables, a recursive link frontier, and `sift discover` (read-only pre-crawl preview). Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
+
+## License
+
+[Apache-2.0](./LICENSE) — © 2026 Deval Shah.

sift_engine-0.3.0/README.md

@@ -0,0 +1,183 @@
+<div align="center">
+
+<img src="assets/sift.gif" alt="sift" width="480">
+
+**Give your coding agent a complete, always-fresh copy of any website — with proof of every answer.**
+
+[![Tests](https://github.com/dvlshah/sift/actions/workflows/tests.yml/badge.svg)](https://github.com/dvlshah/sift/actions/workflows/tests.yml)
+[![PyPI](https://img.shields.io/pypi/v/sift-engine?color=blue)](https://pypi.org/project/sift-engine/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://www.python.org/downloads/)
+[![MCP](https://img.shields.io/badge/MCP-stdio-purple)](https://modelcontextprotocol.io/)
+
+</div>
+
+sift crawls a site into local markdown + structured facts your agent greps, reads, and cites — files on disk, not vectors, read over MCP. Every page is **content-hashed and dated**, so answers trace back to the exact source and snapshot. Re-run anytime; only changed pages are refetched. Self-hosted.
+
+## ❌ Without sift
+
+Your agent's built-in web-fetch gives you:
+
+- **Stale** answers — the provider's crawler cached that page weeks ago
+- **Partial** answers — just the one page it happened to land on
+- **Unprovable** answers — no record of what the source said, or when
+
+## ✅ With sift
+
+- **Always fresh** — you control the crawl; conditional GETs refetch only what changed, and `changed_since` lets your agent pull just the delta instead of re-reading the corpus
+- **Provable** — content-hash + date on every page; cite the source, hash, and snapshot, read any past snapshot (`as_of`), or emit a self-contained `prove` inclusion proof a third party verifies offline (`python -m sift.verify_proof`) without trusting the server — optionally anchored by an RFC-3161 timestamp so the date is witnessed by a third party, not self-asserted
+- **Complete & grep-native** — the whole site on disk, not a handful of retrieved snippets
+- **Self-hosted** — any `http(s)` site you can reach, public or internal; your data stays yours
+
+---
+
+## Get started
+
+Requires Python 3.11+.
+
+### 🤖 Let your coding agent set it up — recommended
+
+Paste one prompt into **Claude Code, Cursor, Codex, or any MCP-aware agent**. It explains sift, asks which site to index, then installs the engine, builds an index, wires up MCP, and shows you how to query it — end to end.
+
+<details>
+<summary><b>📋 Copy the one-paste setup prompt</b></summary>
+
+````text
+Set up **sift** in this project — a deterministic, content-hashed website indexer that
+serves a verifiable markdown corpus to AI agents over MCP (https://github.com/dvlshah/sift).
+Do the steps in order. Needs Python 3.11+.
+
+0 — EXPLAIN, THEN ASK ME (do this BEFORE installing anything)
+First, explain sift to me in 4–6 plain-English lines — assume I've never heard of it:
+  • WHAT it is: it crawls a whole website/docs site into local markdown files (plus structured
+    facts) that you, the agent, can grep / read / cite — files on disk, not a vector database.
+  • CORE FEATURES: (1) complete — the full site, not the few pages a live web-fetch happens to
+    land on; (2) verifiable — every page is content-hashed + dated, so any answer can be proved
+    back to the exact source and snapshot; (3) always-current — re-running the crawl refetches
+    only what changed; (4) read over MCP — you query it with provenance, read-only by default.
+  • WHY IT'S NEEDED: a one-off scrape gives you 3 pages with no proof of what they said or when.
+    sift keeps an agent correct about an evolving body of docs AND able to prove what it cited.
+    (If someone only needs one page once, sift is the wrong tool — a plain fetch is fine.)
+Then ASK ME: "Which website or docs site do you want to index locally?" — wait for my answer
+and use it as TARGET_SITE everywhere below. Do not run any commands until I answer.
+
+1 — INSTALL THE ENGINE
+```bash
+pip install sift-engine          # adds the `sift`, `sift-mcp`, `sift-evals` commands
+sift --version && which sift-mcp  # confirm it's on PATH
+```
+
+2 — INSTALL THE SIFT SKILL INTO THIS REPO
+Download the skill so you (the agent) know how to build, operate, and query an index.
+Read .claude/skills/sift/SKILL.md after downloading — it is the source of truth for the rest.
+```bash
+SKILL=https://raw.githubusercontent.com/dvlshah/sift/v0.3.0/.claude/skills/sift
+mkdir -p .claude/skills/sift/reference
+curl -fsSL $SKILL/SKILL.md -o .claude/skills/sift/SKILL.md
+for f in cli config mcp-tools; do curl -fsSL $SKILL/reference/$f.md -o .claude/skills/sift/reference/$f.md; done
+```
+
+3 — BUILD A SMALL STARTER INDEX (using the TARGET_SITE I gave you in step 0)
+Write a sift.toml (generic profile + host allow-list = the host of TARGET_SITE), then build a
+capped, publishable smoke-test index:
+```bash
+cat > sift.toml <<'TOML'
+[site]
+profile = "sift.sites.generic:GenericProfile"
+[seed]
+host_allow = ["HOST_OF_TARGET_SITE"]      # e.g. docs.example.com — derive from TARGET_SITE
+TOML
+sift init   --root ./sift-index
+sift seed   --root ./sift-index --config sift.toml --from-domain TARGET_SITE
+sift run    --root ./sift-index --config sift.toml --limit 25 --coverage-base planned
+sift verify --root ./sift-index --skip-signature
+```
+Drop `--limit 25 --coverage-base planned` for a full crawl once extraction looks good.
+Preview exactly which URLs a crawl would touch — and how many — *without fetching a
+byte*: `sift discover --from-domain TARGET_SITE` (writes nothing; emits JSON).
+No `sitemap.xml` (or an incomplete one)? After a run, `sift seed --from-frontier`
+extracts in-scope links from the fetched pages and crawls one hop deeper each pass.
+Hardened / bot-blocked host (Cloudflare/Akamai/Imperva) → add `--impersonate-fallback`
+(free, TLS-fingerprint impersonation; `pip install 'sift-engine[impersonate]'`).
+JS-rendered SPA → `pip install 'sift-engine[browser]' && python -m playwright install chromium`,
+then add `[browser]\nenabled = true` (it joins the ladder as a free render tier).
+JS-rendered SPA that has an official API (eCFR, CVE.org, FederalRegister …) → a
+profile's `api_url(url)` fetches the robots-allowed API instead of the shell —
+cleaner and cheaper than a browser, and the citation stays the human page (see
+the bundled `sift.sites.cve:CVEProfile`).
+Still blocked (JS-challenge edges) → `--firecrawl-fallback` (paid; needs FIRECRAWL_API_KEY).
+These compose into one escalation ladder: native → impersonate → browser → Firecrawl.
+sift respects `robots.txt` `Disallow` at seed by default; set `[crawl]` →
+`respect_robots = false` only for sources you have permission to index.
+
+4 — WIRE THE READ-ONLY MCP SERVER
+Use the ABSOLUTE path to ./sift-index. Add this to the project's .mcp.json (Claude Code / Cursor / Codex):
+```json
+{ "mcpServers": { "sift": { "command": "sift-mcp", "args": ["--root", "ABSOLUTE/PATH/TO/sift-index"] } } }
+```
+Claude Code shortcut: `claude mcp add sift -- sift-mcp --root "$(pwd)/sift-index"`
+Then restart the MCP client so the `sift` tools load.
+
+5 — SHOW ME THE QUERY LOOP
+Call the `snapshot_status` tool to confirm the index is published, then explain the loop:
+snapshot_status first → grep_corpus to locate → read_md / read_facts to drill in →
+cite source_url + content_hash + fetched_at. Mention that re-running `sift run` refreshes the
+index, and that the /sift skill covers building, operating, and querying in depth.
+
+(TARGET_SITE = the site I name when you ask in step 0, e.g. https://docs.example.com)
+````
+
+</details>
+
+### 🧑 Or do it yourself
+
+```bash
+pip install sift-engine
+
+# the ATO sitemap below uses a bundled profile, so this runs with zero config
+sift init   --root ./index
+sift seed   --root ./index --from-sitemap https://www.ato.gov.au/sitemap.xml
+sift run    --root ./index --limit 25 --coverage-base planned   # smoke-test first
+sift verify --root ./index --skip-signature
+sift-mcp    --root ./index   # serve the index to your agent
+```
+
+Then point your agent at it (use an absolute path):
+
+```json
+{
+  "mcpServers": {
+    "sift": { "command": "sift-mcp", "args": ["--root", "/abs/path/to/index"] }
+  }
+}
+```
+
+Indexing your own site? Add a `sift.toml` (generic profile + host allow-list) — see [Configuration](.claude/skills/sift/reference/config.md).
+
+---
+
+## How it works
+
+```
+seed → plan → fetch → extract → commit → publish
+```
+
+Five idempotent phases. `publish` runs 5 verification gates, then atomically swaps the `current/` snapshot and writes a Merkle root over every page hash. Deterministic: **same input → same `content_hash` → same Merkle root** — so any reader re-verifies a page in O(1), or the whole snapshot end-to-end with `sift verify`.
+
+Your agent reads the published snapshot **read-only** over MCP: `snapshot_status` → `grep_corpus` → `read_md` / `read_facts` → cite source + hash + date.
+
+## Docs
+
+- **[CLI reference](.claude/skills/sift/reference/cli.md)** — every command and flag
+- **[Configuration & site profiles](.claude/skills/sift/reference/config.md)** — `sift.toml` and the `SiteProfile` contract
+- **[MCP tools](.claude/skills/sift/reference/mcp-tools.md)** — parameters, output caps, multi-index mode
+- **[Corpus format & integrity contract](./corpus.contract.md)** — on-disk layout and what each read tool returns
+- **[Contributing](./CONTRIBUTING.md)** · **[Security](./SECURITY.md)**
+
+> **Open core, Apache-2.0.** This repo is the full open-source engine (pipeline + MCP server) and runs standalone. A hosted platform built on it is in development.
+
+**Status — v0.3.0**; tests green on Python 3.11–3.13. Adds proof-carrying answers (`prove` / `verify-proof` + an RFC-3161 timestamp anchor), time-travel reads (`changed_since`, `diff_md`, `as_of` over retained snapshots), API acquisition/content transports (`api_url` source + `json` extract), robots + SSRF + bot-challenge hardening, digital-PDF tables, a recursive link frontier, and `sift discover` (read-only pre-crawl preview). Known limits: no run-dir GC yet, stdout-only logging, stdio-only MCP transport. Issues & roadmap → [GitHub Issues](https://github.com/dvlshah/sift/issues).
+
+## License
+
+[Apache-2.0](./LICENSE) — © 2026 Deval Shah.

{sift_engine-0.1.0 → sift_engine-0.3.0}/evals/bench/per_stage/plan.py

@@ -57,9 +57,8 @@ def eval_plan_decision_correctness() -> PlanDecisionCorrectnessResult:
     from sift.config import IndexConfig, TierConfig
     from sift.classify import CLASSIFIER_VERSION
     from sift.extract import EXTRACTOR_VERSION
-    from sift.normalize import NORMALIZER_VERSION
+    from sift.normalize import normalizer_version
     from sift import decide as decide_mod
-    from sift import sites as sites_mod
     from sift import paths
 
     # Set up an in-memory-ish manifest in a temp dir
@@ -151,7 +150,7 @@ def eval_plan_decision_correctness() -> PlanDecisionCorrectnessResult:
         plan_phase(
             conn, plan_path, now=now,
             extractor_version=EXTRACTOR_VERSION,
-            normalizer_version=NORMALIZER_VERSION,
+            normalizer_version=normalizer_version(),
             profile=GenericProfile(),
             cfg=cfg,
         )

{sift_engine-0.1.0 → sift_engine-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sift-engine"
-version = "0.1.0"
+version = "0.3.0"
 description = "Sift — deterministic website indexing for grep-first LLM agents"
 readme = "README.md"
 license = "Apache-2.0"
@@ -24,7 +24,9 @@ dependencies = [
     "selectolax>=0.3",
     "mcp>=1.0",
     "pypdf>=4.0",
+    "pdfplumber>=0.11",
     "jsonschema>=4",
+    "protego>=0.3",
 ]
 
 [project.optional-dependencies]
@@ -39,12 +41,18 @@ evals = [
     "jsonschema>=4",
 ]
 # Optional browser-rendering stack for SPA fetch (see docs/design/browser-fetch.md).
-# Install with: pip install 'sift[browser]' && python -m playwright install chromium
-# History: v0.2.0 originally bundled crawl4ai; swapped to bare Playwright after
+# Install with: pip install 'sift-engine[browser]' && python -m playwright install chromium
+# History: an early prototype bundled crawl4ai; swapped to bare Playwright after
 # reproducible hangs on analytics-heavy SPAs. See archive/crawl4ai/README.md.
 browser = [
     "playwright>=1.40",
 ]
+# Tier-2 escalation transport: TLS/HTTP2 fingerprint impersonation. Defeats most
+# Cloudflare/Akamai/Imperva fingerprint blocks with no browser and no per-request
+# cost. Install with: pip install 'sift-engine[impersonate]'
+impersonate = [
+    "curl_cffi>=0.7",
+]
 
 [project.urls]
 Homepage = "https://github.com/dvlshah/sift"

{sift_engine-0.1.0 → sift_engine-0.3.0}/sift/__init__.py

@@ -16,7 +16,7 @@ bumping a version triggers re-derivation from cached raw, not refetch.
 See README.md for the full design.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"
 # Provenance stamp on manifest rows — a *behavioral* version (like
 # EXTRACTOR_VERSION), bumped only when crawl/fetch behavior changes, not on
 # every package release. this is the initial public release; crawl behavior is unchanged, so this stays.

{sift_engine-0.1.0 → sift_engine-0.3.0}/sift/browser.py

@@ -71,7 +71,7 @@ class BrowserFetchError(Exception):
 def _install_hint() -> str:
     return (
         "playwright is not installed. Install with:\n"
-        "    pip install 'sift[browser]' && python -m playwright install chromium"
+        "    pip install 'sift-engine[browser]' && python -m playwright install chromium"
     )