docpull 6.5.0__tar.gz → 6.5.2__tar.gz

docpull-6.5.2/PKG-INFO

@@ -0,0 +1,279 @@
+Metadata-Version: 2.4
+Name: docpull
+Version: 6.5.2
+Summary: Declare, sync, diff, and lock context dependencies for AI agents
+Author-email: Zachary Roth <support@raintree.technology>
+Maintainer-email: Raintree Technology <support@raintree.technology>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/raintree-technology/docpull
+Project-URL: Documentation, https://github.com/raintree-technology/docpull#readme
+Project-URL: Repository, https://github.com/raintree-technology/docpull
+Project-URL: Source Code, https://github.com/raintree-technology/docpull
+Project-URL: Bug Tracker, https://github.com/raintree-technology/docpull/issues
+Project-URL: Releases, https://github.com/raintree-technology/docpull/releases
+Project-URL: MCP Plugin, https://github.com/raintree-technology/docpull/tree/main/plugin
+Project-URL: Comparison Guide, https://github.com/raintree-technology/docpull/blob/main/docs/alternatives.md
+Project-URL: Download Stats, https://pepy.tech/project/docpull
+Project-URL: Changelog, https://github.com/raintree-technology/docpull/blob/main/docs/CHANGELOG.md
+Keywords: python,markdown,documentation,web,website,web-extraction,web-to-markdown,source-ingestion,provenance,context-ci,content-extraction,developer-tools,cli,rag,llm,mcp,ai-agents,context-engineering,context-dependencies,context-packs,source-packs,research,claude,ai-training-data
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: Environment :: Console
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Utilities
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: html2text>=2020.1.16
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: extruct>=0.15.0
+Requires-Dist: aiohttp>=3.14.3
+Requires-Dist: idna>=3.15
+Requires-Dist: regex>=2024.11.6
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: urllib3>=2.7.0
+Provides-Extra: proxy
+Requires-Dist: aiohttp-socks>=0.11.0; extra == "proxy"
+Provides-Extra: normalize
+Requires-Dist: url-normalize>=1.4.0; extra == "normalize"
+Provides-Extra: trafilatura
+Requires-Dist: trafilatura>=1.12.0; extra == "trafilatura"
+Provides-Extra: tokens
+Requires-Dist: tiktoken>=0.7.0; extra == "tokens"
+Provides-Extra: mcp
+Requires-Dist: cryptography>=50.0.0; extra == "mcp"
+Requires-Dist: mcp<2,>=1.28.1; extra == "mcp"
+Requires-Dist: pydantic-settings>=2.14.2; extra == "mcp"
+Requires-Dist: pyjwt>=2.13.0; extra == "mcp"
+Requires-Dist: python-multipart>=0.0.27; extra == "mcp"
+Requires-Dist: starlette>=1.3.1; extra == "mcp"
+Provides-Extra: serve
+Requires-Dist: uvicorn>=0.30.0; extra == "serve"
+Provides-Extra: parquet
+Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
+Provides-Extra: markitdown
+Requires-Dist: pypdf>=6.15.0; extra == "markitdown"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
+Provides-Extra: unstructured
+Requires-Dist: pypdf>=6.15.0; extra == "unstructured"
+Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "unstructured"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=6.15.0; extra == "pdf"
+Provides-Extra: parse
+Requires-Dist: pypdf>=6.15.0; extra == "parse"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
+Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
+Provides-Extra: presidio
+Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
+Provides-Extra: provenance
+Requires-Dist: cryptography>=50.0.0; extra == "provenance"
+Provides-Extra: e2b
+Requires-Dist: e2b>=2.0.0; extra == "e2b"
+Provides-Extra: llm
+Requires-Dist: tiktoken>=0.7.0; extra == "llm"
+Provides-Extra: all
+Requires-Dist: aiohttp-socks>=0.11.0; extra == "all"
+Requires-Dist: url-normalize>=1.4.0; extra == "all"
+Requires-Dist: trafilatura>=1.12.0; extra == "all"
+Requires-Dist: tiktoken>=0.7.0; extra == "all"
+Requires-Dist: cryptography>=50.0.0; extra == "all"
+Requires-Dist: mcp<2,>=1.28.1; extra == "all"
+Requires-Dist: pydantic-settings>=2.14.2; extra == "all"
+Requires-Dist: pyjwt>=2.13.0; extra == "all"
+Requires-Dist: python-multipart>=0.0.27; extra == "all"
+Requires-Dist: starlette>=1.3.1; extra == "all"
+Requires-Dist: uvicorn>=0.30.0; extra == "all"
+Requires-Dist: pyarrow>=16.0.0; extra == "all"
+Requires-Dist: pypdf>=6.15.0; extra == "all"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
+Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "all"
+Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
+Requires-Dist: e2b>=2.0.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: bandit>=1.7.0; extra == "dev"
+Requires-Dist: pip-audit>=2.0.0; extra == "dev"
+Requires-Dist: msgpack>=1.2.1; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: types-requests>=2.31.0; extra == "dev"
+Requires-Dist: types-beautifulsoup4>=4.12.0; extra == "dev"
+Requires-Dist: types-defusedxml>=0.7.0; extra == "dev"
+Requires-Dist: types-pyyaml>=6.0.0; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/raintree-technology/docpull/main/docs/launch-assets/logo-square-light-400.png" alt="DocPull" width="128" />
+</p>
+
+# DocPull
+
+<!-- project-record: docpull -->
+
+**Active open-source project · MIT License**
+
+DocPull turns changing public web sources into cited, reproducible context for AI
+agents and retrieval pipelines. Use it when your application needs to know which
+sources it used, whether they changed, and how to rebuild the same context later.
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
+[![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
+
+<!-- mcp-name: io.github.raintree-technology/docpull -->
+
+## Install and sync your first source
+
+```bash
+pip install docpull
+docpull init stripe-docs
+docpull add https://docs.stripe.com
+docpull sync
+docpull diff
+docpull export context-pack --target cursor
+```
+
+The project stores declared sources in `docpull.yaml` and resolved inputs in
+`.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
+source URLs, content hashes, run IDs, citations, and export metadata.
+
+![DocPull project diff showing changed pages, local semantic categories, and zero failed URLs](docs/launch-assets/docpull-project-diff-demo.png)
+
+No account or paid API is required for this path. Direct fetching, discovery,
+extraction, indexing, pack analysis, and diffs run locally.
+
+## Why use DocPull
+
+- **Reproduce agent context.** Stable IDs, hashes, manifests, and lockfiles show which
+  source versions produced an answer or artifact.
+- **Detect source drift.** Sync and diff documentation, product pages, policies,
+  feeds, repositories, packages, standards, and local documents.
+- **Keep evidence inspectable.** Markdown, NDJSON, SQLite, citations, and provenance
+  sidecars remain readable without a hosted service.
+- **Choose the downstream surface.** Export context for agent clients, vector import,
+  data workflows, or a versioned context-pack release.
+- **Keep expensive routes explicit.** Browser and cloud rendering require an explicit
+  choice and can be blocked with a zero-dollar budget.
+
+## How it works
+
+```text
+declared sources → local acquisition → versioned evidence → diff and validation → export
+```
+
+DocPull's v3 pack contract separates raw extraction, agent-ready context, and
+eval-grade evidence. Validate the level a downstream system requires:
+
+```bash
+docpull pack prepare packs/docs --eval-grade
+docpull pack validate packs/docs --level eval
+docpull ci --prepare
+```
+
+`docpull ci` checks freshness, citation coverage, pack quality, rights metadata, and
+other configured gates. It writes `context-ci.report.json` and `CONTEXT_CI.md`, then
+exits non-zero when a hard gate fails.
+
+## Supported surfaces
+
+| Surface | Use it for | Start here |
+| --- | --- | --- |
+| CLI | Fetch, sync, diff, validate, and export | [`docs/cli-recipes.md`](docs/cli-recipes.md) |
+| Python SDK | Embed acquisition in Python applications | [`docs/surface-contract.md`](docs/surface-contract.md) |
+| MCP server | Give local agent clients source tools | [MCP server](#mcp-server) |
+| TypeScript SDK | Read local packs and invoke the CLI from Node or Bun | [`sdk/js/README.md`](sdk/js/README.md) |
+| Agent plugin | Install the supported MCP workflow in an agent client | [`plugin/README.md`](plugin/README.md) |
+
+Common source shapes include static and server-rendered websites, OpenAPI documents,
+feeds, papers, public GitHub repositories, npm and PyPI packages, standards, datasets,
+transcripts, Wikimedia pages, product and policy pages, and local PDF or office files.
+See [context-pack workflows](docs/context-packs.md) for the complete surface.
+
+## MCP server
+
+```bash
+pip install 'docpull[mcp]'
+docpull mcp
+```
+
+Claude Code can register the same local server:
+
+```bash
+claude mcp add --transport stdio docpull -- docpull mcp
+```
+
+The Python stdio server is the supported release path. The TypeScript code formerly
+documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
+not part of the package contract.
+
+## Limits and security boundary
+
+DocPull is an evidence-acquisition engine, not a hosted competitive-intelligence
+product. It owns fetching, explicit rendering adapters, versioning, citations,
+hashing, validation, replay, and export. Downstream products own scheduling, human
+review, approved claims, legal conclusions, accounts, and notifications.
+
+The default path does not handle complex interactive browser workflows, CAPTCHAs,
+stealth scraping, or private dashboards. JavaScript rendering is explicit. Authenticated
+sources require environment-variable references; DocPull does not persist credential
+values in project artifacts.
+
+Security defaults include HTTPS-only fetching, robots.txt compliance, SSRF and DNS
+rebinding protections, redirect guards, XXE protection, and path-traversal checks. Read
+the [web-source boundary](docs/scraping-boundary.md), [security posture](docs/compliance.md),
+and [evidence-engine decision](docs/adr/0001-evidence-acquisition-engine.md) before
+extending acquisition behavior.
+
+## Documentation and evidence
+
+- [CLI recipes](docs/cli-recipes.md) — Common commands and advanced workflows.
+- [Context dependencies](docs/context-dependencies.md) — Project and lockfile model.
+- [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
+- [Public contracts](docs/contracts.md) — Schemas and versioning rules.
+- [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
+- [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
+- [Changelog](docs/CHANGELOG.md) — Release history.
+
+## Raintree open-source system
+
+DocPull owns evidence acquisition and reproducible agent context. It can be used
+independently; the sibling projects do not imply a required integration or shared
+release cycle.
+
+| Project | Responsibility |
+| --- | --- |
+| [Raintree Standards](https://github.com/raintree-technology/raintree.standards) | Defines governed requirements and evidence. |
+| [Trellis](https://github.com/raintree-technology/trellis) | Enforces shared JavaScript and TypeScript code policy. |
+| [HIG Doctor](https://github.com/raintree-technology/hig-doctor) | Audits interface source and provides HIG guidance. |
+| [PolicyStrata](https://github.com/raintree-technology/policystrata) | Tests cross-layer policy behavior. |
+
+See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
+for current lifecycle and distribution links.
+
+## Project policies
+
+[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
+[Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
+[MIT License](LICENSE)

docpull-6.5.2/README.md

@@ -0,0 +1,153 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/raintree-technology/docpull/main/docs/launch-assets/logo-square-light-400.png" alt="DocPull" width="128" />
+</p>
+
+# DocPull
+
+<!-- project-record: docpull -->
+
+**Active open-source project · MIT License**
+
+DocPull turns changing public web sources into cited, reproducible context for AI
+agents and retrieval pipelines. Use it when your application needs to know which
+sources it used, whether they changed, and how to rebuild the same context later.
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
+[![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
+
+<!-- mcp-name: io.github.raintree-technology/docpull -->
+
+## Install and sync your first source
+
+```bash
+pip install docpull
+docpull init stripe-docs
+docpull add https://docs.stripe.com
+docpull sync
+docpull diff
+docpull export context-pack --target cursor
+```
+
+The project stores declared sources in `docpull.yaml` and resolved inputs in
+`.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
+source URLs, content hashes, run IDs, citations, and export metadata.
+
+![DocPull project diff showing changed pages, local semantic categories, and zero failed URLs](docs/launch-assets/docpull-project-diff-demo.png)
+
+No account or paid API is required for this path. Direct fetching, discovery,
+extraction, indexing, pack analysis, and diffs run locally.
+
+## Why use DocPull
+
+- **Reproduce agent context.** Stable IDs, hashes, manifests, and lockfiles show which
+  source versions produced an answer or artifact.
+- **Detect source drift.** Sync and diff documentation, product pages, policies,
+  feeds, repositories, packages, standards, and local documents.
+- **Keep evidence inspectable.** Markdown, NDJSON, SQLite, citations, and provenance
+  sidecars remain readable without a hosted service.
+- **Choose the downstream surface.** Export context for agent clients, vector import,
+  data workflows, or a versioned context-pack release.
+- **Keep expensive routes explicit.** Browser and cloud rendering require an explicit
+  choice and can be blocked with a zero-dollar budget.
+
+## How it works
+
+```text
+declared sources → local acquisition → versioned evidence → diff and validation → export
+```
+
+DocPull's v3 pack contract separates raw extraction, agent-ready context, and
+eval-grade evidence. Validate the level a downstream system requires:
+
+```bash
+docpull pack prepare packs/docs --eval-grade
+docpull pack validate packs/docs --level eval
+docpull ci --prepare
+```
+
+`docpull ci` checks freshness, citation coverage, pack quality, rights metadata, and
+other configured gates. It writes `context-ci.report.json` and `CONTEXT_CI.md`, then
+exits non-zero when a hard gate fails.
+
+## Supported surfaces
+
+| Surface | Use it for | Start here |
+| --- | --- | --- |
+| CLI | Fetch, sync, diff, validate, and export | [`docs/cli-recipes.md`](docs/cli-recipes.md) |
+| Python SDK | Embed acquisition in Python applications | [`docs/surface-contract.md`](docs/surface-contract.md) |
+| MCP server | Give local agent clients source tools | [MCP server](#mcp-server) |
+| TypeScript SDK | Read local packs and invoke the CLI from Node or Bun | [`sdk/js/README.md`](sdk/js/README.md) |
+| Agent plugin | Install the supported MCP workflow in an agent client | [`plugin/README.md`](plugin/README.md) |
+
+Common source shapes include static and server-rendered websites, OpenAPI documents,
+feeds, papers, public GitHub repositories, npm and PyPI packages, standards, datasets,
+transcripts, Wikimedia pages, product and policy pages, and local PDF or office files.
+See [context-pack workflows](docs/context-packs.md) for the complete surface.
+
+## MCP server
+
+```bash
+pip install 'docpull[mcp]'
+docpull mcp
+```
+
+Claude Code can register the same local server:
+
+```bash
+claude mcp add --transport stdio docpull -- docpull mcp
+```
+
+The Python stdio server is the supported release path. The TypeScript code formerly
+documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
+not part of the package contract.
+
+## Limits and security boundary
+
+DocPull is an evidence-acquisition engine, not a hosted competitive-intelligence
+product. It owns fetching, explicit rendering adapters, versioning, citations,
+hashing, validation, replay, and export. Downstream products own scheduling, human
+review, approved claims, legal conclusions, accounts, and notifications.
+
+The default path does not handle complex interactive browser workflows, CAPTCHAs,
+stealth scraping, or private dashboards. JavaScript rendering is explicit. Authenticated
+sources require environment-variable references; DocPull does not persist credential
+values in project artifacts.
+
+Security defaults include HTTPS-only fetching, robots.txt compliance, SSRF and DNS
+rebinding protections, redirect guards, XXE protection, and path-traversal checks. Read
+the [web-source boundary](docs/scraping-boundary.md), [security posture](docs/compliance.md),
+and [evidence-engine decision](docs/adr/0001-evidence-acquisition-engine.md) before
+extending acquisition behavior.
+
+## Documentation and evidence
+
+- [CLI recipes](docs/cli-recipes.md) — Common commands and advanced workflows.
+- [Context dependencies](docs/context-dependencies.md) — Project and lockfile model.
+- [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
+- [Public contracts](docs/contracts.md) — Schemas and versioning rules.
+- [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
+- [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
+- [Changelog](docs/CHANGELOG.md) — Release history.
+
+## Raintree open-source system
+
+DocPull owns evidence acquisition and reproducible agent context. It can be used
+independently; the sibling projects do not imply a required integration or shared
+release cycle.
+
+| Project | Responsibility |
+| --- | --- |
+| [Raintree Standards](https://github.com/raintree-technology/raintree.standards) | Defines governed requirements and evidence. |
+| [Trellis](https://github.com/raintree-technology/trellis) | Enforces shared JavaScript and TypeScript code policy. |
+| [HIG Doctor](https://github.com/raintree-technology/hig-doctor) | Audits interface source and provides HIG guidance. |
+| [PolicyStrata](https://github.com/raintree-technology/policystrata) | Tests cross-layer policy behavior. |
+
+See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
+for current lifecycle and distribution links.
+
+## Project policies
+
+[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
+[Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
+[MIT License](LICENSE)

{docpull-6.5.0 → docpull-6.5.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.5.0"
+version = "6.5.2"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -86,7 +86,7 @@ dependencies = [
     "html2text>=2020.1.16",
     "defusedxml>=0.7.1",
     "extruct>=0.15.0",
-    "aiohttp>=3.14.0",  # 3.14.0 fixes CVE-2026-34993 and CVE-2026-47265
+    "aiohttp>=3.14.3",  # Security floor for the 2026 request-parsing advisories
     "idna>=3.15",  # Security floor for transitive URL/IDNA handling; Renovate raises as needed
     "regex>=2024.11.6",
     "rich>=13.0.0",
@@ -109,8 +109,8 @@ tokens = [
     "tiktoken>=0.7.0",
 ]
 mcp = [
-    "cryptography>=48.0.1",
-    "mcp>=1.28.1",
+    "cryptography>=50.0.0",
+    "mcp>=1.28.1,<2",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.13.0",
     "python-multipart>=0.0.27",
@@ -123,18 +123,18 @@ parquet = [
     "pyarrow>=16.0.0",
 ]
 markitdown = [
-    "pypdf>=6.14.2",
+    "pypdf>=6.15.0",
     "markitdown[docx,pdf]>=0.1.0",
 ]
 unstructured = [
-    "pypdf>=6.14.2",
+    "pypdf>=6.15.0",
     "unstructured[all-docs]>=0.16.0",
 ]
 pdf = [
-    "pypdf>=6.14.2",
+    "pypdf>=6.15.0",
 ]
 parse = [
-    "pypdf>=6.14.2",
+    "pypdf>=6.15.0",
     "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
 ]
@@ -142,7 +142,7 @@ presidio = [
     "presidio-analyzer>=2.2.0",
 ]
 provenance = [
-    "cryptography>=48.0.1",
+    "cryptography>=50.0.0",
 ]
 e2b = [
     "e2b>=2.0.0",
@@ -155,15 +155,15 @@ all = [
     "url-normalize>=1.4.0",
     "trafilatura>=1.12.0",
     "tiktoken>=0.7.0",
-    "cryptography>=48.0.1",
-    "mcp>=1.28.1",
+    "cryptography>=50.0.0",
+    "mcp>=1.28.1,<2",
     "pydantic-settings>=2.14.2",
     "pyjwt>=2.13.0",
     "python-multipart>=0.0.27",
     "starlette>=1.3.1",
     "uvicorn>=0.30.0",
     "pyarrow>=16.0.0",
-    "pypdf>=6.14.2",
+    "pypdf>=6.15.0",
     "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
     "presidio-analyzer>=2.2.0",
@@ -207,6 +207,9 @@ include = ["docpull*"]
 [tool.setuptools.package-data]
 docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"]
 
+[tool.uv]
+constraint-dependencies = ["h2>=4.4.1"]
+
 [tool.ruff]
 line-length = 110
 target-version = "py310"

{docpull-6.5.0 → docpull-6.5.2}/src/docpull/__init__.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any
 
 from .surface import PUBLIC_SDK_EXPORTS
 
-__version__ = "6.5.0"
+__version__ = "6.5.2"
 
 _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
     **{

{docpull-6.5.0 → docpull-6.5.2}/src/docpull/pipeline/steps/convert.py

@@ -58,6 +58,28 @@ _FRONTMATTER_METADATA_KEYS: tuple[str, ...] = (
 # outline. Code fences are also excluded — see _extract_headings below.
 _HEADING_LINE_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
 
+_ACCESS_WALL_MARKERS: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("bot_challenge", ("client challenge", "verify you are human", "checking your browser", "captcha")),
+    ("auth_wall", ("sign in to continue", "log in to continue", "authentication required")),
+    ("paywall", ("subscribe to continue", "subscription required", "purchase a subscription")),
+)
+
+
+def detect_access_wall(value: bytes | str) -> str | None:
+    """Return a stable category for a short challenge, login, or paywall response."""
+    text = value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value
+    normalized = re.sub(r"\s+", " ", text).casefold()
+    if len(normalized) > 50_000:
+        return None
+    for category, markers in _ACCESS_WALL_MARKERS:
+        if any(marker in normalized for marker in markers):
+            return category
+    if "enable javascript" in normalized and any(
+        marker in normalized for marker in ("proceed", "continue", "challenge", "access")
+    ):
+        return "bot_challenge"
+    return None
+
 
 def _extract_headings(markdown: str, max_level: int = 2, limit: int = 12) -> list[str]:
     """Pull a flat list of top-level headings from converted Markdown.
@@ -236,6 +258,13 @@ class ConvertStep:
             return ctx
 
         try:
+            access_wall = detect_access_wall(ctx.html)
+            if access_wall is not None:
+                ctx.mark_failed(f"access_wall:{access_wall}")
+                if emit:
+                    emit(FetchEvent(type=EventType.FETCH_FAILED, url=ctx.url, error=ctx.error))
+                return ctx
+
             optout = self._check_meta_robots(ctx)
             if optout is not None:
                 reason = f"AI/TDM opt-out (meta robots: {', '.join(optout.matched)})"
@@ -301,6 +330,13 @@ class ConvertStep:
             if looks_like_spa_output(markdown):
                 return self._handle_empty_content(ctx, emit)
 
+            access_wall = detect_access_wall(markdown)
+            if access_wall is not None:
+                ctx.mark_failed(f"access_wall:{access_wall}")
+                if emit:
+                    emit(FetchEvent(type=EventType.FETCH_FAILED, url=ctx.url, error=ctx.error))
+                return ctx
+
             markdown = clean_article_markdown(markdown, url=ctx.url, metadata=ctx.metadata)
             headings, heading_count = _heading_summary(markdown)