docpull 5.0.1__tar.gz → 5.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docpull-5.0.1/src/docpull.egg-info → docpull-5.5.0}/PKG-INFO +132 -20
- {docpull-5.0.1 → docpull-5.5.0}/README.md +130 -18
- {docpull-5.0.1 → docpull-5.5.0}/pyproject.toml +2 -2
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/__init__.py +22 -1
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/cli.py +105 -1
- docpull-5.5.0/src/docpull/context_packs/__init__.py +20 -0
- docpull-5.5.0/src/docpull/context_packs/brand.py +556 -0
- docpull-5.5.0/src/docpull/context_packs/cli.py +266 -0
- docpull-5.5.0/src/docpull/context_packs/common.py +953 -0
- docpull-5.5.0/src/docpull/context_packs/product.py +495 -0
- docpull-5.5.0/src/docpull/context_packs/schema_extract.py +304 -0
- docpull-5.5.0/src/docpull/context_packs/search.py +402 -0
- docpull-5.5.0/src/docpull/context_packs/styleguide.py +382 -0
- docpull-5.5.0/src/docpull/context_packs/visuals.py +455 -0
- docpull-5.5.0/src/docpull/hosted.py +720 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/http/client.py +6 -2
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/mcp/server.py +368 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/mcp/tools.py +38 -10
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/config.py +2 -2
- docpull-5.5.0/src/docpull/project.py +3398 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/rendering.py +34 -4
- docpull-5.5.0/src/docpull/share.py +547 -0
- {docpull-5.0.1 → docpull-5.5.0/src/docpull.egg-info}/PKG-INFO +132 -20
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull.egg-info/SOURCES.txt +16 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_ci_policy.py +23 -2
- docpull-5.5.0/tests/test_context_packs.py +310 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_conversion.py +2 -2
- docpull-5.5.0/tests/test_hosted.py +268 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_mcp_tools.py +27 -0
- docpull-5.5.0/tests/test_project.py +563 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_rendering.py +120 -10
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_security_hardening.py +54 -2
- docpull-5.5.0/tests/test_share.py +101 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_surface_contract.py +29 -2
- {docpull-5.0.1 → docpull-5.5.0}/LICENSE +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/setup.cfg +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/__main__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/accounting.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/auth_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/benchmark.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/cache/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/cache/frontier.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/cache/manager.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/cache/streaming_dedup.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/chunking.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/extractor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/filings.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/markdown.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/protocols.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/special_cases.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/conversion/trafilatura_extractor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/core/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/core/fetcher.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/_fetch.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/composite.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/contracts.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/crawler.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/filters.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/link_extractors/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/link_extractors/enhanced.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/link_extractors/protocols.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/link_extractors/static.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/protocols.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery/sitemap.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/discovery_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/doctor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/evidence_pack.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/exports.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/fixtures/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/fixtures/parallel-search-extract.json +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/graph.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/http/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/http/protocols.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/http/rate_limiter.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/judge.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/local_workflows.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/mcp/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/mcp/sources.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/metadata_extractor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/document.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/events.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/profiles.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/models/run.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/monitor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pack_reader.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pack_tools.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/parallel_workflows.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/parity.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/parity_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/passk.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/base.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/manifest.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/chunk.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/convert.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/dedup.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/fetch.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/metadata.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/render.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/save.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/save_json.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/save_ndjson.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/save_okf.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/save_sqlite.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/pipeline/steps/validate.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/policy.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/policy_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/provider_adapters.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/provider_capabilities.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/provider_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/provider_keys.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/provider_probes.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/py.typed +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/scraper.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/security/__init__.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/security/download_policy.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/security/robots.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/security/url_validator.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/server.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/skill_export.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/source_scoring.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull/time_utils.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull.egg-info/dependency_links.txt +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull.egg-info/entry_points.txt +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull.egg-info/requires.txt +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/src/docpull.egg-info/top_level.txt +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_accounting.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_auth_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_benchmark.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_cache_conditional_get.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_chunking.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_convert_step_new.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_discovery.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_discovery_contracts.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_doctor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_document_record.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_evidence_pack.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_exports.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_frontier_resume.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_graph.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_integration.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_judge.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_link_extractors.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_local_workflows.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_mcp_server.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_metadata_extractor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_monitor.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_naming.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_outputs_e2e.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_pack_server.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_pack_tools.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_parallel_workflows.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_parity_workflows.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_passk.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_pipeline.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_policy.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_policy_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_provider_adapters.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_provider_cli.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_provider_keys.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_provider_probes.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_rate_limiter.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_real_site_regressions.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_save_ndjson.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_save_sqlite.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_special_cases.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_time_utils.py +0 -0
- {docpull-5.0.1 → docpull-5.5.0}/tests/test_trafilatura_extractor.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docpull
|
|
3
|
-
Version: 5.0
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 5.5.0
|
|
4
|
+
Summary: Keep AI agents synced with changing public docs and web sources
|
|
5
5
|
Author-email: Zachary Roth <support@raintree.technology>
|
|
6
6
|
Maintainer-email: Raintree Technology <support@raintree.technology>
|
|
7
7
|
License-Expression: MIT
|
|
@@ -112,7 +112,7 @@ Dynamic: license-file
|
|
|
112
112
|
|
|
113
113
|
# docpull
|
|
114
114
|
|
|
115
|
-
**
|
|
115
|
+
**Keep AI agents synced with changing public docs. Browser-free by default.**
|
|
116
116
|
|
|
117
117
|
[](https://www.python.org/downloads/)
|
|
118
118
|
[](https://badge.fury.io/py/docpull)
|
|
@@ -120,10 +120,15 @@ Dynamic: license-file
|
|
|
120
120
|
[](https://github.com/raintree-technology/docpull/stargazers)
|
|
121
121
|
[](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
123
|
+
DocPull turns public docs and web sources into refreshable, cited, agent-ready
|
|
124
|
+
context packs. Define source sets once, sync them over time, diff what changed,
|
|
125
|
+
and export fresh context for Cursor, Claude, Codex, OpenAI, LlamaIndex,
|
|
126
|
+
LangChain, MCP clients, and RAG pipelines.
|
|
127
|
+
|
|
128
|
+
The original `docpull URL ...` workflow still works: fetch public or explicitly
|
|
129
|
+
authorized static/server-rendered web pages and write clean Markdown, NDJSON,
|
|
130
|
+
SQLite, or OKF outputs. Project mode adds the persistent evidence lifecycle on
|
|
131
|
+
top: sources, runs, diffs, exports, evals, accounting, and local auditability.
|
|
127
132
|
|
|
128
133
|
DocPull is local-first: direct fetching, sitemap/link discovery, extraction,
|
|
129
134
|
indexing, pack intelligence, and local `agent-browser` rendering can run with no
|
|
@@ -152,6 +157,34 @@ reason unless you explicitly opt into the local `agent-browser` renderer. See
|
|
|
152
157
|
pip install docpull
|
|
153
158
|
```
|
|
154
159
|
|
|
160
|
+
## Project Quickstart
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
docpull init stripe-docs
|
|
164
|
+
docpull add https://docs.stripe.com
|
|
165
|
+
docpull sync
|
|
166
|
+
docpull diff
|
|
167
|
+
docpull export context-pack --target cursor
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Example diff after a later sync:
|
|
171
|
+
|
|
172
|
+
```text
|
|
173
|
+
Project diff: +4 -2 ~18 api=2 pricing=1
|
|
174
|
+
|
|
175
|
+
Changed pages:
|
|
176
|
+
- /payments/payment-intents
|
|
177
|
+
likely API behavior change
|
|
178
|
+
- /billing/subscriptions
|
|
179
|
+
pricing / billing change
|
|
180
|
+
- /webhooks
|
|
181
|
+
likely API behavior change
|
|
182
|
+
|
|
183
|
+
0 failed URLs
|
|
184
|
+
0 robots blocked
|
|
185
|
+
0 paid/cloud routes used
|
|
186
|
+
```
|
|
187
|
+
|
|
155
188
|
Install optional extras as needed:
|
|
156
189
|
|
|
157
190
|
```bash
|
|
@@ -170,7 +203,11 @@ install. Install an `agent-browser` compatible CLI separately, put it on
|
|
|
170
203
|
`PATH`, or set `DOCPULL_AGENT_BROWSER_BIN=/path/to/agent-browser`. Verify the
|
|
171
204
|
runtime with `docpull render --check`. Render targets must use HTTPS except for
|
|
172
205
|
localhost/loopback HTTP during local testing, and DocPull keeps renderer action
|
|
173
|
-
permissions locked down to HTML retrieval only.
|
|
206
|
+
permissions locked down to HTML retrieval only. Because the current
|
|
207
|
+
`agent-browser` CLI contract cannot enforce redirect, subresource, or
|
|
208
|
+
connect-time DNS allow-lists, network browser rendering fails closed unless the
|
|
209
|
+
operator sets `DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1` for trusted targets.
|
|
210
|
+
For localhost/loopback HTTP tests, set `DOCPULL_RENDER_ALLOW_LOCAL_TARGETS=1`.
|
|
174
211
|
|
|
175
212
|
For stronger isolation, cloud runtimes are available explicitly:
|
|
176
213
|
`docpull render URL --runtime vercel` uses the Vercel Sandbox CLI and Vercel
|
|
@@ -188,17 +225,17 @@ Use `--budget 0` when a run must not make paid-capable provider or cloud calls:
|
|
|
188
225
|
```bash
|
|
189
226
|
docpull https://docs.example.com --budget 0 -o ./docs/example
|
|
190
227
|
docpull discover scan https://docs.example.com -o ./packs/discovery
|
|
191
|
-
docpull render https://example.com/app --runtime local --budget 0
|
|
228
|
+
DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1 docpull render https://example.com/app --runtime local --budget 0
|
|
192
229
|
docpull providers context-pack "Find official docs" --provider all --dry-run --budget 0 --json
|
|
193
230
|
docpull benchmark quick --zero-dollar --target-set zero-dollar --provider all
|
|
194
231
|
```
|
|
195
232
|
|
|
196
233
|
Under a zero budget, local cache, direct HTTP, sitemap/static-link discovery,
|
|
197
234
|
local extraction, local indexing, pack analysis, monitors, and local
|
|
198
|
-
`agent-browser` rendering remain allowed. Live Tavily, Exa,
|
|
199
|
-
Sandbox, and E2B calls are blocked before execution. Runs
|
|
200
|
-
paid-capable route write `run.accounting.json` with
|
|
201
|
-
HTTP/cache, browser, and blocked-action metadata.
|
|
235
|
+
`agent-browser` rendering for trusted targets remain allowed. Live Tavily, Exa,
|
|
236
|
+
Parallel, Vercel Sandbox, and E2B calls are blocked before execution. Runs
|
|
237
|
+
involving a budget or paid-capable route write `run.accounting.json` with
|
|
238
|
+
non-secret route, cost, HTTP/cache, browser, and blocked-action metadata.
|
|
202
239
|
|
|
203
240
|
Use `docpull discover scan URL` to build a provider-free discovery pack from
|
|
204
241
|
open site hints: `llms.txt`, RSS/Atom feeds, OpenAPI specs, sitemap indexes,
|
|
@@ -207,11 +244,11 @@ contract as provider imports and URL/sitemap files, so the next step is still
|
|
|
207
244
|
`docpull discover select` or `docpull discover fetch`.
|
|
208
245
|
|
|
209
246
|
When a zero-dollar benchmark or local run is partial, DocPull reports the
|
|
210
|
-
lowest-friction escalation path before spending money: local
|
|
211
|
-
first, BYOK providers next, and cloud rendering only when
|
|
212
|
-
infrastructure is the blocker. Benchmark reports include
|
|
213
|
-
estimated paid request counts, and estimated paid cost
|
|
214
|
-
provider or cloud call is made.
|
|
247
|
+
lowest-friction escalation path before spending money: trusted-target local
|
|
248
|
+
`--render fallback` first, BYOK providers next, and cloud rendering only when
|
|
249
|
+
local rendering or infrastructure is the blocker. Benchmark reports include
|
|
250
|
+
suggested commands, estimated paid request counts, and estimated paid cost
|
|
251
|
+
guards before any provider or cloud call is made.
|
|
215
252
|
|
|
216
253
|
The `zero-dollar` benchmark target set is the Phase 2 measurement matrix. It
|
|
217
254
|
keeps the existing docs/provider targets and adds JS-heavy docs, pricing,
|
|
@@ -232,6 +269,80 @@ bundled provider billing. The hosted boundary does not change the OSS default:
|
|
|
232
269
|
no hidden paid calls, no CAPTCHA bypass, no stealth scraping, and no claim of a
|
|
233
270
|
proprietary web-scale index.
|
|
234
271
|
|
|
272
|
+
## Persistent Projects
|
|
273
|
+
|
|
274
|
+
Use project mode when a source corpus needs to stay fresh over time. A project
|
|
275
|
+
is a local `docpull.yaml` plus a `.docpull/` state directory containing run
|
|
276
|
+
history, cache, manifests, context-pack exports, eval sets, and a SQLite index.
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
docpull init stripe-docs
|
|
280
|
+
docpull add https://docs.stripe.com
|
|
281
|
+
docpull sync
|
|
282
|
+
docpull diff
|
|
283
|
+
docpull export context-pack --target cursor
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
Each sync writes a normal local DocPull pack under `.docpull/runs/<run_id>/`,
|
|
287
|
+
including `run.json`, `documents.jsonl`, `chunks.jsonl`, `manifest.json`,
|
|
288
|
+
`documents.ndjson`, `corpus.manifest.json`, `sources.md`,
|
|
289
|
+
`source-health.json`, `local.pack.json`, and accounting metadata.
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
# Inspect the latest project state
|
|
293
|
+
docpull status
|
|
294
|
+
|
|
295
|
+
# Show run history
|
|
296
|
+
docpull history
|
|
297
|
+
|
|
298
|
+
# Diff the latest two runs, with deterministic local categories by default
|
|
299
|
+
docpull diff
|
|
300
|
+
|
|
301
|
+
# Write a review summary for the latest run
|
|
302
|
+
docpull review
|
|
303
|
+
|
|
304
|
+
# Create a versioned context-pack release
|
|
305
|
+
docpull release context-pack --target cursor --tag stripe-docs-v1
|
|
306
|
+
|
|
307
|
+
# Generate eval cases from changed or latest documents
|
|
308
|
+
docpull eval-set --limit 25
|
|
309
|
+
|
|
310
|
+
# One-command project sync, diff, and export for one source
|
|
311
|
+
docpull watch https://docs.stripe.com --export cursor --alert changes
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
`docpull diff` is hash-based and deterministic locally. Optional BYOK semantic
|
|
315
|
+
summaries are advisory and skip cleanly when no model key is configured.
|
|
316
|
+
Use `docpull add URL --discover` or `docpull sync --update-discovery` to
|
|
317
|
+
refresh and persist discovered source URLs in `docpull.yaml`; sync then uses
|
|
318
|
+
that stored URL set for repeatable exact refreshes.
|
|
319
|
+
|
|
320
|
+
For authenticated sources, store only environment variable references in
|
|
321
|
+
`docpull.yaml`; DocPull resolves values in memory at sync time and writes only
|
|
322
|
+
masked auth type/readiness to status, manifests, reviews, releases, and
|
|
323
|
+
webhooks:
|
|
324
|
+
|
|
325
|
+
```yaml
|
|
326
|
+
sources:
|
|
327
|
+
- name: internal-docs
|
|
328
|
+
url: https://docs.example.com
|
|
329
|
+
auth:
|
|
330
|
+
type: bearer_env
|
|
331
|
+
env: EXAMPLE_DOCS_TOKEN
|
|
332
|
+
policy: explicit-private
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
The commercial control-plane shape is API-led but not API-only: the local CLI
|
|
336
|
+
remains the trust-building engine, while hosted DocPull manages projects, sync
|
|
337
|
+
jobs, diffs, exports, releases, and signed webhooks through `/v1` JSON
|
|
338
|
+
endpoints. The hosted ASGI MVP lives in `docpull.hosted`; `docpull remote ...`
|
|
339
|
+
stores hosted API connection metadata and calls the same project lifecycle
|
|
340
|
+
remotely. Remote bearer tokens are sent only to HTTPS API URLs by default;
|
|
341
|
+
`--allow-insecure-local-http` is limited to localhost/loopback development.
|
|
342
|
+
|
|
343
|
+
The launch screenshot for this flow lives at
|
|
344
|
+
[`docs/launch-assets/docpull-project-diff-demo.png`](docs/launch-assets/docpull-project-diff-demo.png).
|
|
345
|
+
|
|
235
346
|
## 30-Second Usage
|
|
236
347
|
|
|
237
348
|
```bash
|
|
@@ -417,7 +528,8 @@ For those cases, use browser automation, such as Playwright, then pass rendered
|
|
|
417
528
|
HTML or exported content into your pipeline. For simple public JS-rendered
|
|
418
529
|
pages, `docpull render` and `--render fallback` provide an explicit local
|
|
419
530
|
fallback without changing the default crawler behavior. The fallback requires
|
|
420
|
-
the optional external `agent-browser` backend
|
|
531
|
+
the optional external `agent-browser` backend and
|
|
532
|
+
`DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1` for trusted network targets.
|
|
421
533
|
|
|
422
534
|
## How It Compares
|
|
423
535
|
|
|
@@ -501,7 +613,7 @@ part of the package release contract.
|
|
|
501
613
|
rendering, authenticated-source checks, and cron-friendly monitors:
|
|
502
614
|
`docpull policy`, `docpull discover`, `docpull refresh`,
|
|
503
615
|
`docpull pack audit`, `docpull answer-pack`, `docpull export`,
|
|
504
|
-
`docpull serve`, `docpull render`, `docpull auth check`, and
|
|
616
|
+
`docpull serve`, `docpull share`, `docpull render`, `docpull auth check`, and
|
|
505
617
|
`docpull monitor`.
|
|
506
618
|
- `docpull export` writes local files for OpenAI vector JSONL, LangChain,
|
|
507
619
|
LlamaIndex, DSPy, Sheets CSV/TSV, n8n workflow JSON, Vercel AI SDK JSON,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# docpull
|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**Keep AI agents synced with changing public docs. Browser-free by default.**
|
|
4
4
|
|
|
5
5
|
[](https://www.python.org/downloads/)
|
|
6
6
|
[](https://badge.fury.io/py/docpull)
|
|
@@ -8,10 +8,15 @@
|
|
|
8
8
|
[](https://github.com/raintree-technology/docpull/stargazers)
|
|
9
9
|
[](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
DocPull turns public docs and web sources into refreshable, cited, agent-ready
|
|
12
|
+
context packs. Define source sets once, sync them over time, diff what changed,
|
|
13
|
+
and export fresh context for Cursor, Claude, Codex, OpenAI, LlamaIndex,
|
|
14
|
+
LangChain, MCP clients, and RAG pipelines.
|
|
15
|
+
|
|
16
|
+
The original `docpull URL ...` workflow still works: fetch public or explicitly
|
|
17
|
+
authorized static/server-rendered web pages and write clean Markdown, NDJSON,
|
|
18
|
+
SQLite, or OKF outputs. Project mode adds the persistent evidence lifecycle on
|
|
19
|
+
top: sources, runs, diffs, exports, evals, accounting, and local auditability.
|
|
15
20
|
|
|
16
21
|
DocPull is local-first: direct fetching, sitemap/link discovery, extraction,
|
|
17
22
|
indexing, pack intelligence, and local `agent-browser` rendering can run with no
|
|
@@ -40,6 +45,34 @@ reason unless you explicitly opt into the local `agent-browser` renderer. See
|
|
|
40
45
|
pip install docpull
|
|
41
46
|
```
|
|
42
47
|
|
|
48
|
+
## Project Quickstart
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
docpull init stripe-docs
|
|
52
|
+
docpull add https://docs.stripe.com
|
|
53
|
+
docpull sync
|
|
54
|
+
docpull diff
|
|
55
|
+
docpull export context-pack --target cursor
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Example diff after a later sync:
|
|
59
|
+
|
|
60
|
+
```text
|
|
61
|
+
Project diff: +4 -2 ~18 api=2 pricing=1
|
|
62
|
+
|
|
63
|
+
Changed pages:
|
|
64
|
+
- /payments/payment-intents
|
|
65
|
+
likely API behavior change
|
|
66
|
+
- /billing/subscriptions
|
|
67
|
+
pricing / billing change
|
|
68
|
+
- /webhooks
|
|
69
|
+
likely API behavior change
|
|
70
|
+
|
|
71
|
+
0 failed URLs
|
|
72
|
+
0 robots blocked
|
|
73
|
+
0 paid/cloud routes used
|
|
74
|
+
```
|
|
75
|
+
|
|
43
76
|
Install optional extras as needed:
|
|
44
77
|
|
|
45
78
|
```bash
|
|
@@ -58,7 +91,11 @@ install. Install an `agent-browser` compatible CLI separately, put it on
|
|
|
58
91
|
`PATH`, or set `DOCPULL_AGENT_BROWSER_BIN=/path/to/agent-browser`. Verify the
|
|
59
92
|
runtime with `docpull render --check`. Render targets must use HTTPS except for
|
|
60
93
|
localhost/loopback HTTP during local testing, and DocPull keeps renderer action
|
|
61
|
-
permissions locked down to HTML retrieval only.
|
|
94
|
+
permissions locked down to HTML retrieval only. Because the current
|
|
95
|
+
`agent-browser` CLI contract cannot enforce redirect, subresource, or
|
|
96
|
+
connect-time DNS allow-lists, network browser rendering fails closed unless the
|
|
97
|
+
operator sets `DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1` for trusted targets.
|
|
98
|
+
For localhost/loopback HTTP tests, set `DOCPULL_RENDER_ALLOW_LOCAL_TARGETS=1`.
|
|
62
99
|
|
|
63
100
|
For stronger isolation, cloud runtimes are available explicitly:
|
|
64
101
|
`docpull render URL --runtime vercel` uses the Vercel Sandbox CLI and Vercel
|
|
@@ -76,17 +113,17 @@ Use `--budget 0` when a run must not make paid-capable provider or cloud calls:
|
|
|
76
113
|
```bash
|
|
77
114
|
docpull https://docs.example.com --budget 0 -o ./docs/example
|
|
78
115
|
docpull discover scan https://docs.example.com -o ./packs/discovery
|
|
79
|
-
docpull render https://example.com/app --runtime local --budget 0
|
|
116
|
+
DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1 docpull render https://example.com/app --runtime local --budget 0
|
|
80
117
|
docpull providers context-pack "Find official docs" --provider all --dry-run --budget 0 --json
|
|
81
118
|
docpull benchmark quick --zero-dollar --target-set zero-dollar --provider all
|
|
82
119
|
```
|
|
83
120
|
|
|
84
121
|
Under a zero budget, local cache, direct HTTP, sitemap/static-link discovery,
|
|
85
122
|
local extraction, local indexing, pack analysis, monitors, and local
|
|
86
|
-
`agent-browser` rendering remain allowed. Live Tavily, Exa,
|
|
87
|
-
Sandbox, and E2B calls are blocked before execution. Runs
|
|
88
|
-
paid-capable route write `run.accounting.json` with
|
|
89
|
-
HTTP/cache, browser, and blocked-action metadata.
|
|
123
|
+
`agent-browser` rendering for trusted targets remain allowed. Live Tavily, Exa,
|
|
124
|
+
Parallel, Vercel Sandbox, and E2B calls are blocked before execution. Runs
|
|
125
|
+
involving a budget or paid-capable route write `run.accounting.json` with
|
|
126
|
+
non-secret route, cost, HTTP/cache, browser, and blocked-action metadata.
|
|
90
127
|
|
|
91
128
|
Use `docpull discover scan URL` to build a provider-free discovery pack from
|
|
92
129
|
open site hints: `llms.txt`, RSS/Atom feeds, OpenAPI specs, sitemap indexes,
|
|
@@ -95,11 +132,11 @@ contract as provider imports and URL/sitemap files, so the next step is still
|
|
|
95
132
|
`docpull discover select` or `docpull discover fetch`.
|
|
96
133
|
|
|
97
134
|
When a zero-dollar benchmark or local run is partial, DocPull reports the
|
|
98
|
-
lowest-friction escalation path before spending money: local
|
|
99
|
-
first, BYOK providers next, and cloud rendering only when
|
|
100
|
-
infrastructure is the blocker. Benchmark reports include
|
|
101
|
-
estimated paid request counts, and estimated paid cost
|
|
102
|
-
provider or cloud call is made.
|
|
135
|
+
lowest-friction escalation path before spending money: trusted-target local
|
|
136
|
+
`--render fallback` first, BYOK providers next, and cloud rendering only when
|
|
137
|
+
local rendering or infrastructure is the blocker. Benchmark reports include
|
|
138
|
+
suggested commands, estimated paid request counts, and estimated paid cost
|
|
139
|
+
guards before any provider or cloud call is made.
|
|
103
140
|
|
|
104
141
|
The `zero-dollar` benchmark target set is the Phase 2 measurement matrix. It
|
|
105
142
|
keeps the existing docs/provider targets and adds JS-heavy docs, pricing,
|
|
@@ -120,6 +157,80 @@ bundled provider billing. The hosted boundary does not change the OSS default:
|
|
|
120
157
|
no hidden paid calls, no CAPTCHA bypass, no stealth scraping, and no claim of a
|
|
121
158
|
proprietary web-scale index.
|
|
122
159
|
|
|
160
|
+
## Persistent Projects
|
|
161
|
+
|
|
162
|
+
Use project mode when a source corpus needs to stay fresh over time. A project
|
|
163
|
+
is a local `docpull.yaml` plus a `.docpull/` state directory containing run
|
|
164
|
+
history, cache, manifests, context-pack exports, eval sets, and a SQLite index.
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
docpull init stripe-docs
|
|
168
|
+
docpull add https://docs.stripe.com
|
|
169
|
+
docpull sync
|
|
170
|
+
docpull diff
|
|
171
|
+
docpull export context-pack --target cursor
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Each sync writes a normal local DocPull pack under `.docpull/runs/<run_id>/`,
|
|
175
|
+
including `run.json`, `documents.jsonl`, `chunks.jsonl`, `manifest.json`,
|
|
176
|
+
`documents.ndjson`, `corpus.manifest.json`, `sources.md`,
|
|
177
|
+
`source-health.json`, `local.pack.json`, and accounting metadata.
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
# Inspect the latest project state
|
|
181
|
+
docpull status
|
|
182
|
+
|
|
183
|
+
# Show run history
|
|
184
|
+
docpull history
|
|
185
|
+
|
|
186
|
+
# Diff the latest two runs, with deterministic local categories by default
|
|
187
|
+
docpull diff
|
|
188
|
+
|
|
189
|
+
# Write a review summary for the latest run
|
|
190
|
+
docpull review
|
|
191
|
+
|
|
192
|
+
# Create a versioned context-pack release
|
|
193
|
+
docpull release context-pack --target cursor --tag stripe-docs-v1
|
|
194
|
+
|
|
195
|
+
# Generate eval cases from changed or latest documents
|
|
196
|
+
docpull eval-set --limit 25
|
|
197
|
+
|
|
198
|
+
# One-command project sync, diff, and export for one source
|
|
199
|
+
docpull watch https://docs.stripe.com --export cursor --alert changes
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
`docpull diff` is hash-based and deterministic locally. Optional BYOK semantic
|
|
203
|
+
summaries are advisory and skip cleanly when no model key is configured.
|
|
204
|
+
Use `docpull add URL --discover` or `docpull sync --update-discovery` to
|
|
205
|
+
refresh and persist discovered source URLs in `docpull.yaml`; sync then uses
|
|
206
|
+
that stored URL set for repeatable exact refreshes.
|
|
207
|
+
|
|
208
|
+
For authenticated sources, store only environment variable references in
|
|
209
|
+
`docpull.yaml`; DocPull resolves values in memory at sync time and writes only
|
|
210
|
+
masked auth type/readiness to status, manifests, reviews, releases, and
|
|
211
|
+
webhooks:
|
|
212
|
+
|
|
213
|
+
```yaml
|
|
214
|
+
sources:
|
|
215
|
+
- name: internal-docs
|
|
216
|
+
url: https://docs.example.com
|
|
217
|
+
auth:
|
|
218
|
+
type: bearer_env
|
|
219
|
+
env: EXAMPLE_DOCS_TOKEN
|
|
220
|
+
policy: explicit-private
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
The commercial control-plane shape is API-led but not API-only: the local CLI
|
|
224
|
+
remains the trust-building engine, while hosted DocPull manages projects, sync
|
|
225
|
+
jobs, diffs, exports, releases, and signed webhooks through `/v1` JSON
|
|
226
|
+
endpoints. The hosted ASGI MVP lives in `docpull.hosted`; `docpull remote ...`
|
|
227
|
+
stores hosted API connection metadata and calls the same project lifecycle
|
|
228
|
+
remotely. Remote bearer tokens are sent only to HTTPS API URLs by default;
|
|
229
|
+
`--allow-insecure-local-http` is limited to localhost/loopback development.
|
|
230
|
+
|
|
231
|
+
The launch screenshot for this flow lives at
|
|
232
|
+
[`docs/launch-assets/docpull-project-diff-demo.png`](docs/launch-assets/docpull-project-diff-demo.png).
|
|
233
|
+
|
|
123
234
|
## 30-Second Usage
|
|
124
235
|
|
|
125
236
|
```bash
|
|
@@ -305,7 +416,8 @@ For those cases, use browser automation, such as Playwright, then pass rendered
|
|
|
305
416
|
HTML or exported content into your pipeline. For simple public JS-rendered
|
|
306
417
|
pages, `docpull render` and `--render fallback` provide an explicit local
|
|
307
418
|
fallback without changing the default crawler behavior. The fallback requires
|
|
308
|
-
the optional external `agent-browser` backend
|
|
419
|
+
the optional external `agent-browser` backend and
|
|
420
|
+
`DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1` for trusted network targets.
|
|
309
421
|
|
|
310
422
|
## How It Compares
|
|
311
423
|
|
|
@@ -389,7 +501,7 @@ part of the package release contract.
|
|
|
389
501
|
rendering, authenticated-source checks, and cron-friendly monitors:
|
|
390
502
|
`docpull policy`, `docpull discover`, `docpull refresh`,
|
|
391
503
|
`docpull pack audit`, `docpull answer-pack`, `docpull export`,
|
|
392
|
-
`docpull serve`, `docpull render`, `docpull auth check`, and
|
|
504
|
+
`docpull serve`, `docpull share`, `docpull render`, `docpull auth check`, and
|
|
393
505
|
`docpull monitor`.
|
|
394
506
|
- `docpull export` writes local files for OpenAI vector JSONL, LangChain,
|
|
395
507
|
LlamaIndex, DSPy, Sheets CSV/TSV, n8n workflow JSON, Vercel AI SDK JSON,
|
|
@@ -4,9 +4,9 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "docpull"
|
|
7
|
-
version = "5.0
|
|
7
|
+
version = "5.5.0"
|
|
8
8
|
dynamic = []
|
|
9
|
-
description = "
|
|
9
|
+
description = "Keep AI agents synced with changing public docs and web sources"
|
|
10
10
|
readme = {file = "README.md", content-type = "text/markdown"}
|
|
11
11
|
requires-python = ">=3.10"
|
|
12
12
|
license = "MIT"
|
|
@@ -14,9 +14,18 @@ Usage:
|
|
|
14
14
|
print(event)
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
|
-
__version__ = "5.0
|
|
17
|
+
__version__ = "5.5.0"
|
|
18
18
|
|
|
19
19
|
from .cache import CacheManager, StreamingDeduplicator
|
|
20
|
+
from .context_packs import (
|
|
21
|
+
build_brand_pack,
|
|
22
|
+
build_image_pack,
|
|
23
|
+
build_product_pack,
|
|
24
|
+
build_search_pack,
|
|
25
|
+
build_styleguide_pack,
|
|
26
|
+
capture_screenshot_pack,
|
|
27
|
+
extract_schema,
|
|
28
|
+
)
|
|
20
29
|
from .conversion.chunking import Chunk, TokenCounter, chunk_markdown
|
|
21
30
|
from .core.fetcher import Fetcher, fetch_blocking, fetch_one
|
|
22
31
|
from .exports import ExportResult, export_pack
|
|
@@ -94,6 +103,7 @@ from .scraper import (
|
|
|
94
103
|
scrape_site,
|
|
95
104
|
)
|
|
96
105
|
from .server import PackASGIApp, PackServerError, create_pack_app
|
|
106
|
+
from .share import ReportHTTPServer, ShareError, create_report_server, render_report_document
|
|
97
107
|
|
|
98
108
|
__all__ = [
|
|
99
109
|
"__version__",
|
|
@@ -103,6 +113,13 @@ __all__ = [
|
|
|
103
113
|
"refresh_pack",
|
|
104
114
|
"audit_pack",
|
|
105
115
|
"answer_pack",
|
|
116
|
+
"build_brand_pack",
|
|
117
|
+
"build_styleguide_pack",
|
|
118
|
+
"build_product_pack",
|
|
119
|
+
"extract_schema",
|
|
120
|
+
"build_image_pack",
|
|
121
|
+
"capture_screenshot_pack",
|
|
122
|
+
"build_search_pack",
|
|
106
123
|
"extract_pack",
|
|
107
124
|
"map_sources",
|
|
108
125
|
"crawl_pack",
|
|
@@ -134,6 +151,10 @@ __all__ = [
|
|
|
134
151
|
"create_pack_app",
|
|
135
152
|
"PackASGIApp",
|
|
136
153
|
"PackServerError",
|
|
154
|
+
"create_report_server",
|
|
155
|
+
"render_report_document",
|
|
156
|
+
"ReportHTTPServer",
|
|
157
|
+
"ShareError",
|
|
137
158
|
"ScrapeResult",
|
|
138
159
|
"ScrapeRunResult",
|
|
139
160
|
"Scraper",
|
|
@@ -191,6 +191,17 @@ Examples:
|
|
|
191
191
|
docpull https://example.com --include-paths "/api/*" --exclude-paths "/changelog/*"
|
|
192
192
|
|
|
193
193
|
Subcommands:
|
|
194
|
+
init Create a persistent DocPull project
|
|
195
|
+
add Add a source to a project
|
|
196
|
+
sync Sync configured project sources
|
|
197
|
+
diff Diff project runs
|
|
198
|
+
status Show project status
|
|
199
|
+
history Show project run history
|
|
200
|
+
review Write a source-health/diff review for a run
|
|
201
|
+
release Create versioned context-pack release artifacts
|
|
202
|
+
remote Call a hosted DocPull API control plane
|
|
203
|
+
eval-set Generate local eval cases from project runs
|
|
204
|
+
watch Sync, diff, and export one source
|
|
194
205
|
render Render one public URL or check the optional agent-browser backend
|
|
195
206
|
discover Build/select/fetch provider-neutral discovery packs
|
|
196
207
|
extract-pack Extract known URLs into a provider-neutral local pack
|
|
@@ -198,6 +209,13 @@ Subcommands:
|
|
|
198
209
|
crawl-pack Select mapped candidates and fetch a local pack
|
|
199
210
|
research-pack Produce cited local research results from a pack
|
|
200
211
|
entities-pack Build local entity/list packs from existing evidence
|
|
212
|
+
brand-pack Build an evidence-backed local brand profile
|
|
213
|
+
styleguide-pack Extract local design tokens from a public site
|
|
214
|
+
product-pack Normalize product and pricing evidence
|
|
215
|
+
extract-schema Extract a JSON shape from local evidence
|
|
216
|
+
image-pack Build a local visual asset manifest
|
|
217
|
+
screenshot-pack Capture an explicit browser-rendered screenshot
|
|
218
|
+
search-pack Build a local-first search result pack
|
|
201
219
|
policy Validate or explain source policy files
|
|
202
220
|
auth Check authenticated source access without writing content
|
|
203
221
|
refresh Refresh an existing local pack and write change reports
|
|
@@ -206,6 +224,7 @@ Subcommands:
|
|
|
206
224
|
answer-pack Answer from local pack evidence with citations
|
|
207
225
|
export Export a pack for agent or RAG tools
|
|
208
226
|
serve Serve a local pack over localhost JSON routes
|
|
227
|
+
share Serve a Markdown or HTML report at a local URL
|
|
209
228
|
monitor Run cron-friendly local pack monitors
|
|
210
229
|
parallel Build optional Parallel-backed context packs
|
|
211
230
|
provider Check provider keys and run provider-backed context packs
|
|
@@ -1253,6 +1272,7 @@ def _run_render_doctor_cli() -> int:
|
|
|
1253
1272
|
exit_code = 1
|
|
1254
1273
|
console.print()
|
|
1255
1274
|
console.print("Cloud runtimes execute the same agent-browser JSON contract inside a sandbox/template.")
|
|
1275
|
+
console.print("Set DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1 only for trusted render targets.")
|
|
1256
1276
|
console.print("Use `docpull render init e2b` or `docpull render init vercel` for template recipes.")
|
|
1257
1277
|
return exit_code
|
|
1258
1278
|
|
|
@@ -1286,10 +1306,13 @@ agent-browser install
|
|
|
1286
1306
|
|
|
1287
1307
|
# Then render through the same DocPull contract:
|
|
1288
1308
|
export E2B_API_KEY=<your-e2b-api-key>
|
|
1309
|
+
export DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1
|
|
1289
1310
|
docpull render https://example.com --runtime e2b --template {template}
|
|
1290
1311
|
|
|
1291
1312
|
# For smoke tests that may consume provider quota:
|
|
1292
|
-
|
|
1313
|
+
DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1 \\
|
|
1314
|
+
DOCPULL_LIVE_CLOUD_RENDER=1 \\
|
|
1315
|
+
.venv/bin/python -m pytest tests/test_rendering.py -q
|
|
1293
1316
|
"""
|
|
1294
1317
|
|
|
1295
1318
|
|
|
@@ -1302,6 +1325,7 @@ npm install -g agent-browser
|
|
|
1302
1325
|
agent-browser install
|
|
1303
1326
|
|
|
1304
1327
|
# Then render through the same DocPull contract:
|
|
1328
|
+
export DOCPULL_RENDER_TRUSTED_BROWSER_TARGETS=1
|
|
1305
1329
|
docpull render https://example.com --runtime vercel --cloud-agent-browser-install skip
|
|
1306
1330
|
|
|
1307
1331
|
# If you intentionally want a cold runtime install and npm is available:
|
|
@@ -1349,6 +1373,50 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
1349
1373
|
raw_argv = list(argv) if argv is not None else sys.argv[1:]
|
|
1350
1374
|
if raw_argv and raw_argv[0] == "render":
|
|
1351
1375
|
return run_render_cli(raw_argv[1:])
|
|
1376
|
+
if raw_argv and raw_argv[0] == "init":
|
|
1377
|
+
from .project import run_init_cli
|
|
1378
|
+
|
|
1379
|
+
return run_init_cli(raw_argv[1:])
|
|
1380
|
+
if raw_argv and raw_argv[0] == "add":
|
|
1381
|
+
from .project import run_add_cli
|
|
1382
|
+
|
|
1383
|
+
return run_add_cli(raw_argv[1:])
|
|
1384
|
+
if raw_argv and raw_argv[0] == "sync":
|
|
1385
|
+
from .project import run_sync_cli
|
|
1386
|
+
|
|
1387
|
+
return run_sync_cli(raw_argv[1:])
|
|
1388
|
+
if raw_argv and raw_argv[0] == "diff":
|
|
1389
|
+
from .project import run_diff_cli
|
|
1390
|
+
|
|
1391
|
+
return run_diff_cli(raw_argv[1:])
|
|
1392
|
+
if raw_argv and raw_argv[0] == "status":
|
|
1393
|
+
from .project import run_status_cli
|
|
1394
|
+
|
|
1395
|
+
return run_status_cli(raw_argv[1:])
|
|
1396
|
+
if raw_argv and raw_argv[0] == "history":
|
|
1397
|
+
from .project import run_history_cli
|
|
1398
|
+
|
|
1399
|
+
return run_history_cli(raw_argv[1:])
|
|
1400
|
+
if raw_argv and raw_argv[0] == "review":
|
|
1401
|
+
from .project import run_review_cli
|
|
1402
|
+
|
|
1403
|
+
return run_review_cli(raw_argv[1:])
|
|
1404
|
+
if raw_argv and raw_argv[0] == "release":
|
|
1405
|
+
from .project import run_release_cli
|
|
1406
|
+
|
|
1407
|
+
return run_release_cli(raw_argv[1:])
|
|
1408
|
+
if raw_argv and raw_argv[0] == "remote":
|
|
1409
|
+
from .project import run_remote_cli
|
|
1410
|
+
|
|
1411
|
+
return run_remote_cli(raw_argv[1:])
|
|
1412
|
+
if raw_argv and raw_argv[0] == "eval-set":
|
|
1413
|
+
from .project import run_eval_set_cli
|
|
1414
|
+
|
|
1415
|
+
return run_eval_set_cli(raw_argv[1:])
|
|
1416
|
+
if raw_argv and raw_argv[0] == "watch":
|
|
1417
|
+
from .project import run_watch_cli
|
|
1418
|
+
|
|
1419
|
+
return run_watch_cli(raw_argv[1:])
|
|
1352
1420
|
if raw_argv and raw_argv[0] == "mcp":
|
|
1353
1421
|
from .mcp.server import run_mcp_server
|
|
1354
1422
|
|
|
@@ -1369,7 +1437,15 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
1369
1437
|
from .server import run_serve_cli
|
|
1370
1438
|
|
|
1371
1439
|
return run_serve_cli(raw_argv[1:])
|
|
1440
|
+
if raw_argv and raw_argv[0] == "share":
|
|
1441
|
+
from .share import run_share_cli
|
|
1442
|
+
|
|
1443
|
+
return run_share_cli(raw_argv[1:])
|
|
1372
1444
|
if raw_argv and raw_argv[0] == "export":
|
|
1445
|
+
if len(raw_argv) > 1 and raw_argv[1] == "context-pack":
|
|
1446
|
+
from .project import run_project_export_cli
|
|
1447
|
+
|
|
1448
|
+
return run_project_export_cli(raw_argv[2:])
|
|
1373
1449
|
from .exports import run_export_cli
|
|
1374
1450
|
|
|
1375
1451
|
return run_export_cli(raw_argv[1:])
|
|
@@ -1425,6 +1501,34 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
1425
1501
|
from .parity_cli import run_entities_pack_cli
|
|
1426
1502
|
|
|
1427
1503
|
return run_entities_pack_cli(raw_argv[1:])
|
|
1504
|
+
if raw_argv and raw_argv[0] == "brand-pack":
|
|
1505
|
+
from .context_packs.cli import run_brand_pack_cli
|
|
1506
|
+
|
|
1507
|
+
return run_brand_pack_cli(raw_argv[1:])
|
|
1508
|
+
if raw_argv and raw_argv[0] == "styleguide-pack":
|
|
1509
|
+
from .context_packs.cli import run_styleguide_pack_cli
|
|
1510
|
+
|
|
1511
|
+
return run_styleguide_pack_cli(raw_argv[1:])
|
|
1512
|
+
if raw_argv and raw_argv[0] == "product-pack":
|
|
1513
|
+
from .context_packs.cli import run_product_pack_cli
|
|
1514
|
+
|
|
1515
|
+
return run_product_pack_cli(raw_argv[1:])
|
|
1516
|
+
if raw_argv and raw_argv[0] == "extract-schema":
|
|
1517
|
+
from .context_packs.cli import run_extract_schema_cli
|
|
1518
|
+
|
|
1519
|
+
return run_extract_schema_cli(raw_argv[1:])
|
|
1520
|
+
if raw_argv and raw_argv[0] == "image-pack":
|
|
1521
|
+
from .context_packs.cli import run_image_pack_cli
|
|
1522
|
+
|
|
1523
|
+
return run_image_pack_cli(raw_argv[1:])
|
|
1524
|
+
if raw_argv and raw_argv[0] == "screenshot-pack":
|
|
1525
|
+
from .context_packs.cli import run_screenshot_pack_cli
|
|
1526
|
+
|
|
1527
|
+
return run_screenshot_pack_cli(raw_argv[1:])
|
|
1528
|
+
if raw_argv and raw_argv[0] == "search-pack":
|
|
1529
|
+
from .context_packs.cli import run_search_pack_cli
|
|
1530
|
+
|
|
1531
|
+
return run_search_pack_cli(raw_argv[1:])
|
|
1428
1532
|
if raw_argv and raw_argv[0] in {"provider", "providers"}:
|
|
1429
1533
|
from .provider_cli import run_provider_cli
|
|
1430
1534
|
|