docpull 6.5.5__tar.gz → 6.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docpull-6.5.5/src/docpull.egg-info → docpull-6.8.0}/PKG-INFO +30 -22
- {docpull-6.5.5 → docpull-6.8.0}/README.md +14 -5
- {docpull-6.5.5 → docpull-6.8.0}/pyproject.toml +29 -21
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/__init__.py +4 -1
- docpull-6.8.0/src/docpull/_jsonutil.py +48 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/accounting.py +0 -33
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/frontier.py +8 -1
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cli.py +20 -0
- docpull-6.8.0/src/docpull/compat_api.py +469 -0
- docpull-6.8.0/src/docpull/compat_jobs.py +355 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_ci.py +9 -19
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/_legacy_cli.py +0 -71
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/common.py +0 -20
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/feed.py +9 -17
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/openapi.py +4 -13
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/package.py +1 -4
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/typed.py +3 -28
- docpull-6.8.0/src/docpull/context_query.py +493 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/core/fetcher.py +63 -29
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/__init__.py +1 -38
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/_fetch.py +0 -8
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/crawler.py +4 -1
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/sitemap.py +6 -0
- docpull-6.8.0/src/docpull/discovery_index.py +356 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/document_parse.py +5 -3
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/eval_grade.py +6 -117
- docpull-6.8.0/src/docpull/frozen.py +123 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/graph.py +1 -9
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/client.py +5 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/rate_limiter.py +25 -7
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/local_workflows.py +29 -164
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/server.py +64 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/events.py +2 -2
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/monitor.py +1 -1
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/output_contract.py +3 -13
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pack_reader.py +3 -23
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pack_tools.py +22 -29
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/base.py +11 -3
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/convert.py +2 -2
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/project.py +170 -313
- docpull-6.8.0/src/docpull/registry.py +326 -0
- docpull-6.8.0/src/docpull/research.py +315 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/server.py +85 -1
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/surface.py +8 -2
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/time_utils.py +4 -4
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/warc.py +2 -2
- docpull-6.8.0/src/docpull/web_search.py +312 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/workflows.py +15 -35
- {docpull-6.5.5 → docpull-6.8.0/src/docpull.egg-info}/PKG-INFO +30 -22
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/SOURCES.txt +28 -37
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/requires.txt +20 -14
- docpull-6.8.0/tests/test_cache_lock_regressions.py +172 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_ci_policy.py +5 -5
- docpull-6.8.0/tests/test_compat_api.py +136 -0
- docpull-6.8.0/tests/test_compat_jobs.py +176 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_context_packs.py +0 -127
- docpull-6.8.0/tests/test_context_query.py +328 -0
- docpull-6.8.0/tests/test_crawl_regressions.py +165 -0
- docpull-6.8.0/tests/test_discovery_index.py +158 -0
- docpull-6.8.0/tests/test_fetch_lifecycle.py +236 -0
- docpull-6.8.0/tests/test_frozen.py +65 -0
- docpull-6.8.0/tests/test_jsonutil.py +35 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_local_workflows.py +3 -35
- docpull-6.8.0/tests/test_model_cache_lock.py +305 -0
- docpull-6.8.0/tests/test_model_crawl_politeness.py +334 -0
- docpull-6.8.0/tests/test_model_fetch_pipeline.py +300 -0
- docpull-6.8.0/tests/test_model_jobs_monitor.py +242 -0
- docpull-6.8.0/tests/test_monitor_fetch_failures.py +104 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_outputs_e2e.py +0 -104
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_pack_tools.py +28 -81
- docpull-6.8.0/tests/test_parser_dependency_support.py +54 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_project.py +182 -34
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_real_feature_smoke_script.py +41 -0
- docpull-6.8.0/tests/test_registry.py +185 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_release_a_plus_check_script.py +9 -0
- docpull-6.8.0/tests/test_release_registry_workflows.py +188 -0
- docpull-6.8.0/tests/test_research.py +163 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_time_utils.py +3 -3
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_warc.py +2 -2
- docpull-6.8.0/tests/test_web_search.py +214 -0
- docpull-6.5.5/src/docpull/benchmark.py +0 -4459
- docpull-6.5.5/src/docpull/context_packs/schema_extract.py +0 -595
- docpull-6.5.5/src/docpull/context_packs/search.py +0 -426
- docpull-6.5.5/src/docpull/discovery/contracts.py +0 -1816
- docpull-6.5.5/src/docpull/discovery_cli.py +0 -449
- docpull-6.5.5/src/docpull/evidence_pack.py +0 -1051
- docpull-6.5.5/src/docpull/fixtures/parallel-search-extract.json +0 -103
- docpull-6.5.5/src/docpull/free_core.py +0 -2193
- docpull-6.5.5/src/docpull/free_core_smoke.py +0 -474
- docpull-6.5.5/src/docpull/hosted.py +0 -720
- docpull-6.5.5/src/docpull/llm_client.py +0 -97
- docpull-6.5.5/src/docpull/native_integrations.py +0 -88
- docpull-6.5.5/src/docpull/parallel_workflows.py +0 -6022
- docpull-6.5.5/src/docpull/parity.py +0 -953
- docpull-6.5.5/src/docpull/parity_cli.py +0 -276
- docpull-6.5.5/src/docpull/passk.py +0 -200
- docpull-6.5.5/src/docpull/provider_adapters.py +0 -1409
- docpull-6.5.5/src/docpull/provider_capabilities.py +0 -190
- docpull-6.5.5/src/docpull/provider_cli.py +0 -1363
- docpull-6.5.5/src/docpull/provider_keys.py +0 -277
- docpull-6.5.5/src/docpull/provider_probes.py +0 -615
- docpull-6.5.5/src/docpull/scraper.py +0 -351
- docpull-6.5.5/tests/test_archive_discovery.py +0 -204
- docpull-6.5.5/tests/test_benchmark.py +0 -1369
- docpull-6.5.5/tests/test_discovery_contracts.py +0 -662
- docpull-6.5.5/tests/test_evidence_pack.py +0 -258
- docpull-6.5.5/tests/test_free_core.py +0 -929
- docpull-6.5.5/tests/test_hosted.py +0 -268
- docpull-6.5.5/tests/test_llm_extract.py +0 -221
- docpull-6.5.5/tests/test_native_integrations.py +0 -29
- docpull-6.5.5/tests/test_parallel_workflows.py +0 -2321
- docpull-6.5.5/tests/test_parity_workflows.py +0 -186
- docpull-6.5.5/tests/test_passk.py +0 -83
- docpull-6.5.5/tests/test_provider_adapters.py +0 -140
- docpull-6.5.5/tests/test_provider_cli.py +0 -467
- docpull-6.5.5/tests/test_provider_keys.py +0 -179
- docpull-6.5.5/tests/test_provider_probes.py +0 -216
- {docpull-6.5.5 → docpull-6.8.0}/LICENSE +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/setup.cfg +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/__main__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/acquisition_workflows.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/agent_publish.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/auth_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/basis.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/manager.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/streaming_dedup.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/change_events.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_aliases.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/brand.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/dataset.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/paper.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/policy_pack.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/product.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/relationship.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/repo.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/standards.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/styleguide.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/transcript.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/typed_models.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/visuals.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/website.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/wiki.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/workflow_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/contracts.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/contracts_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/article_cleanup.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/chunking.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/ensemble.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/extractor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/filings.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/markdown.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/protocols.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/special_cases.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/trafilatura_extractor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/core/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/composite.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/filters.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/enhanced.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/protocols.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/static.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/protocols.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/doctor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/document_worker.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/evidence.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/export_formats.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/exports.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/fixtures/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/protocols.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/_records.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/langchain.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/llamaindex.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/judge.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/sources.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/tools.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/metadata_extractor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/config.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/document.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/profiles.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/run.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/schema.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/manifest.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/chunk.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/dedup.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/fetch.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/metadata.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/render.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_json.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_ndjson.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_okf.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_sqlite.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/screen.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/validate.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/policy.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/policy_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/provenance.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/py.typed +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/redaction.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/rendering.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/artifact-manifest.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/basis.v2.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/change-event.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/citation-map.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/document.v3.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/intelligence-bundle.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/pack.v3.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/provenance.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/relationship-pack.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/rights.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/run-identity.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/website-snapshot.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/workflow-request.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/workflow-result.v1.schema.json +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/__init__.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/download_policy.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/injection.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/optout.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/robots.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/url_validator.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/share.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/skill_export.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull/source_scoring.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/dependency_links.txt +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/entry_points.txt +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/top_level.txt +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_accounting.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_acquisition_workflows.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_auth_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_basis.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_cache_conditional_get.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_chunking.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_claim_audit_script.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_context_ci.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_conversion.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_convert_step_new.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_discovery.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_doctor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_document_parse.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_document_record.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_exports.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_extractor_ensemble.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_feed_pack.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_frontier_resume.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_graph.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_http_efficiency.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_import_efficiency.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_injection_screen.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_integration.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_integrations.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_judge.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_link_extractors.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_live_web_smoke.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_local_pack_workflows.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_mcp_server.py +1 -1
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_mcp_tools.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_metadata_extractor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_monitor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_naming.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_openapi_pack.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_optional_extras_installed.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_optout.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_output_contract.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_pack_server.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_pipeline.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_policy.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_policy_cli.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_provenance.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_public_feature_smoke.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_qa_judge.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_rate_limiter.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_real_site_regressions.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_relationship_workflow.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_rendering.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_reproducible_release_build.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_save_ndjson.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_save_sqlite.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_security_hardening.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_share.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_source_scoring.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_special_cases.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_surface_contract.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_token_metrics.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_trafilatura_extractor.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_typed_knowledge_packs.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_website_snapshot.py +0 -0
- {docpull-6.5.5 → docpull-6.8.0}/tests/test_workflow_contracts.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: docpull
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.8.0
|
|
4
4
|
Summary: Declare, sync, diff, and lock context dependencies for AI agents
|
|
5
5
|
Author-email: Zachary Roth <support@raintree.technology>
|
|
6
6
|
Maintainer-email: Raintree Technology <support@raintree.technology>
|
|
@@ -30,27 +30,26 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
|
30
30
|
Classifier: Natural Language :: English
|
|
31
31
|
Classifier: Operating System :: OS Independent
|
|
32
32
|
Classifier: Programming Language :: Python :: 3
|
|
33
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
34
33
|
Classifier: Programming Language :: Python :: 3.11
|
|
35
34
|
Classifier: Programming Language :: Python :: 3.12
|
|
36
35
|
Classifier: Programming Language :: Python :: 3.13
|
|
37
36
|
Classifier: Programming Language :: Python :: 3.14
|
|
38
37
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
39
38
|
Classifier: Typing :: Typed
|
|
40
|
-
Requires-Python: >=3.
|
|
39
|
+
Requires-Python: >=3.11
|
|
41
40
|
Description-Content-Type: text/markdown
|
|
42
41
|
License-File: LICENSE
|
|
43
42
|
Requires-Dist: beautifulsoup4>=4.12.0
|
|
44
43
|
Requires-Dist: html2text>=2020.1.16
|
|
45
44
|
Requires-Dist: defusedxml>=0.7.1
|
|
46
45
|
Requires-Dist: extruct>=0.15.0
|
|
47
|
-
Requires-Dist: aiohttp>=3.14.
|
|
46
|
+
Requires-Dist: aiohttp>=3.14.4
|
|
48
47
|
Requires-Dist: idna>=3.15
|
|
49
48
|
Requires-Dist: regex>=2024.11.6
|
|
50
49
|
Requires-Dist: rich>=13.0.0
|
|
51
50
|
Requires-Dist: pyyaml>=6.0
|
|
52
51
|
Requires-Dist: pydantic>=2.0
|
|
53
|
-
Requires-Dist: urllib3>=2.
|
|
52
|
+
Requires-Dist: urllib3>=2.8.0
|
|
54
53
|
Provides-Extra: proxy
|
|
55
54
|
Requires-Dist: aiohttp-socks>=0.11.0; extra == "proxy"
|
|
56
55
|
Provides-Extra: normalize
|
|
@@ -61,9 +60,9 @@ Provides-Extra: tokens
|
|
|
61
60
|
Requires-Dist: tiktoken>=0.7.0; extra == "tokens"
|
|
62
61
|
Provides-Extra: mcp
|
|
63
62
|
Requires-Dist: cryptography>=50.0.0; extra == "mcp"
|
|
64
|
-
Requires-Dist: mcp<2,>=1.
|
|
63
|
+
Requires-Dist: mcp<2,>=1.30.0; extra == "mcp"
|
|
65
64
|
Requires-Dist: pydantic-settings>=2.14.2; extra == "mcp"
|
|
66
|
-
Requires-Dist: pyjwt>=2.
|
|
65
|
+
Requires-Dist: pyjwt>=2.15.1; extra == "mcp"
|
|
67
66
|
Requires-Dist: python-multipart>=0.0.27; extra == "mcp"
|
|
68
67
|
Requires-Dist: starlette>=1.3.1; extra == "mcp"
|
|
69
68
|
Provides-Extra: serve
|
|
@@ -71,17 +70,17 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
|
|
|
71
70
|
Provides-Extra: parquet
|
|
72
71
|
Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
|
|
73
72
|
Provides-Extra: markitdown
|
|
74
|
-
Requires-Dist: pypdf>=6.
|
|
73
|
+
Requires-Dist: pypdf>=6.21.1; extra == "markitdown"
|
|
75
74
|
Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
|
|
76
75
|
Provides-Extra: unstructured
|
|
77
|
-
Requires-Dist: pypdf>=6.
|
|
78
|
-
Requires-Dist: unstructured[all-docs]>=0.
|
|
76
|
+
Requires-Dist: pypdf>=6.21.1; extra == "unstructured"
|
|
77
|
+
Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "unstructured"
|
|
79
78
|
Provides-Extra: pdf
|
|
80
|
-
Requires-Dist: pypdf>=6.
|
|
79
|
+
Requires-Dist: pypdf>=6.21.1; extra == "pdf"
|
|
81
80
|
Provides-Extra: parse
|
|
82
|
-
Requires-Dist: pypdf>=6.
|
|
81
|
+
Requires-Dist: pypdf>=6.21.1; extra == "parse"
|
|
83
82
|
Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
|
|
84
|
-
Requires-Dist: unstructured[all-docs]>=0.
|
|
83
|
+
Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "parse"
|
|
85
84
|
Provides-Extra: presidio
|
|
86
85
|
Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
|
|
87
86
|
Provides-Extra: provenance
|
|
@@ -96,16 +95,16 @@ Requires-Dist: url-normalize>=1.4.0; extra == "all"
|
|
|
96
95
|
Requires-Dist: trafilatura>=1.12.0; extra == "all"
|
|
97
96
|
Requires-Dist: tiktoken>=0.7.0; extra == "all"
|
|
98
97
|
Requires-Dist: cryptography>=50.0.0; extra == "all"
|
|
99
|
-
Requires-Dist: mcp<2,>=1.
|
|
98
|
+
Requires-Dist: mcp<2,>=1.30.0; extra == "all"
|
|
100
99
|
Requires-Dist: pydantic-settings>=2.14.2; extra == "all"
|
|
101
|
-
Requires-Dist: pyjwt>=2.
|
|
100
|
+
Requires-Dist: pyjwt>=2.15.1; extra == "all"
|
|
102
101
|
Requires-Dist: python-multipart>=0.0.27; extra == "all"
|
|
103
102
|
Requires-Dist: starlette>=1.3.1; extra == "all"
|
|
104
103
|
Requires-Dist: uvicorn>=0.30.0; extra == "all"
|
|
105
104
|
Requires-Dist: pyarrow>=16.0.0; extra == "all"
|
|
106
|
-
Requires-Dist: pypdf>=6.
|
|
105
|
+
Requires-Dist: pypdf>=6.21.1; extra == "all"
|
|
107
106
|
Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
|
|
108
|
-
Requires-Dist: unstructured[all-docs]>=0.
|
|
107
|
+
Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "all"
|
|
109
108
|
Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
|
|
110
109
|
Requires-Dist: e2b>=2.0.0; extra == "all"
|
|
111
110
|
Provides-Extra: dev
|
|
@@ -138,7 +137,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
|
|
|
138
137
|
agents and retrieval pipelines. Use it when your application needs to know which
|
|
139
138
|
sources it used, whether they changed, and how to rebuild the same context later.
|
|
140
139
|
|
|
141
|
-
[](https://www.python.org/downloads/)
|
|
142
141
|
[](https://pypi.org/project/docpull/)
|
|
143
142
|
[](LICENSE)
|
|
144
143
|
|
|
@@ -151,6 +150,7 @@ pip install docpull
|
|
|
151
150
|
docpull init stripe-docs
|
|
152
151
|
docpull add https://docs.stripe.com
|
|
153
152
|
docpull sync
|
|
153
|
+
docpull query "retry policy" --max-tokens 4000
|
|
154
154
|
docpull diff
|
|
155
155
|
docpull export context-pack --target cursor
|
|
156
156
|
```
|
|
@@ -159,6 +159,11 @@ The project stores declared sources in `docpull.yaml` and resolved inputs in
|
|
|
159
159
|
`.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
|
|
160
160
|
source URLs, content hashes, run IDs, citations, and export metadata.
|
|
161
161
|
|
|
162
|
+
`docpull query` retrieves deduplicated passages from locked archives with version
|
|
163
|
+
filters, a response budget, and exact source spans. The same query is available
|
|
164
|
+
through the Python SDK and MCP. See [Query locked context](docs/context-dependencies.md#query-locked-context)
|
|
165
|
+
for budget semantics and queries across projects.
|
|
166
|
+
|
|
162
167
|

|
|
163
168
|
|
|
164
169
|
No account or paid API is required for this path. Direct fetching, discovery,
|
|
@@ -224,9 +229,7 @@ Claude Code can register the same local server:
|
|
|
224
229
|
claude mcp add --transport stdio docpull -- docpull mcp
|
|
225
230
|
```
|
|
226
231
|
|
|
227
|
-
The Python stdio server is the supported
|
|
228
|
-
documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
|
|
229
|
-
not part of the package contract.
|
|
232
|
+
The Python stdio server is the supported MCP server.
|
|
230
233
|
|
|
231
234
|
## Limits and security boundary
|
|
232
235
|
|
|
@@ -253,6 +256,8 @@ extending acquisition behavior.
|
|
|
253
256
|
- [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
|
|
254
257
|
- [Public contracts](docs/contracts.md) — Schemas and versioning rules.
|
|
255
258
|
- [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
|
|
259
|
+
- [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
|
|
260
|
+
- [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
|
|
256
261
|
- [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
|
|
257
262
|
- [Changelog](docs/CHANGELOG.md) — Release history.
|
|
258
263
|
|
|
@@ -272,8 +277,11 @@ release cycle.
|
|
|
272
277
|
See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
|
|
273
278
|
for current lifecycle and distribution links.
|
|
274
279
|
|
|
280
|
+
Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
|
|
281
|
+
spending, investments, assets, debts, and net worth in one current record.
|
|
282
|
+
|
|
275
283
|
## Project policies
|
|
276
284
|
|
|
277
|
-
[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
|
|
285
|
+
[Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
|
|
278
286
|
[Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
|
|
279
287
|
[MIT License](LICENSE)
|
|
@@ -12,7 +12,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
|
|
|
12
12
|
agents and retrieval pipelines. Use it when your application needs to know which
|
|
13
13
|
sources it used, whether they changed, and how to rebuild the same context later.
|
|
14
14
|
|
|
15
|
-
[](https://www.python.org/downloads/)
|
|
16
16
|
[](https://pypi.org/project/docpull/)
|
|
17
17
|
[](LICENSE)
|
|
18
18
|
|
|
@@ -25,6 +25,7 @@ pip install docpull
|
|
|
25
25
|
docpull init stripe-docs
|
|
26
26
|
docpull add https://docs.stripe.com
|
|
27
27
|
docpull sync
|
|
28
|
+
docpull query "retry policy" --max-tokens 4000
|
|
28
29
|
docpull diff
|
|
29
30
|
docpull export context-pack --target cursor
|
|
30
31
|
```
|
|
@@ -33,6 +34,11 @@ The project stores declared sources in `docpull.yaml` and resolved inputs in
|
|
|
33
34
|
`.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
|
|
34
35
|
source URLs, content hashes, run IDs, citations, and export metadata.
|
|
35
36
|
|
|
37
|
+
`docpull query` retrieves deduplicated passages from locked archives with version
|
|
38
|
+
filters, a response budget, and exact source spans. The same query is available
|
|
39
|
+
through the Python SDK and MCP. See [Query locked context](docs/context-dependencies.md#query-locked-context)
|
|
40
|
+
for budget semantics and queries across projects.
|
|
41
|
+
|
|
36
42
|

|
|
37
43
|
|
|
38
44
|
No account or paid API is required for this path. Direct fetching, discovery,
|
|
@@ -98,9 +104,7 @@ Claude Code can register the same local server:
|
|
|
98
104
|
claude mcp add --transport stdio docpull -- docpull mcp
|
|
99
105
|
```
|
|
100
106
|
|
|
101
|
-
The Python stdio server is the supported
|
|
102
|
-
documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
|
|
103
|
-
not part of the package contract.
|
|
107
|
+
The Python stdio server is the supported MCP server.
|
|
104
108
|
|
|
105
109
|
## Limits and security boundary
|
|
106
110
|
|
|
@@ -127,6 +131,8 @@ extending acquisition behavior.
|
|
|
127
131
|
- [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
|
|
128
132
|
- [Public contracts](docs/contracts.md) — Schemas and versioning rules.
|
|
129
133
|
- [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
|
|
134
|
+
- [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
|
|
135
|
+
- [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
|
|
130
136
|
- [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
|
|
131
137
|
- [Changelog](docs/CHANGELOG.md) — Release history.
|
|
132
138
|
|
|
@@ -146,8 +152,11 @@ release cycle.
|
|
|
146
152
|
See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
|
|
147
153
|
for current lifecycle and distribution links.
|
|
148
154
|
|
|
155
|
+
Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
|
|
156
|
+
spending, investments, assets, debts, and net worth in one current record.
|
|
157
|
+
|
|
149
158
|
## Project policies
|
|
150
159
|
|
|
151
|
-
[Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
|
|
160
|
+
[Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
|
|
152
161
|
[Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
|
|
153
162
|
[MIT License](LICENSE)
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "docpull"
|
|
7
|
-
version = "6.
|
|
7
|
+
version = "6.8.0"
|
|
8
8
|
dynamic = []
|
|
9
9
|
description = "Declare, sync, diff, and lock context dependencies for AI agents"
|
|
10
10
|
readme = {file = "README.md", content-type = "text/markdown"}
|
|
11
|
-
requires-python = ">=3.
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
12
|
license = "MIT"
|
|
13
13
|
license-files = ["LICENSE"]
|
|
14
14
|
authors = [
|
|
@@ -71,7 +71,6 @@ classifiers = [
|
|
|
71
71
|
|
|
72
72
|
# Programming Language
|
|
73
73
|
"Programming Language :: Python :: 3",
|
|
74
|
-
"Programming Language :: Python :: 3.10",
|
|
75
74
|
"Programming Language :: Python :: 3.11",
|
|
76
75
|
"Programming Language :: Python :: 3.12",
|
|
77
76
|
"Programming Language :: Python :: 3.13",
|
|
@@ -86,13 +85,13 @@ dependencies = [
|
|
|
86
85
|
"html2text>=2020.1.16",
|
|
87
86
|
"defusedxml>=0.7.1",
|
|
88
87
|
"extruct>=0.15.0",
|
|
89
|
-
"aiohttp>=3.14.
|
|
88
|
+
"aiohttp>=3.14.4", # Security floor for the 2026 request-parsing advisories
|
|
90
89
|
"idna>=3.15", # Security floor for transitive URL/IDNA handling; Renovate raises as needed
|
|
91
90
|
"regex>=2024.11.6",
|
|
92
91
|
"rich>=13.0.0",
|
|
93
92
|
"pyyaml>=6.0",
|
|
94
93
|
"pydantic>=2.0",
|
|
95
|
-
"urllib3>=2.
|
|
94
|
+
"urllib3>=2.8.0", # Security floor for transitive HTTP clients; Renovate raises as needed
|
|
96
95
|
]
|
|
97
96
|
|
|
98
97
|
[project.optional-dependencies]
|
|
@@ -110,9 +109,9 @@ tokens = [
|
|
|
110
109
|
]
|
|
111
110
|
mcp = [
|
|
112
111
|
"cryptography>=50.0.0",
|
|
113
|
-
"mcp>=1.
|
|
112
|
+
"mcp>=1.30.0,<2",
|
|
114
113
|
"pydantic-settings>=2.14.2",
|
|
115
|
-
"pyjwt>=2.
|
|
114
|
+
"pyjwt>=2.15.1",
|
|
116
115
|
"python-multipart>=0.0.27",
|
|
117
116
|
"starlette>=1.3.1",
|
|
118
117
|
]
|
|
@@ -123,20 +122,20 @@ parquet = [
|
|
|
123
122
|
"pyarrow>=16.0.0",
|
|
124
123
|
]
|
|
125
124
|
markitdown = [
|
|
126
|
-
"pypdf>=6.
|
|
125
|
+
"pypdf>=6.21.1",
|
|
127
126
|
"markitdown[docx,pdf]>=0.1.0",
|
|
128
127
|
]
|
|
129
128
|
unstructured = [
|
|
130
|
-
"pypdf>=6.
|
|
131
|
-
"unstructured[all-docs]>=0.
|
|
129
|
+
"pypdf>=6.21.1",
|
|
130
|
+
"unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
|
|
132
131
|
]
|
|
133
132
|
pdf = [
|
|
134
|
-
"pypdf>=6.
|
|
133
|
+
"pypdf>=6.21.1",
|
|
135
134
|
]
|
|
136
135
|
parse = [
|
|
137
|
-
"pypdf>=6.
|
|
136
|
+
"pypdf>=6.21.1",
|
|
138
137
|
"markitdown[docx,pdf]>=0.1.0",
|
|
139
|
-
"unstructured[all-docs]>=0.
|
|
138
|
+
"unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
|
|
140
139
|
]
|
|
141
140
|
presidio = [
|
|
142
141
|
"presidio-analyzer>=2.2.0",
|
|
@@ -156,16 +155,16 @@ all = [
|
|
|
156
155
|
"trafilatura>=1.12.0",
|
|
157
156
|
"tiktoken>=0.7.0",
|
|
158
157
|
"cryptography>=50.0.0",
|
|
159
|
-
"mcp>=1.
|
|
158
|
+
"mcp>=1.30.0,<2",
|
|
160
159
|
"pydantic-settings>=2.14.2",
|
|
161
|
-
"pyjwt>=2.
|
|
160
|
+
"pyjwt>=2.15.1",
|
|
162
161
|
"python-multipart>=0.0.27",
|
|
163
162
|
"starlette>=1.3.1",
|
|
164
163
|
"uvicorn>=0.30.0",
|
|
165
164
|
"pyarrow>=16.0.0",
|
|
166
|
-
"pypdf>=6.
|
|
165
|
+
"pypdf>=6.21.1",
|
|
167
166
|
"markitdown[docx,pdf]>=0.1.0",
|
|
168
|
-
"unstructured[all-docs]>=0.
|
|
167
|
+
"unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
|
|
169
168
|
"presidio-analyzer>=2.2.0",
|
|
170
169
|
"e2b>=2.0.0",
|
|
171
170
|
]
|
|
@@ -208,18 +207,27 @@ include = ["docpull*"]
|
|
|
208
207
|
docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"]
|
|
209
208
|
|
|
210
209
|
[tool.uv]
|
|
211
|
-
constraint-dependencies = [
|
|
210
|
+
constraint-dependencies = [
|
|
211
|
+
"h2>=4.4.1",
|
|
212
|
+
"multidict>=6.9.1",
|
|
213
|
+
"accelerate>=1.15.0",
|
|
214
|
+
"soupsieve>=2.9",
|
|
215
|
+
"virtualenv>=21.14.9",
|
|
216
|
+
]
|
|
212
217
|
|
|
213
218
|
[tool.ruff]
|
|
214
219
|
line-length = 110
|
|
215
|
-
target-version = "
|
|
220
|
+
target-version = "py311"
|
|
216
221
|
|
|
217
222
|
[tool.ruff.lint]
|
|
218
223
|
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM"]
|
|
219
|
-
ignore = [
|
|
224
|
+
ignore = [
|
|
225
|
+
"A003", # Allow 'type' and 'format' as field names in data models
|
|
226
|
+
"UP042", # StrEnum changes str() output for existing str, Enum classes
|
|
227
|
+
]
|
|
220
228
|
|
|
221
229
|
[tool.mypy]
|
|
222
|
-
python_version = "3.
|
|
230
|
+
python_version = "3.11"
|
|
223
231
|
warn_return_any = true
|
|
224
232
|
warn_unused_configs = true
|
|
225
233
|
disallow_untyped_defs = true
|
|
@@ -15,9 +15,11 @@ from typing import TYPE_CHECKING, Any
|
|
|
15
15
|
|
|
16
16
|
from .surface import PUBLIC_SDK_EXPORTS
|
|
17
17
|
|
|
18
|
-
__version__ = "6.
|
|
18
|
+
__version__ = "6.8.0"
|
|
19
19
|
|
|
20
20
|
_LAZY_EXPORTS: dict[str, tuple[str, str]] = {
|
|
21
|
+
"query_context": (".context_query", "query_context"),
|
|
22
|
+
"ContextQueryError": (".context_query", "ContextQueryError"),
|
|
21
23
|
**{
|
|
22
24
|
name: (".context_packs", name)
|
|
23
25
|
for name in (
|
|
@@ -207,6 +209,7 @@ if TYPE_CHECKING:
|
|
|
207
209
|
capture_screenshot_pack,
|
|
208
210
|
validate_website_snapshot_pack,
|
|
209
211
|
)
|
|
212
|
+
from .context_query import ContextQueryError, query_context
|
|
210
213
|
from .contracts import (
|
|
211
214
|
ArtifactManifest,
|
|
212
215
|
ChangeEvent,
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Shared JSON file and value helpers for internal modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
_SLUG_RE = re.compile(r"[^a-zA-Z0-9]+")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def read_json(path: Path, *, error: type[Exception], required: bool = True, default: Any = None) -> Any:
|
|
14
|
+
"""Load JSON from ``path``; a missing optional file returns ``default``."""
|
|
15
|
+
if not path.exists():
|
|
16
|
+
if required:
|
|
17
|
+
raise error(f"Missing required file: {path}")
|
|
18
|
+
return default
|
|
19
|
+
try:
|
|
20
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
21
|
+
except json.JSONDecodeError as err:
|
|
22
|
+
raise error(f"Invalid JSON in {path}: {err}") from err
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def write_json(path: Path, payload: Any, *, sort_keys: bool = False) -> None:
|
|
26
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
27
|
+
text = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=sort_keys)
|
|
28
|
+
path.write_text(text + "\n", encoding="utf-8")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def dict_value(value: Any) -> dict[str, Any]:
|
|
32
|
+
return value if isinstance(value, dict) else {}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def list_value(value: Any) -> list[Any]:
|
|
36
|
+
return value if isinstance(value, list) else []
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def string_list(value: Any) -> list[str]:
|
|
40
|
+
"""Return the non-blank, stripped items of a list; anything else is empty."""
|
|
41
|
+
if not isinstance(value, list):
|
|
42
|
+
return []
|
|
43
|
+
return [str(item).strip() for item in value if str(item).strip()]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def slugify(value: str, *, fallback: str) -> str:
|
|
47
|
+
slug = _SLUG_RE.sub("-", value).strip("-").lower()
|
|
48
|
+
return slug[:80].strip("-") or fallback
|
|
@@ -114,39 +114,6 @@ def parse_budget_value(value: str) -> float:
|
|
|
114
114
|
return parsed
|
|
115
115
|
|
|
116
116
|
|
|
117
|
-
def extract_budget_flags(argv: list[str] | None) -> tuple[list[str] | None, float | None, bool]:
|
|
118
|
-
"""Strip cross-command ``--budget`` and ``--explain-route`` flags.
|
|
119
|
-
|
|
120
|
-
This supports subcommands where adding a parent argparse option would force
|
|
121
|
-
users to place the flag before the subcommand.
|
|
122
|
-
"""
|
|
123
|
-
if argv is None:
|
|
124
|
-
return None, None, False
|
|
125
|
-
cleaned: list[str] = []
|
|
126
|
-
budget: float | None = None
|
|
127
|
-
explain_route = False
|
|
128
|
-
index = 0
|
|
129
|
-
while index < len(argv):
|
|
130
|
-
item = argv[index]
|
|
131
|
-
if item == "--explain-route":
|
|
132
|
-
explain_route = True
|
|
133
|
-
index += 1
|
|
134
|
-
continue
|
|
135
|
-
if item == "--budget":
|
|
136
|
-
if index + 1 >= len(argv):
|
|
137
|
-
raise ValueError("--budget requires a value")
|
|
138
|
-
budget = parse_budget_value(argv[index + 1])
|
|
139
|
-
index += 2
|
|
140
|
-
continue
|
|
141
|
-
if item.startswith("--budget="):
|
|
142
|
-
budget = parse_budget_value(item.split("=", 1)[1])
|
|
143
|
-
index += 1
|
|
144
|
-
continue
|
|
145
|
-
cleaned.append(item)
|
|
146
|
-
index += 1
|
|
147
|
-
return cleaned, budget, explain_route
|
|
148
|
-
|
|
149
|
-
|
|
150
117
|
def paid_action_blocked(
|
|
151
118
|
budget_limit_usd: float | None,
|
|
152
119
|
*,
|
|
@@ -89,6 +89,7 @@ class FrontierStore:
|
|
|
89
89
|
self.created_at: str | None = None
|
|
90
90
|
self.updated_at: str | None = None
|
|
91
91
|
self._journal_needs_separator = False
|
|
92
|
+
self._loaded_from_disk = False
|
|
92
93
|
self._load()
|
|
93
94
|
|
|
94
95
|
def _load(self) -> None:
|
|
@@ -100,6 +101,7 @@ class FrontierStore:
|
|
|
100
101
|
else:
|
|
101
102
|
self._load_snapshot(data)
|
|
102
103
|
self._replay_journal()
|
|
104
|
+
self._loaded_from_disk = bool(self.entries)
|
|
103
105
|
|
|
104
106
|
def _load_snapshot(self, data: object) -> None:
|
|
105
107
|
if not isinstance(data, dict) or data.get("schema_version") != FRONTIER_SCHEMA_VERSION:
|
|
@@ -197,7 +199,11 @@ class FrontierStore:
|
|
|
197
199
|
self.save()
|
|
198
200
|
|
|
199
201
|
def initialize(self, *, start_url: str, run_fingerprint: dict[str, object]) -> None:
|
|
200
|
-
|
|
202
|
+
stale = self.start_url != start_url or self.run_fingerprint != run_fingerprint
|
|
203
|
+
# A never-initialized store holding entries from this process (streaming workers
|
|
204
|
+
# finish pages before the first snapshot) keeps them; entries read from disk are
|
|
205
|
+
# from an unknown crawl and are dropped.
|
|
206
|
+
if stale and (self.start_url is not None or self._loaded_from_disk):
|
|
201
207
|
self.entries.clear()
|
|
202
208
|
self.created_at = utc_now_iso()
|
|
203
209
|
self.start_url = start_url
|
|
@@ -259,3 +265,4 @@ class FrontierStore:
|
|
|
259
265
|
self.created_at = None
|
|
260
266
|
self.updated_at = None
|
|
261
267
|
self._journal_needs_separator = False
|
|
268
|
+
self._loaded_from_disk = False
|
|
@@ -1623,6 +1623,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
1623
1623
|
from .project import run_install_cli
|
|
1624
1624
|
|
|
1625
1625
|
return run_install_cli(raw_argv[1:])
|
|
1626
|
+
if raw_argv and raw_argv[0] == "query":
|
|
1627
|
+
from .context_query import run_query_cli
|
|
1628
|
+
|
|
1629
|
+
return run_query_cli(raw_argv[1:])
|
|
1626
1630
|
if raw_argv and raw_argv[0] == "deps":
|
|
1627
1631
|
from .project import run_deps_cli
|
|
1628
1632
|
|
|
@@ -1683,6 +1687,22 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
1683
1687
|
from .graph import run_graph_cli
|
|
1684
1688
|
|
|
1685
1689
|
return run_graph_cli(raw_argv[1:])
|
|
1690
|
+
if raw_argv and raw_argv[0] == "registry":
|
|
1691
|
+
from .registry import run_registry_cli
|
|
1692
|
+
|
|
1693
|
+
return run_registry_cli(raw_argv[1:])
|
|
1694
|
+
if raw_argv and raw_argv[0] == "research":
|
|
1695
|
+
from .research import run_research_cli
|
|
1696
|
+
|
|
1697
|
+
return run_research_cli(raw_argv[1:])
|
|
1698
|
+
if raw_argv and raw_argv[0] == "index":
|
|
1699
|
+
from .discovery_index import run_index_cli
|
|
1700
|
+
|
|
1701
|
+
return run_index_cli(raw_argv[1:])
|
|
1702
|
+
if raw_argv and raw_argv[0] == "search":
|
|
1703
|
+
from .web_search import run_search_cli
|
|
1704
|
+
|
|
1705
|
+
return run_search_cli(raw_argv[1:])
|
|
1686
1706
|
if raw_argv and raw_argv[0] == "serve":
|
|
1687
1707
|
from .server import run_serve_cli
|
|
1688
1708
|
|