docpull 6.5.5__tar.gz → 6.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. {docpull-6.5.5/src/docpull.egg-info → docpull-6.8.0}/PKG-INFO +30 -22
  2. {docpull-6.5.5 → docpull-6.8.0}/README.md +14 -5
  3. {docpull-6.5.5 → docpull-6.8.0}/pyproject.toml +29 -21
  4. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/__init__.py +4 -1
  5. docpull-6.8.0/src/docpull/_jsonutil.py +48 -0
  6. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/accounting.py +0 -33
  7. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/frontier.py +8 -1
  8. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cli.py +20 -0
  9. docpull-6.8.0/src/docpull/compat_api.py +469 -0
  10. docpull-6.8.0/src/docpull/compat_jobs.py +355 -0
  11. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_ci.py +9 -19
  12. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/_legacy_cli.py +0 -71
  13. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/common.py +0 -20
  14. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/feed.py +9 -17
  15. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/openapi.py +4 -13
  16. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/package.py +1 -4
  17. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/typed.py +3 -28
  18. docpull-6.8.0/src/docpull/context_query.py +493 -0
  19. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/core/fetcher.py +63 -29
  20. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/__init__.py +1 -38
  21. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/_fetch.py +0 -8
  22. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/crawler.py +4 -1
  23. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/sitemap.py +6 -0
  24. docpull-6.8.0/src/docpull/discovery_index.py +356 -0
  25. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/document_parse.py +5 -3
  26. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/eval_grade.py +6 -117
  27. docpull-6.8.0/src/docpull/frozen.py +123 -0
  28. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/graph.py +1 -9
  29. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/client.py +5 -0
  30. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/rate_limiter.py +25 -7
  31. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/local_workflows.py +29 -164
  32. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/server.py +64 -0
  33. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/events.py +2 -2
  34. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/monitor.py +1 -1
  35. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/output_contract.py +3 -13
  36. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pack_reader.py +3 -23
  37. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pack_tools.py +22 -29
  38. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/base.py +11 -3
  39. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/convert.py +2 -2
  40. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/project.py +170 -313
  41. docpull-6.8.0/src/docpull/registry.py +326 -0
  42. docpull-6.8.0/src/docpull/research.py +315 -0
  43. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/server.py +85 -1
  44. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/surface.py +8 -2
  45. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/time_utils.py +4 -4
  46. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/warc.py +2 -2
  47. docpull-6.8.0/src/docpull/web_search.py +312 -0
  48. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/workflows.py +15 -35
  49. {docpull-6.5.5 → docpull-6.8.0/src/docpull.egg-info}/PKG-INFO +30 -22
  50. {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/SOURCES.txt +28 -37
  51. {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/requires.txt +20 -14
  52. docpull-6.8.0/tests/test_cache_lock_regressions.py +172 -0
  53. {docpull-6.5.5 → docpull-6.8.0}/tests/test_ci_policy.py +5 -5
  54. docpull-6.8.0/tests/test_compat_api.py +136 -0
  55. docpull-6.8.0/tests/test_compat_jobs.py +176 -0
  56. {docpull-6.5.5 → docpull-6.8.0}/tests/test_context_packs.py +0 -127
  57. docpull-6.8.0/tests/test_context_query.py +328 -0
  58. docpull-6.8.0/tests/test_crawl_regressions.py +165 -0
  59. docpull-6.8.0/tests/test_discovery_index.py +158 -0
  60. docpull-6.8.0/tests/test_fetch_lifecycle.py +236 -0
  61. docpull-6.8.0/tests/test_frozen.py +65 -0
  62. docpull-6.8.0/tests/test_jsonutil.py +35 -0
  63. {docpull-6.5.5 → docpull-6.8.0}/tests/test_local_workflows.py +3 -35
  64. docpull-6.8.0/tests/test_model_cache_lock.py +305 -0
  65. docpull-6.8.0/tests/test_model_crawl_politeness.py +334 -0
  66. docpull-6.8.0/tests/test_model_fetch_pipeline.py +300 -0
  67. docpull-6.8.0/tests/test_model_jobs_monitor.py +242 -0
  68. docpull-6.8.0/tests/test_monitor_fetch_failures.py +104 -0
  69. {docpull-6.5.5 → docpull-6.8.0}/tests/test_outputs_e2e.py +0 -104
  70. {docpull-6.5.5 → docpull-6.8.0}/tests/test_pack_tools.py +28 -81
  71. docpull-6.8.0/tests/test_parser_dependency_support.py +54 -0
  72. {docpull-6.5.5 → docpull-6.8.0}/tests/test_project.py +182 -34
  73. {docpull-6.5.5 → docpull-6.8.0}/tests/test_real_feature_smoke_script.py +41 -0
  74. docpull-6.8.0/tests/test_registry.py +185 -0
  75. {docpull-6.5.5 → docpull-6.8.0}/tests/test_release_a_plus_check_script.py +9 -0
  76. docpull-6.8.0/tests/test_release_registry_workflows.py +188 -0
  77. docpull-6.8.0/tests/test_research.py +163 -0
  78. {docpull-6.5.5 → docpull-6.8.0}/tests/test_time_utils.py +3 -3
  79. {docpull-6.5.5 → docpull-6.8.0}/tests/test_warc.py +2 -2
  80. docpull-6.8.0/tests/test_web_search.py +214 -0
  81. docpull-6.5.5/src/docpull/benchmark.py +0 -4459
  82. docpull-6.5.5/src/docpull/context_packs/schema_extract.py +0 -595
  83. docpull-6.5.5/src/docpull/context_packs/search.py +0 -426
  84. docpull-6.5.5/src/docpull/discovery/contracts.py +0 -1816
  85. docpull-6.5.5/src/docpull/discovery_cli.py +0 -449
  86. docpull-6.5.5/src/docpull/evidence_pack.py +0 -1051
  87. docpull-6.5.5/src/docpull/fixtures/parallel-search-extract.json +0 -103
  88. docpull-6.5.5/src/docpull/free_core.py +0 -2193
  89. docpull-6.5.5/src/docpull/free_core_smoke.py +0 -474
  90. docpull-6.5.5/src/docpull/hosted.py +0 -720
  91. docpull-6.5.5/src/docpull/llm_client.py +0 -97
  92. docpull-6.5.5/src/docpull/native_integrations.py +0 -88
  93. docpull-6.5.5/src/docpull/parallel_workflows.py +0 -6022
  94. docpull-6.5.5/src/docpull/parity.py +0 -953
  95. docpull-6.5.5/src/docpull/parity_cli.py +0 -276
  96. docpull-6.5.5/src/docpull/passk.py +0 -200
  97. docpull-6.5.5/src/docpull/provider_adapters.py +0 -1409
  98. docpull-6.5.5/src/docpull/provider_capabilities.py +0 -190
  99. docpull-6.5.5/src/docpull/provider_cli.py +0 -1363
  100. docpull-6.5.5/src/docpull/provider_keys.py +0 -277
  101. docpull-6.5.5/src/docpull/provider_probes.py +0 -615
  102. docpull-6.5.5/src/docpull/scraper.py +0 -351
  103. docpull-6.5.5/tests/test_archive_discovery.py +0 -204
  104. docpull-6.5.5/tests/test_benchmark.py +0 -1369
  105. docpull-6.5.5/tests/test_discovery_contracts.py +0 -662
  106. docpull-6.5.5/tests/test_evidence_pack.py +0 -258
  107. docpull-6.5.5/tests/test_free_core.py +0 -929
  108. docpull-6.5.5/tests/test_hosted.py +0 -268
  109. docpull-6.5.5/tests/test_llm_extract.py +0 -221
  110. docpull-6.5.5/tests/test_native_integrations.py +0 -29
  111. docpull-6.5.5/tests/test_parallel_workflows.py +0 -2321
  112. docpull-6.5.5/tests/test_parity_workflows.py +0 -186
  113. docpull-6.5.5/tests/test_passk.py +0 -83
  114. docpull-6.5.5/tests/test_provider_adapters.py +0 -140
  115. docpull-6.5.5/tests/test_provider_cli.py +0 -467
  116. docpull-6.5.5/tests/test_provider_keys.py +0 -179
  117. docpull-6.5.5/tests/test_provider_probes.py +0 -216
  118. {docpull-6.5.5 → docpull-6.8.0}/LICENSE +0 -0
  119. {docpull-6.5.5 → docpull-6.8.0}/setup.cfg +0 -0
  120. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/__main__.py +0 -0
  121. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/acquisition_workflows.py +0 -0
  122. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/agent_publish.py +0 -0
  123. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/auth_cli.py +0 -0
  124. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/basis.py +0 -0
  125. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/__init__.py +0 -0
  126. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/manager.py +0 -0
  127. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/cache/streaming_dedup.py +0 -0
  128. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/change_events.py +0 -0
  129. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_aliases.py +0 -0
  130. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/__init__.py +0 -0
  131. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/brand.py +0 -0
  132. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/cli.py +0 -0
  133. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/dataset.py +0 -0
  134. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/paper.py +0 -0
  135. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/policy_pack.py +0 -0
  136. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/product.py +0 -0
  137. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/relationship.py +0 -0
  138. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/repo.py +0 -0
  139. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/standards.py +0 -0
  140. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/styleguide.py +0 -0
  141. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/transcript.py +0 -0
  142. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/typed_models.py +0 -0
  143. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/visuals.py +0 -0
  144. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/website.py +0 -0
  145. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/wiki.py +0 -0
  146. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/context_packs/workflow_cli.py +0 -0
  147. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/contracts.py +0 -0
  148. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/contracts_cli.py +0 -0
  149. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/__init__.py +0 -0
  150. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/article_cleanup.py +0 -0
  151. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/chunking.py +0 -0
  152. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/ensemble.py +0 -0
  153. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/extractor.py +0 -0
  154. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/filings.py +0 -0
  155. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/markdown.py +0 -0
  156. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/protocols.py +0 -0
  157. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/special_cases.py +0 -0
  158. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/conversion/trafilatura_extractor.py +0 -0
  159. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/core/__init__.py +0 -0
  160. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/composite.py +0 -0
  161. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/filters.py +0 -0
  162. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/__init__.py +0 -0
  163. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/enhanced.py +0 -0
  164. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/protocols.py +0 -0
  165. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/link_extractors/static.py +0 -0
  166. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/discovery/protocols.py +0 -0
  167. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/doctor.py +0 -0
  168. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/document_worker.py +0 -0
  169. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/evidence.py +0 -0
  170. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/export_formats.py +0 -0
  171. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/exports.py +0 -0
  172. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/fixtures/__init__.py +0 -0
  173. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/__init__.py +0 -0
  174. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/http/protocols.py +0 -0
  175. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/__init__.py +0 -0
  176. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/_records.py +0 -0
  177. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/langchain.py +0 -0
  178. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/integrations/llamaindex.py +0 -0
  179. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/judge.py +0 -0
  180. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/__init__.py +0 -0
  181. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/sources.py +0 -0
  182. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/mcp/tools.py +0 -0
  183. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/metadata_extractor.py +0 -0
  184. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/__init__.py +0 -0
  185. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/config.py +0 -0
  186. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/document.py +0 -0
  187. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/profiles.py +0 -0
  188. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/run.py +0 -0
  189. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/models/schema.py +0 -0
  190. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/__init__.py +0 -0
  191. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/manifest.py +0 -0
  192. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/__init__.py +0 -0
  193. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/chunk.py +0 -0
  194. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/dedup.py +0 -0
  195. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/fetch.py +0 -0
  196. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/metadata.py +0 -0
  197. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/render.py +0 -0
  198. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save.py +0 -0
  199. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_json.py +0 -0
  200. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_ndjson.py +0 -0
  201. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_okf.py +0 -0
  202. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/save_sqlite.py +0 -0
  203. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/screen.py +0 -0
  204. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/pipeline/steps/validate.py +0 -0
  205. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/policy.py +0 -0
  206. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/policy_cli.py +0 -0
  207. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/provenance.py +0 -0
  208. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/py.typed +0 -0
  209. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/redaction.py +0 -0
  210. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/rendering.py +0 -0
  211. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/artifact-manifest.v1.schema.json +0 -0
  212. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/basis.v2.schema.json +0 -0
  213. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/change-event.v1.schema.json +0 -0
  214. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/citation-map.v1.schema.json +0 -0
  215. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/document.v3.schema.json +0 -0
  216. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/intelligence-bundle.v1.schema.json +0 -0
  217. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/pack.v3.schema.json +0 -0
  218. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/provenance.v1.schema.json +0 -0
  219. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/relationship-pack.v1.schema.json +0 -0
  220. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/rights.v1.schema.json +0 -0
  221. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/run-identity.v1.schema.json +0 -0
  222. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/website-snapshot.v1.schema.json +0 -0
  223. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/workflow-request.v1.schema.json +0 -0
  224. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/schemas/workflow-result.v1.schema.json +0 -0
  225. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/__init__.py +0 -0
  226. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/download_policy.py +0 -0
  227. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/injection.py +0 -0
  228. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/optout.py +0 -0
  229. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/robots.py +0 -0
  230. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/security/url_validator.py +0 -0
  231. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/share.py +0 -0
  232. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/skill_export.py +0 -0
  233. {docpull-6.5.5 → docpull-6.8.0}/src/docpull/source_scoring.py +0 -0
  234. {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/dependency_links.txt +0 -0
  235. {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/entry_points.txt +0 -0
  236. {docpull-6.5.5 → docpull-6.8.0}/src/docpull.egg-info/top_level.txt +0 -0
  237. {docpull-6.5.5 → docpull-6.8.0}/tests/test_accounting.py +0 -0
  238. {docpull-6.5.5 → docpull-6.8.0}/tests/test_acquisition_workflows.py +0 -0
  239. {docpull-6.5.5 → docpull-6.8.0}/tests/test_auth_cli.py +0 -0
  240. {docpull-6.5.5 → docpull-6.8.0}/tests/test_basis.py +0 -0
  241. {docpull-6.5.5 → docpull-6.8.0}/tests/test_cache_conditional_get.py +0 -0
  242. {docpull-6.5.5 → docpull-6.8.0}/tests/test_chunking.py +0 -0
  243. {docpull-6.5.5 → docpull-6.8.0}/tests/test_claim_audit_script.py +0 -0
  244. {docpull-6.5.5 → docpull-6.8.0}/tests/test_cli.py +0 -0
  245. {docpull-6.5.5 → docpull-6.8.0}/tests/test_context_ci.py +0 -0
  246. {docpull-6.5.5 → docpull-6.8.0}/tests/test_conversion.py +0 -0
  247. {docpull-6.5.5 → docpull-6.8.0}/tests/test_convert_step_new.py +0 -0
  248. {docpull-6.5.5 → docpull-6.8.0}/tests/test_discovery.py +0 -0
  249. {docpull-6.5.5 → docpull-6.8.0}/tests/test_doctor.py +0 -0
  250. {docpull-6.5.5 → docpull-6.8.0}/tests/test_document_parse.py +0 -0
  251. {docpull-6.5.5 → docpull-6.8.0}/tests/test_document_record.py +0 -0
  252. {docpull-6.5.5 → docpull-6.8.0}/tests/test_exports.py +0 -0
  253. {docpull-6.5.5 → docpull-6.8.0}/tests/test_extractor_ensemble.py +0 -0
  254. {docpull-6.5.5 → docpull-6.8.0}/tests/test_feed_pack.py +0 -0
  255. {docpull-6.5.5 → docpull-6.8.0}/tests/test_frontier_resume.py +0 -0
  256. {docpull-6.5.5 → docpull-6.8.0}/tests/test_graph.py +0 -0
  257. {docpull-6.5.5 → docpull-6.8.0}/tests/test_http_efficiency.py +0 -0
  258. {docpull-6.5.5 → docpull-6.8.0}/tests/test_import_efficiency.py +0 -0
  259. {docpull-6.5.5 → docpull-6.8.0}/tests/test_injection_screen.py +0 -0
  260. {docpull-6.5.5 → docpull-6.8.0}/tests/test_integration.py +0 -0
  261. {docpull-6.5.5 → docpull-6.8.0}/tests/test_integrations.py +0 -0
  262. {docpull-6.5.5 → docpull-6.8.0}/tests/test_judge.py +0 -0
  263. {docpull-6.5.5 → docpull-6.8.0}/tests/test_link_extractors.py +0 -0
  264. {docpull-6.5.5 → docpull-6.8.0}/tests/test_live_web_smoke.py +0 -0
  265. {docpull-6.5.5 → docpull-6.8.0}/tests/test_local_pack_workflows.py +0 -0
  266. {docpull-6.5.5 → docpull-6.8.0}/tests/test_mcp_server.py +1 -1
  267. {docpull-6.5.5 → docpull-6.8.0}/tests/test_mcp_tools.py +0 -0
  268. {docpull-6.5.5 → docpull-6.8.0}/tests/test_metadata_extractor.py +0 -0
  269. {docpull-6.5.5 → docpull-6.8.0}/tests/test_monitor.py +0 -0
  270. {docpull-6.5.5 → docpull-6.8.0}/tests/test_naming.py +0 -0
  271. {docpull-6.5.5 → docpull-6.8.0}/tests/test_openapi_pack.py +0 -0
  272. {docpull-6.5.5 → docpull-6.8.0}/tests/test_optional_extras_installed.py +0 -0
  273. {docpull-6.5.5 → docpull-6.8.0}/tests/test_optout.py +0 -0
  274. {docpull-6.5.5 → docpull-6.8.0}/tests/test_output_contract.py +0 -0
  275. {docpull-6.5.5 → docpull-6.8.0}/tests/test_pack_server.py +0 -0
  276. {docpull-6.5.5 → docpull-6.8.0}/tests/test_pipeline.py +0 -0
  277. {docpull-6.5.5 → docpull-6.8.0}/tests/test_policy.py +0 -0
  278. {docpull-6.5.5 → docpull-6.8.0}/tests/test_policy_cli.py +0 -0
  279. {docpull-6.5.5 → docpull-6.8.0}/tests/test_provenance.py +0 -0
  280. {docpull-6.5.5 → docpull-6.8.0}/tests/test_public_feature_smoke.py +0 -0
  281. {docpull-6.5.5 → docpull-6.8.0}/tests/test_qa_judge.py +0 -0
  282. {docpull-6.5.5 → docpull-6.8.0}/tests/test_rate_limiter.py +0 -0
  283. {docpull-6.5.5 → docpull-6.8.0}/tests/test_real_site_regressions.py +0 -0
  284. {docpull-6.5.5 → docpull-6.8.0}/tests/test_relationship_workflow.py +0 -0
  285. {docpull-6.5.5 → docpull-6.8.0}/tests/test_rendering.py +0 -0
  286. {docpull-6.5.5 → docpull-6.8.0}/tests/test_reproducible_release_build.py +0 -0
  287. {docpull-6.5.5 → docpull-6.8.0}/tests/test_save_ndjson.py +0 -0
  288. {docpull-6.5.5 → docpull-6.8.0}/tests/test_save_sqlite.py +0 -0
  289. {docpull-6.5.5 → docpull-6.8.0}/tests/test_security_hardening.py +0 -0
  290. {docpull-6.5.5 → docpull-6.8.0}/tests/test_share.py +0 -0
  291. {docpull-6.5.5 → docpull-6.8.0}/tests/test_source_scoring.py +0 -0
  292. {docpull-6.5.5 → docpull-6.8.0}/tests/test_special_cases.py +0 -0
  293. {docpull-6.5.5 → docpull-6.8.0}/tests/test_surface_contract.py +0 -0
  294. {docpull-6.5.5 → docpull-6.8.0}/tests/test_token_metrics.py +0 -0
  295. {docpull-6.5.5 → docpull-6.8.0}/tests/test_trafilatura_extractor.py +0 -0
  296. {docpull-6.5.5 → docpull-6.8.0}/tests/test_typed_knowledge_packs.py +0 -0
  297. {docpull-6.5.5 → docpull-6.8.0}/tests/test_website_snapshot.py +0 -0
  298. {docpull-6.5.5 → docpull-6.8.0}/tests/test_workflow_contracts.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: docpull
3
- Version: 6.5.5
3
+ Version: 6.8.0
4
4
  Summary: Declare, sync, diff, and lock context dependencies for AI agents
5
5
  Author-email: Zachary Roth <support@raintree.technology>
6
6
  Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -30,27 +30,26 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
30
30
  Classifier: Natural Language :: English
31
31
  Classifier: Operating System :: OS Independent
32
32
  Classifier: Programming Language :: Python :: 3
33
- Classifier: Programming Language :: Python :: 3.10
34
33
  Classifier: Programming Language :: Python :: 3.11
35
34
  Classifier: Programming Language :: Python :: 3.12
36
35
  Classifier: Programming Language :: Python :: 3.13
37
36
  Classifier: Programming Language :: Python :: 3.14
38
37
  Classifier: Programming Language :: Python :: 3 :: Only
39
38
  Classifier: Typing :: Typed
40
- Requires-Python: >=3.10
39
+ Requires-Python: >=3.11
41
40
  Description-Content-Type: text/markdown
42
41
  License-File: LICENSE
43
42
  Requires-Dist: beautifulsoup4>=4.12.0
44
43
  Requires-Dist: html2text>=2020.1.16
45
44
  Requires-Dist: defusedxml>=0.7.1
46
45
  Requires-Dist: extruct>=0.15.0
47
- Requires-Dist: aiohttp>=3.14.3
46
+ Requires-Dist: aiohttp>=3.14.4
48
47
  Requires-Dist: idna>=3.15
49
48
  Requires-Dist: regex>=2024.11.6
50
49
  Requires-Dist: rich>=13.0.0
51
50
  Requires-Dist: pyyaml>=6.0
52
51
  Requires-Dist: pydantic>=2.0
53
- Requires-Dist: urllib3>=2.7.0
52
+ Requires-Dist: urllib3>=2.8.0
54
53
  Provides-Extra: proxy
55
54
  Requires-Dist: aiohttp-socks>=0.11.0; extra == "proxy"
56
55
  Provides-Extra: normalize
@@ -61,9 +60,9 @@ Provides-Extra: tokens
61
60
  Requires-Dist: tiktoken>=0.7.0; extra == "tokens"
62
61
  Provides-Extra: mcp
63
62
  Requires-Dist: cryptography>=50.0.0; extra == "mcp"
64
- Requires-Dist: mcp<2,>=1.28.1; extra == "mcp"
63
+ Requires-Dist: mcp<2,>=1.30.0; extra == "mcp"
65
64
  Requires-Dist: pydantic-settings>=2.14.2; extra == "mcp"
66
- Requires-Dist: pyjwt>=2.13.0; extra == "mcp"
65
+ Requires-Dist: pyjwt>=2.15.1; extra == "mcp"
67
66
  Requires-Dist: python-multipart>=0.0.27; extra == "mcp"
68
67
  Requires-Dist: starlette>=1.3.1; extra == "mcp"
69
68
  Provides-Extra: serve
@@ -71,17 +70,17 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
71
70
  Provides-Extra: parquet
72
71
  Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
73
72
  Provides-Extra: markitdown
74
- Requires-Dist: pypdf>=6.15.0; extra == "markitdown"
73
+ Requires-Dist: pypdf>=6.21.1; extra == "markitdown"
75
74
  Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
76
75
  Provides-Extra: unstructured
77
- Requires-Dist: pypdf>=6.15.0; extra == "unstructured"
78
- Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "unstructured"
76
+ Requires-Dist: pypdf>=6.21.1; extra == "unstructured"
77
+ Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "unstructured"
79
78
  Provides-Extra: pdf
80
- Requires-Dist: pypdf>=6.15.0; extra == "pdf"
79
+ Requires-Dist: pypdf>=6.21.1; extra == "pdf"
81
80
  Provides-Extra: parse
82
- Requires-Dist: pypdf>=6.15.0; extra == "parse"
81
+ Requires-Dist: pypdf>=6.21.1; extra == "parse"
83
82
  Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
84
- Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
83
+ Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "parse"
85
84
  Provides-Extra: presidio
86
85
  Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
87
86
  Provides-Extra: provenance
@@ -96,16 +95,16 @@ Requires-Dist: url-normalize>=1.4.0; extra == "all"
96
95
  Requires-Dist: trafilatura>=1.12.0; extra == "all"
97
96
  Requires-Dist: tiktoken>=0.7.0; extra == "all"
98
97
  Requires-Dist: cryptography>=50.0.0; extra == "all"
99
- Requires-Dist: mcp<2,>=1.28.1; extra == "all"
98
+ Requires-Dist: mcp<2,>=1.30.0; extra == "all"
100
99
  Requires-Dist: pydantic-settings>=2.14.2; extra == "all"
101
- Requires-Dist: pyjwt>=2.13.0; extra == "all"
100
+ Requires-Dist: pyjwt>=2.15.1; extra == "all"
102
101
  Requires-Dist: python-multipart>=0.0.27; extra == "all"
103
102
  Requires-Dist: starlette>=1.3.1; extra == "all"
104
103
  Requires-Dist: uvicorn>=0.30.0; extra == "all"
105
104
  Requires-Dist: pyarrow>=16.0.0; extra == "all"
106
- Requires-Dist: pypdf>=6.15.0; extra == "all"
105
+ Requires-Dist: pypdf>=6.21.1; extra == "all"
107
106
  Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
108
- Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "all"
107
+ Requires-Dist: unstructured[all-docs]>=0.24.1; (python_version >= "3.11" and python_version < "3.14") and extra == "all"
109
108
  Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
110
109
  Requires-Dist: e2b>=2.0.0; extra == "all"
111
110
  Provides-Extra: dev
@@ -138,7 +137,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
138
137
  agents and retrieval pipelines. Use it when your application needs to know which
139
138
  sources it used, whether they changed, and how to rebuild the same context later.
140
139
 
141
- [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
140
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
142
141
  [![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
143
142
  [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
144
143
 
@@ -151,6 +150,7 @@ pip install docpull
151
150
  docpull init stripe-docs
152
151
  docpull add https://docs.stripe.com
153
152
  docpull sync
153
+ docpull query "retry policy" --max-tokens 4000
154
154
  docpull diff
155
155
  docpull export context-pack --target cursor
156
156
  ```
@@ -159,6 +159,11 @@ The project stores declared sources in `docpull.yaml` and resolved inputs in
159
159
  `.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
160
160
  source URLs, content hashes, run IDs, citations, and export metadata.
161
161
 
162
+ `docpull query` retrieves deduplicated passages from locked archives with version
163
+ filters, a response budget, and exact source spans. The same query is available
164
+ through the Python SDK and MCP. See [Query locked context](docs/context-dependencies.md#query-locked-context)
165
+ for budget semantics and queries across projects.
166
+
162
167
  ![DocPull project diff showing changed pages, local semantic categories, and zero failed URLs](docs/launch-assets/docpull-project-diff-demo.png)
163
168
 
164
169
  No account or paid API is required for this path. Direct fetching, discovery,
@@ -224,9 +229,7 @@ Claude Code can register the same local server:
224
229
  claude mcp add --transport stdio docpull -- docpull mcp
225
230
  ```
226
231
 
227
- The Python stdio server is the supported release path. The TypeScript code formerly
228
- documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
229
- not part of the package contract.
232
+ The Python stdio server is the supported MCP server.
230
233
 
231
234
  ## Limits and security boundary
232
235
 
@@ -253,6 +256,8 @@ extending acquisition behavior.
253
256
  - [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
254
257
  - [Public contracts](docs/contracts.md) — Schemas and versioning rules.
255
258
  - [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
259
+ - [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
260
+ - [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
256
261
  - [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
257
262
  - [Changelog](docs/CHANGELOG.md) — Release history.
258
263
 
@@ -272,8 +277,11 @@ release cycle.
272
277
  See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
273
278
  for current lifecycle and distribution links.
274
279
 
280
+ Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
281
+ spending, investments, assets, debts, and net worth in one current record.
282
+
275
283
  ## Project policies
276
284
 
277
- [Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
285
+ [Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
278
286
  [Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
279
287
  [MIT License](LICENSE)
@@ -12,7 +12,7 @@ DocPull turns changing public web sources into cited, reproducible context for A
12
12
  agents and retrieval pipelines. Use it when your application needs to know which
13
13
  sources it used, whether they changed, and how to rebuild the same context later.
14
14
 
15
- [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
15
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
16
16
  [![PyPI version](https://img.shields.io/pypi/v/docpull.svg?label=package)](https://pypi.org/project/docpull/)
17
17
  [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](LICENSE)
18
18
 
@@ -25,6 +25,7 @@ pip install docpull
25
25
  docpull init stripe-docs
26
26
  docpull add https://docs.stripe.com
27
27
  docpull sync
28
+ docpull query "retry policy" --max-tokens 4000
28
29
  docpull diff
29
30
  docpull export context-pack --target cursor
30
31
  ```
@@ -33,6 +34,11 @@ The project stores declared sources in `docpull.yaml` and resolved inputs in
33
34
  `.docpull/context.lock.json`. Later syncs produce a hash-based diff while preserving
34
35
  source URLs, content hashes, run IDs, citations, and export metadata.
35
36
 
37
+ `docpull query` retrieves deduplicated passages from locked archives with version
38
+ filters, a response budget, and exact source spans. The same query is available
39
+ through the Python SDK and MCP. See [Query locked context](docs/context-dependencies.md#query-locked-context)
40
+ for budget semantics and queries across projects.
41
+
36
42
  ![DocPull project diff showing changed pages, local semantic categories, and zero failed URLs](docs/launch-assets/docpull-project-diff-demo.png)
37
43
 
38
44
  No account or paid API is required for this path. Direct fetching, discovery,
@@ -98,9 +104,7 @@ Claude Code can register the same local server:
98
104
  claude mcp add --transport stdio docpull -- docpull mcp
99
105
  ```
100
106
 
101
- The Python stdio server is the supported release path. The TypeScript code formerly
102
- documented under `mcp/` is an [internal semantic-search lab](docs/internal-mcp-lab.md),
103
- not part of the package contract.
107
+ The Python stdio server is the supported MCP server.
104
108
 
105
109
  ## Limits and security boundary
106
110
 
@@ -127,6 +131,8 @@ extending acquisition behavior.
127
131
  - [Context Pack Contract v3](docs/context-pack-contract-v3.md) — Artifact levels and compatibility.
128
132
  - [Public contracts](docs/contracts.md) — Schemas and versioning rules.
129
133
  - [Alternatives](docs/alternatives.md) — Browser automation and hosted extraction tradeoffs.
134
+ - [JavaScript SDK and loaders](docs/sdk-js.md) — TypeScript client and LangChain/LlamaIndex loaders.
135
+ - [Dependency security](docs/dependency-security.md) — Locked dependency sets and optional parser support.
130
136
  - [Evaluation lab](bench/docs/index.md) — Reproducible methods, results, and claim boundaries.
131
137
  - [Changelog](docs/CHANGELOG.md) — Release history.
132
138
 
@@ -146,8 +152,11 @@ release cycle.
146
152
  See the [Raintree open-source portfolio](https://raintree.technology/portfolio#open-source)
147
153
  for current lifecycle and distribution links.
148
154
 
155
+ Raintree also builds [BetterOff](https://betteroff.finance), which keeps a household's
156
+ spending, investments, assets, debts, and net worth in one current record.
157
+
149
158
  ## Project policies
150
159
 
151
- [Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) ·
160
+ [Contributing](.github/CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](.github/SECURITY.md) ·
152
161
  [Metrics and evidence limits](METRICS.md) · [Source repository](https://github.com/raintree-technology/docpull) ·
153
162
  [MIT License](LICENSE)
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "docpull"
7
- version = "6.5.5"
7
+ version = "6.8.0"
8
8
  dynamic = []
9
9
  description = "Declare, sync, diff, and lock context dependencies for AI agents"
10
10
  readme = {file = "README.md", content-type = "text/markdown"}
11
- requires-python = ">=3.10"
11
+ requires-python = ">=3.11"
12
12
  license = "MIT"
13
13
  license-files = ["LICENSE"]
14
14
  authors = [
@@ -71,7 +71,6 @@ classifiers = [
71
71
 
72
72
  # Programming Language
73
73
  "Programming Language :: Python :: 3",
74
- "Programming Language :: Python :: 3.10",
75
74
  "Programming Language :: Python :: 3.11",
76
75
  "Programming Language :: Python :: 3.12",
77
76
  "Programming Language :: Python :: 3.13",
@@ -86,13 +85,13 @@ dependencies = [
86
85
  "html2text>=2020.1.16",
87
86
  "defusedxml>=0.7.1",
88
87
  "extruct>=0.15.0",
89
- "aiohttp>=3.14.3", # Security floor for the 2026 request-parsing advisories
88
+ "aiohttp>=3.14.4", # Security floor for the 2026 request-parsing advisories
90
89
  "idna>=3.15", # Security floor for transitive URL/IDNA handling; Renovate raises as needed
91
90
  "regex>=2024.11.6",
92
91
  "rich>=13.0.0",
93
92
  "pyyaml>=6.0",
94
93
  "pydantic>=2.0",
95
- "urllib3>=2.7.0", # Security floor for transitive HTTP clients; Renovate raises as needed
94
+ "urllib3>=2.8.0", # Security floor for transitive HTTP clients; Renovate raises as needed
96
95
  ]
97
96
 
98
97
  [project.optional-dependencies]
@@ -110,9 +109,9 @@ tokens = [
110
109
  ]
111
110
  mcp = [
112
111
  "cryptography>=50.0.0",
113
- "mcp>=1.28.1,<2",
112
+ "mcp>=1.30.0,<2",
114
113
  "pydantic-settings>=2.14.2",
115
- "pyjwt>=2.13.0",
114
+ "pyjwt>=2.15.1",
116
115
  "python-multipart>=0.0.27",
117
116
  "starlette>=1.3.1",
118
117
  ]
@@ -123,20 +122,20 @@ parquet = [
123
122
  "pyarrow>=16.0.0",
124
123
  ]
125
124
  markitdown = [
126
- "pypdf>=6.15.0",
125
+ "pypdf>=6.21.1",
127
126
  "markitdown[docx,pdf]>=0.1.0",
128
127
  ]
129
128
  unstructured = [
130
- "pypdf>=6.15.0",
131
- "unstructured[all-docs]>=0.16.0",
129
+ "pypdf>=6.21.1",
130
+ "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
132
131
  ]
133
132
  pdf = [
134
- "pypdf>=6.15.0",
133
+ "pypdf>=6.21.1",
135
134
  ]
136
135
  parse = [
137
- "pypdf>=6.15.0",
136
+ "pypdf>=6.21.1",
138
137
  "markitdown[docx,pdf]>=0.1.0",
139
- "unstructured[all-docs]>=0.16.0",
138
+ "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
140
139
  ]
141
140
  presidio = [
142
141
  "presidio-analyzer>=2.2.0",
@@ -156,16 +155,16 @@ all = [
156
155
  "trafilatura>=1.12.0",
157
156
  "tiktoken>=0.7.0",
158
157
  "cryptography>=50.0.0",
159
- "mcp>=1.28.1,<2",
158
+ "mcp>=1.30.0,<2",
160
159
  "pydantic-settings>=2.14.2",
161
- "pyjwt>=2.13.0",
160
+ "pyjwt>=2.15.1",
162
161
  "python-multipart>=0.0.27",
163
162
  "starlette>=1.3.1",
164
163
  "uvicorn>=0.30.0",
165
164
  "pyarrow>=16.0.0",
166
- "pypdf>=6.15.0",
165
+ "pypdf>=6.21.1",
167
166
  "markitdown[docx,pdf]>=0.1.0",
168
- "unstructured[all-docs]>=0.16.0",
167
+ "unstructured[all-docs]>=0.24.1; python_version >= '3.11' and python_version < '3.14'",
169
168
  "presidio-analyzer>=2.2.0",
170
169
  "e2b>=2.0.0",
171
170
  ]
@@ -208,18 +207,27 @@ include = ["docpull*"]
208
207
  docpull = ["py.typed", "fixtures/*.json", "schemas/*.schema.json"]
209
208
 
210
209
  [tool.uv]
211
- constraint-dependencies = ["h2>=4.4.1"]
210
+ constraint-dependencies = [
211
+ "h2>=4.4.1",
212
+ "multidict>=6.9.1",
213
+ "accelerate>=1.15.0",
214
+ "soupsieve>=2.9",
215
+ "virtualenv>=21.14.9",
216
+ ]
212
217
 
213
218
  [tool.ruff]
214
219
  line-length = 110
215
- target-version = "py310"
220
+ target-version = "py311"
216
221
 
217
222
  [tool.ruff.lint]
218
223
  select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "SIM"]
219
- ignore = ["A003"] # Allow 'type' and 'format' as field names in data models
224
+ ignore = [
225
+ "A003", # Allow 'type' and 'format' as field names in data models
226
+ "UP042", # StrEnum changes str() output for existing str, Enum classes
227
+ ]
220
228
 
221
229
  [tool.mypy]
222
- python_version = "3.10"
230
+ python_version = "3.11"
223
231
  warn_return_any = true
224
232
  warn_unused_configs = true
225
233
  disallow_untyped_defs = true
@@ -15,9 +15,11 @@ from typing import TYPE_CHECKING, Any
15
15
 
16
16
  from .surface import PUBLIC_SDK_EXPORTS
17
17
 
18
- __version__ = "6.5.5"
18
+ __version__ = "6.8.0"
19
19
 
20
20
  _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
21
+ "query_context": (".context_query", "query_context"),
22
+ "ContextQueryError": (".context_query", "ContextQueryError"),
21
23
  **{
22
24
  name: (".context_packs", name)
23
25
  for name in (
@@ -207,6 +209,7 @@ if TYPE_CHECKING:
207
209
  capture_screenshot_pack,
208
210
  validate_website_snapshot_pack,
209
211
  )
212
+ from .context_query import ContextQueryError, query_context
210
213
  from .contracts import (
211
214
  ArtifactManifest,
212
215
  ChangeEvent,
@@ -0,0 +1,48 @@
1
+ """Shared JSON file and value helpers for internal modules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ _SLUG_RE = re.compile(r"[^a-zA-Z0-9]+")
11
+
12
+
13
+ def read_json(path: Path, *, error: type[Exception], required: bool = True, default: Any = None) -> Any:
14
+ """Load JSON from ``path``; a missing optional file returns ``default``."""
15
+ if not path.exists():
16
+ if required:
17
+ raise error(f"Missing required file: {path}")
18
+ return default
19
+ try:
20
+ return json.loads(path.read_text(encoding="utf-8"))
21
+ except json.JSONDecodeError as err:
22
+ raise error(f"Invalid JSON in {path}: {err}") from err
23
+
24
+
25
+ def write_json(path: Path, payload: Any, *, sort_keys: bool = False) -> None:
26
+ path.parent.mkdir(parents=True, exist_ok=True)
27
+ text = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=sort_keys)
28
+ path.write_text(text + "\n", encoding="utf-8")
29
+
30
+
31
+ def dict_value(value: Any) -> dict[str, Any]:
32
+ return value if isinstance(value, dict) else {}
33
+
34
+
35
+ def list_value(value: Any) -> list[Any]:
36
+ return value if isinstance(value, list) else []
37
+
38
+
39
+ def string_list(value: Any) -> list[str]:
40
+ """Return the non-blank, stripped items of a list; anything else is empty."""
41
+ if not isinstance(value, list):
42
+ return []
43
+ return [str(item).strip() for item in value if str(item).strip()]
44
+
45
+
46
+ def slugify(value: str, *, fallback: str) -> str:
47
+ slug = _SLUG_RE.sub("-", value).strip("-").lower()
48
+ return slug[:80].strip("-") or fallback
@@ -114,39 +114,6 @@ def parse_budget_value(value: str) -> float:
114
114
  return parsed
115
115
 
116
116
 
117
- def extract_budget_flags(argv: list[str] | None) -> tuple[list[str] | None, float | None, bool]:
118
- """Strip cross-command ``--budget`` and ``--explain-route`` flags.
119
-
120
- This supports subcommands where adding a parent argparse option would force
121
- users to place the flag before the subcommand.
122
- """
123
- if argv is None:
124
- return None, None, False
125
- cleaned: list[str] = []
126
- budget: float | None = None
127
- explain_route = False
128
- index = 0
129
- while index < len(argv):
130
- item = argv[index]
131
- if item == "--explain-route":
132
- explain_route = True
133
- index += 1
134
- continue
135
- if item == "--budget":
136
- if index + 1 >= len(argv):
137
- raise ValueError("--budget requires a value")
138
- budget = parse_budget_value(argv[index + 1])
139
- index += 2
140
- continue
141
- if item.startswith("--budget="):
142
- budget = parse_budget_value(item.split("=", 1)[1])
143
- index += 1
144
- continue
145
- cleaned.append(item)
146
- index += 1
147
- return cleaned, budget, explain_route
148
-
149
-
150
117
  def paid_action_blocked(
151
118
  budget_limit_usd: float | None,
152
119
  *,
@@ -89,6 +89,7 @@ class FrontierStore:
89
89
  self.created_at: str | None = None
90
90
  self.updated_at: str | None = None
91
91
  self._journal_needs_separator = False
92
+ self._loaded_from_disk = False
92
93
  self._load()
93
94
 
94
95
  def _load(self) -> None:
@@ -100,6 +101,7 @@ class FrontierStore:
100
101
  else:
101
102
  self._load_snapshot(data)
102
103
  self._replay_journal()
104
+ self._loaded_from_disk = bool(self.entries)
103
105
 
104
106
  def _load_snapshot(self, data: object) -> None:
105
107
  if not isinstance(data, dict) or data.get("schema_version") != FRONTIER_SCHEMA_VERSION:
@@ -197,7 +199,11 @@ class FrontierStore:
197
199
  self.save()
198
200
 
199
201
  def initialize(self, *, start_url: str, run_fingerprint: dict[str, object]) -> None:
200
- if self.start_url != start_url or self.run_fingerprint != run_fingerprint:
202
+ stale = self.start_url != start_url or self.run_fingerprint != run_fingerprint
203
+ # A never-initialized store holding entries from this process (streaming workers
204
+ # finish pages before the first snapshot) keeps them; entries read from disk are
205
+ # from an unknown crawl and are dropped.
206
+ if stale and (self.start_url is not None or self._loaded_from_disk):
201
207
  self.entries.clear()
202
208
  self.created_at = utc_now_iso()
203
209
  self.start_url = start_url
@@ -259,3 +265,4 @@ class FrontierStore:
259
265
  self.created_at = None
260
266
  self.updated_at = None
261
267
  self._journal_needs_separator = False
268
+ self._loaded_from_disk = False
@@ -1623,6 +1623,10 @@ def main(argv: list[str] | None = None) -> int:
1623
1623
  from .project import run_install_cli
1624
1624
 
1625
1625
  return run_install_cli(raw_argv[1:])
1626
+ if raw_argv and raw_argv[0] == "query":
1627
+ from .context_query import run_query_cli
1628
+
1629
+ return run_query_cli(raw_argv[1:])
1626
1630
  if raw_argv and raw_argv[0] == "deps":
1627
1631
  from .project import run_deps_cli
1628
1632
 
@@ -1683,6 +1687,22 @@ def main(argv: list[str] | None = None) -> int:
1683
1687
  from .graph import run_graph_cli
1684
1688
 
1685
1689
  return run_graph_cli(raw_argv[1:])
1690
+ if raw_argv and raw_argv[0] == "registry":
1691
+ from .registry import run_registry_cli
1692
+
1693
+ return run_registry_cli(raw_argv[1:])
1694
+ if raw_argv and raw_argv[0] == "research":
1695
+ from .research import run_research_cli
1696
+
1697
+ return run_research_cli(raw_argv[1:])
1698
+ if raw_argv and raw_argv[0] == "index":
1699
+ from .discovery_index import run_index_cli
1700
+
1701
+ return run_index_cli(raw_argv[1:])
1702
+ if raw_argv and raw_argv[0] == "search":
1703
+ from .web_search import run_search_cli
1704
+
1705
+ return run_search_cli(raw_argv[1:])
1686
1706
  if raw_argv and raw_argv[0] == "serve":
1687
1707
  from .server import run_serve_cli
1688
1708