mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,784 @@
|
|
|
1
|
+
"""Read-only local search across separately sealed public and private indexes.
|
|
2
|
+
|
|
3
|
+
The two stores are captured and verified independently. Their results meet only as immutable
|
|
4
|
+
identities in this module; no combined index is written. Similarity orders candidates but grants
|
|
5
|
+
nothing: private reuse still requires the closed fact filters and one fresh exact-Build replay,
|
|
6
|
+
while public use still requires the source facts gate.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
|
|
18
|
+
from mostlyright.data_harness.local_search.contracts import (
|
|
19
|
+
BuildField,
|
|
20
|
+
BuildIndexEntry,
|
|
21
|
+
BuildRecipeLineage,
|
|
22
|
+
BuildSourceFacts,
|
|
23
|
+
BuildTimeCoverage,
|
|
24
|
+
LocalSearchContractError,
|
|
25
|
+
SealedProfileFactsV1,
|
|
26
|
+
)
|
|
27
|
+
from mostlyright.data_harness.local_search.query_trace import (
|
|
28
|
+
PrivateBuildQueryTrace,
|
|
29
|
+
build_private_query_trace,
|
|
30
|
+
)
|
|
31
|
+
from mostlyright.data_harness.local_search.retrieval import (
|
|
32
|
+
BuildFilters,
|
|
33
|
+
retrieve_build_candidates,
|
|
34
|
+
)
|
|
35
|
+
from mostlyright.data_harness.local_search.sealed import (
|
|
36
|
+
CapturedBuildIndexSnapshot,
|
|
37
|
+
capture_build_index_snapshot,
|
|
38
|
+
)
|
|
39
|
+
from mostlyright.data_harness.pipeline import (
|
|
40
|
+
RECIPE_MANIFEST_VERSION,
|
|
41
|
+
BuildError,
|
|
42
|
+
VerifiedSnapshot,
|
|
43
|
+
open_verified_snapshot,
|
|
44
|
+
)
|
|
45
|
+
from mostlyright.data_harness.sources.catalog.embedding import (
|
|
46
|
+
EmbeddingBackend,
|
|
47
|
+
LexicalHashingBackend,
|
|
48
|
+
resolve_local_neural_backend,
|
|
49
|
+
)
|
|
50
|
+
from mostlyright.data_harness.sources.catalog.gating import CatalogRequirement
|
|
51
|
+
from mostlyright.data_harness.sources.catalog.packed_retrieval import (
|
|
52
|
+
CatalogSearchWorkLimits,
|
|
53
|
+
detect_public_store_kind,
|
|
54
|
+
)
|
|
55
|
+
from mostlyright.data_harness.sources.catalog.query_trace import (
|
|
56
|
+
CatalogPackedQueryTrace,
|
|
57
|
+
CatalogQueryTraceV2,
|
|
58
|
+
PackedCatalogAnswer,
|
|
59
|
+
_recommend_loaded,
|
|
60
|
+
_trace_candidates,
|
|
61
|
+
query_packed_catalog_with_trace,
|
|
62
|
+
)
|
|
63
|
+
from mostlyright.data_harness.sources.catalog.retrieval import _rank_retrieval
|
|
64
|
+
from mostlyright.data_harness.sources.catalog.retrieval_manifest import (
|
|
65
|
+
LoadedRetrievalCatalog,
|
|
66
|
+
load_retrieval_catalog,
|
|
67
|
+
)
|
|
68
|
+
from mostlyright.data_harness.sources.contracts import SourceContractError
|
|
69
|
+
|
|
70
|
+
LOCAL_SEARCH_TRACE_VERSION = "harness-combined-local-search-trace.v1"
|
|
71
|
+
LOCAL_SEARCH_PAYLOAD_VERSION = "mr-data-local-search.v1"
|
|
72
|
+
|
|
73
|
+
#: ``public_incomplete`` exists so a public retrieval that ran out of its stated budget can never
|
|
74
|
+
#: be presented as an answer. Without it the only honest outcomes left would be ``no_match`` (the
|
|
75
|
+
#: catalog holds nothing like this -- a claim the search never established) or ``use_source`` from
|
|
76
|
+
#: whatever fraction of the ranges the budget happened to reach.
|
|
77
|
+
_RECOMMENDATION_KINDS = frozenset({"reuse_build", "use_source", "no_match", "public_incomplete"})
|
|
78
|
+
_PUBLIC_STORE_KINDS = frozenset({"packed", "retrieval_v2"})
|
|
79
|
+
_BUILD_DISPOSITIONS = frozenset({"not_selected", "verified", "rejected"})
|
|
80
|
+
_BUILD_DIAGNOSTICS = frozenset(
|
|
81
|
+
{"build_locator_invalid", "build_verification_failed", "build_identity_mismatch"}
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True)
|
|
86
|
+
class VerifiedBuildEvidence:
|
|
87
|
+
"""The three identities corroborated by one replay-verified selected Build snapshot."""
|
|
88
|
+
|
|
89
|
+
facts_sha256: str
|
|
90
|
+
candidate_sha256: str
|
|
91
|
+
table_sha256: str
|
|
92
|
+
|
|
93
|
+
def to_dict(self) -> dict[str, str]:
|
|
94
|
+
return {
|
|
95
|
+
"facts_sha256": self.facts_sha256,
|
|
96
|
+
"candidate_sha256": self.candidate_sha256,
|
|
97
|
+
"table_sha256": self.table_sha256,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True)
|
|
102
|
+
class BuildSelectionTrace:
|
|
103
|
+
"""Typed outcome of the sole fresh-Build verification opportunity."""
|
|
104
|
+
|
|
105
|
+
selected_facts_sha256: str | None
|
|
106
|
+
disposition: str
|
|
107
|
+
diagnostic: str | None
|
|
108
|
+
verified: VerifiedBuildEvidence | None
|
|
109
|
+
|
|
110
|
+
def __post_init__(self) -> None:
|
|
111
|
+
if self.disposition not in _BUILD_DISPOSITIONS:
|
|
112
|
+
raise LocalSearchContractError(
|
|
113
|
+
"LOCAL_SEARCH_SERVICE_DISPOSITION",
|
|
114
|
+
"trace.build_selection.disposition",
|
|
115
|
+
"must be not_selected, verified, or rejected",
|
|
116
|
+
)
|
|
117
|
+
if self.disposition == "not_selected":
|
|
118
|
+
valid = (
|
|
119
|
+
self.selected_facts_sha256 is None
|
|
120
|
+
and self.diagnostic is None
|
|
121
|
+
and self.verified is None
|
|
122
|
+
)
|
|
123
|
+
elif self.disposition == "verified":
|
|
124
|
+
valid = (
|
|
125
|
+
self.selected_facts_sha256 is not None
|
|
126
|
+
and self.diagnostic is None
|
|
127
|
+
and self.verified is not None
|
|
128
|
+
and self.verified.facts_sha256 == self.selected_facts_sha256
|
|
129
|
+
)
|
|
130
|
+
else:
|
|
131
|
+
valid = (
|
|
132
|
+
self.selected_facts_sha256 is not None
|
|
133
|
+
and self.diagnostic in _BUILD_DIAGNOSTICS
|
|
134
|
+
and self.verified is None
|
|
135
|
+
)
|
|
136
|
+
if not valid:
|
|
137
|
+
raise LocalSearchContractError(
|
|
138
|
+
"LOCAL_SEARCH_SERVICE_DISPOSITION",
|
|
139
|
+
"trace.build_selection",
|
|
140
|
+
"selection identities, diagnostic, and disposition disagree",
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
def to_dict(self) -> dict[str, Any]:
|
|
144
|
+
return {
|
|
145
|
+
"selected_facts_sha256": self.selected_facts_sha256,
|
|
146
|
+
"disposition": self.disposition,
|
|
147
|
+
"diagnostic": self.diagnostic,
|
|
148
|
+
"verified": None if self.verified is None else self.verified.to_dict(),
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@dataclass(frozen=True)
|
|
153
|
+
class CombinedLocalSearchTrace:
|
|
154
|
+
"""Canonical evidence for both independent queries and the final precedence decision."""
|
|
155
|
+
|
|
156
|
+
public_store_kind: str
|
|
157
|
+
public_manifest_sha256: str
|
|
158
|
+
private_manifest_sha256: str
|
|
159
|
+
public_backend_coordinate: str
|
|
160
|
+
private_backend_coordinate: str
|
|
161
|
+
question_digest: str
|
|
162
|
+
question_sha256: str
|
|
163
|
+
question_binding_sha256: str
|
|
164
|
+
filters_sha256: str
|
|
165
|
+
public: CatalogQueryTraceV2 | CatalogPackedQueryTrace
|
|
166
|
+
private: PrivateBuildQueryTrace
|
|
167
|
+
build_selection: BuildSelectionTrace
|
|
168
|
+
recommendation_kind: str
|
|
169
|
+
schema_version: str = LOCAL_SEARCH_TRACE_VERSION
|
|
170
|
+
|
|
171
|
+
def __post_init__(self) -> None:
|
|
172
|
+
if self.schema_version != LOCAL_SEARCH_TRACE_VERSION:
|
|
173
|
+
raise LocalSearchContractError(
|
|
174
|
+
"LOCAL_SEARCH_SERVICE_VERSION",
|
|
175
|
+
"trace.schema_version",
|
|
176
|
+
f"must equal {LOCAL_SEARCH_TRACE_VERSION!r}",
|
|
177
|
+
)
|
|
178
|
+
if self.public_store_kind not in _PUBLIC_STORE_KINDS:
|
|
179
|
+
raise LocalSearchContractError(
|
|
180
|
+
"LOCAL_SEARCH_SERVICE_STORE",
|
|
181
|
+
"trace.public_store_kind",
|
|
182
|
+
"must name the packed or the v2 public store",
|
|
183
|
+
)
|
|
184
|
+
packed = self.public_store_kind == "packed"
|
|
185
|
+
if packed is not isinstance(self.public, CatalogPackedQueryTrace):
|
|
186
|
+
raise LocalSearchContractError(
|
|
187
|
+
"LOCAL_SEARCH_SERVICE_STORE",
|
|
188
|
+
"trace.public",
|
|
189
|
+
"the public evidence must be the evidence of the store it names",
|
|
190
|
+
)
|
|
191
|
+
if self.recommendation_kind not in _RECOMMENDATION_KINDS:
|
|
192
|
+
raise LocalSearchContractError(
|
|
193
|
+
"LOCAL_SEARCH_SERVICE_RECOMMENDATION",
|
|
194
|
+
"trace.recommendation_kind",
|
|
195
|
+
"must be reuse_build, use_source, no_match, or public_incomplete",
|
|
196
|
+
)
|
|
197
|
+
bindings = (
|
|
198
|
+
self.public_manifest_sha256
|
|
199
|
+
== (self.public.head_sha256 if packed else self.public.retrieval_manifest_sha256),
|
|
200
|
+
self.private_manifest_sha256 == self.private.private_index_manifest_sha256,
|
|
201
|
+
self.public_backend_coordinate == self.public.backend_coordinate,
|
|
202
|
+
self.private_backend_coordinate == self.private.backend_coordinate,
|
|
203
|
+
self.question_digest == self.public.question_digest,
|
|
204
|
+
self.question_sha256 == self.private.question_sha256,
|
|
205
|
+
self.question_binding_sha256
|
|
206
|
+
== canonical_sha256(
|
|
207
|
+
{
|
|
208
|
+
"public_question_digest": self.public.question_digest,
|
|
209
|
+
"private_question_sha256": self.private.question_sha256,
|
|
210
|
+
}
|
|
211
|
+
),
|
|
212
|
+
self.filters_sha256 == self.private.filters_sha256,
|
|
213
|
+
self.build_selection.selected_facts_sha256 == self.private.selected_facts_sha256,
|
|
214
|
+
)
|
|
215
|
+
if not all(bindings):
|
|
216
|
+
raise LocalSearchContractError(
|
|
217
|
+
"LOCAL_SEARCH_SERVICE_BINDING",
|
|
218
|
+
"trace",
|
|
219
|
+
"combined trace disagrees with a retained query result",
|
|
220
|
+
)
|
|
221
|
+
if self.build_selection.disposition == "verified":
|
|
222
|
+
selected = tuple(
|
|
223
|
+
candidate
|
|
224
|
+
for candidate in self.private.candidates
|
|
225
|
+
if candidate.facts_sha256 == self.private.selected_facts_sha256
|
|
226
|
+
)
|
|
227
|
+
verified = self.build_selection.verified
|
|
228
|
+
if (
|
|
229
|
+
len(selected) != 1
|
|
230
|
+
or verified is None
|
|
231
|
+
or selected[0].candidate_sha256 != verified.candidate_sha256
|
|
232
|
+
or selected[0].table_sha256 != verified.table_sha256
|
|
233
|
+
or selected[0].facts_sha256 != verified.facts_sha256
|
|
234
|
+
):
|
|
235
|
+
raise LocalSearchContractError(
|
|
236
|
+
"LOCAL_SEARCH_SERVICE_BINDING",
|
|
237
|
+
"trace.build_selection.verified",
|
|
238
|
+
"verified Build identities must match the selected private candidate",
|
|
239
|
+
)
|
|
240
|
+
expected_kind = (
|
|
241
|
+
"reuse_build"
|
|
242
|
+
if self.build_selection.disposition == "verified"
|
|
243
|
+
else "public_incomplete"
|
|
244
|
+
if packed and self.public.status == "incomplete"
|
|
245
|
+
else "use_source"
|
|
246
|
+
if self.public.admitted
|
|
247
|
+
else "no_match"
|
|
248
|
+
)
|
|
249
|
+
if self.recommendation_kind != expected_kind:
|
|
250
|
+
raise LocalSearchContractError(
|
|
251
|
+
"LOCAL_SEARCH_SERVICE_PRECEDENCE",
|
|
252
|
+
"trace.recommendation_kind",
|
|
253
|
+
"must follow reuse_build, public_incomplete, use_source, no_match precedence",
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
@property
|
|
257
|
+
def digest(self) -> str:
|
|
258
|
+
return canonical_sha256(self.to_dict())
|
|
259
|
+
|
|
260
|
+
def to_dict(self) -> dict[str, Any]:
|
|
261
|
+
return {
|
|
262
|
+
"schema_version": self.schema_version,
|
|
263
|
+
"public_store_kind": self.public_store_kind,
|
|
264
|
+
"public_manifest_sha256": self.public_manifest_sha256,
|
|
265
|
+
"private_manifest_sha256": self.private_manifest_sha256,
|
|
266
|
+
"public_backend_coordinate": self.public_backend_coordinate,
|
|
267
|
+
"private_backend_coordinate": self.private_backend_coordinate,
|
|
268
|
+
"question_digest": self.question_digest,
|
|
269
|
+
"question_sha256": self.question_sha256,
|
|
270
|
+
"question_binding_sha256": self.question_binding_sha256,
|
|
271
|
+
"filters_sha256": self.filters_sha256,
|
|
272
|
+
"public": self.public.to_dict(),
|
|
273
|
+
"private": self.private.to_dict(),
|
|
274
|
+
"build_selection": self.build_selection.to_dict(),
|
|
275
|
+
"recommendation_kind": self.recommendation_kind,
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
@dataclass(frozen=True)
|
|
280
|
+
class CapturedLocalSearch:
|
|
281
|
+
"""One response plus the exact independent in-memory observations that produced it."""
|
|
282
|
+
|
|
283
|
+
payload: dict[str, Any]
|
|
284
|
+
trace: CombinedLocalSearchTrace
|
|
285
|
+
public_snapshot: LoadedRetrievalCatalog | PackedCatalogAnswer
|
|
286
|
+
private_snapshot: CapturedBuildIndexSnapshot
|
|
287
|
+
verified_build: VerifiedBuildEvidence | None
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def capture_local_search(
|
|
291
|
+
*,
|
|
292
|
+
public_index_root: Path,
|
|
293
|
+
expected_public_manifest_sha256: str,
|
|
294
|
+
private_index_root: Path,
|
|
295
|
+
expected_private_manifest_sha256: str,
|
|
296
|
+
build_root: Path,
|
|
297
|
+
question: str,
|
|
298
|
+
source_requirement: CatalogRequirement,
|
|
299
|
+
build_filters: BuildFilters,
|
|
300
|
+
public_backend: str,
|
|
301
|
+
allow_lexical_fallback: bool,
|
|
302
|
+
private_backend: str,
|
|
303
|
+
neural_model_root: Path | None = None,
|
|
304
|
+
public_packed_receipt: Mapping[str, Any] | None = None,
|
|
305
|
+
catalog_work_limits: CatalogSearchWorkLimits | None = None,
|
|
306
|
+
limit: int = 5,
|
|
307
|
+
) -> CapturedLocalSearch:
|
|
308
|
+
"""Return one deterministic recommendation from separately verified local stores.
|
|
309
|
+
|
|
310
|
+
The public store is dispatched by what is installed at its root. A range-local packed head is
|
|
311
|
+
answered by the bounded packed reader and *requires* both its publication receipt and a closed
|
|
312
|
+
work contract; a v2 retrieval manifest is answered by its sealed reader and accepts neither.
|
|
313
|
+
There is deliberately no default work contract: an omitted budget is a refusal, never an
|
|
314
|
+
unbounded search nobody asked for.
|
|
315
|
+
"""
|
|
316
|
+
|
|
317
|
+
if not isinstance(build_root, Path):
|
|
318
|
+
raise LocalSearchContractError("LOCAL_SEARCH_TYPE", "build_root", "must be a pathlib.Path")
|
|
319
|
+
store_kind = detect_public_store_kind(public_index_root)
|
|
320
|
+
packed = store_kind == "packed"
|
|
321
|
+
if packed and (public_packed_receipt is None or catalog_work_limits is None):
|
|
322
|
+
raise LocalSearchContractError(
|
|
323
|
+
"LOCAL_SEARCH_SERVICE_WORK_LIMITS",
|
|
324
|
+
"catalog_work_limits",
|
|
325
|
+
"a packed public store is searched only under its receipt and a stated work budget",
|
|
326
|
+
)
|
|
327
|
+
if not packed and (public_packed_receipt is not None or catalog_work_limits is not None):
|
|
328
|
+
raise LocalSearchContractError(
|
|
329
|
+
"LOCAL_SEARCH_SERVICE_WORK_LIMITS",
|
|
330
|
+
"catalog_work_limits",
|
|
331
|
+
"a v2 public store takes neither a packed receipt nor packed work limits",
|
|
332
|
+
)
|
|
333
|
+
if not packed and store_kind != "retrieval_v2":
|
|
334
|
+
raise LocalSearchContractError(
|
|
335
|
+
"LOCAL_SEARCH_SERVICE_STORE",
|
|
336
|
+
"public_index_root",
|
|
337
|
+
"combined local search answers from a packed head or a v2 retrieval manifest",
|
|
338
|
+
)
|
|
339
|
+
public_neural_backend, private_encoder = _resolve_backends(
|
|
340
|
+
public_backend=public_backend,
|
|
341
|
+
private_backend=private_backend,
|
|
342
|
+
neural_model_root=neural_model_root,
|
|
343
|
+
allow_lexical_fallback=allow_lexical_fallback,
|
|
344
|
+
)
|
|
345
|
+
if packed:
|
|
346
|
+
assert public_packed_receipt is not None and catalog_work_limits is not None
|
|
347
|
+
public_snapshot: LoadedRetrievalCatalog | PackedCatalogAnswer = (
|
|
348
|
+
query_packed_catalog_with_trace(
|
|
349
|
+
directory=public_index_root,
|
|
350
|
+
expected_head_sha256=expected_public_manifest_sha256,
|
|
351
|
+
receipt=public_packed_receipt,
|
|
352
|
+
backend=_packed_query_backend(
|
|
353
|
+
public_backend=public_backend,
|
|
354
|
+
neural=public_neural_backend,
|
|
355
|
+
allow_lexical_fallback=allow_lexical_fallback,
|
|
356
|
+
),
|
|
357
|
+
question=question,
|
|
358
|
+
requirement=source_requirement,
|
|
359
|
+
work_limits=catalog_work_limits,
|
|
360
|
+
limit=limit,
|
|
361
|
+
)
|
|
362
|
+
)
|
|
363
|
+
public_trace: CatalogQueryTraceV2 | CatalogPackedQueryTrace = public_snapshot.trace
|
|
364
|
+
public_recommendation = None
|
|
365
|
+
else:
|
|
366
|
+
public_snapshot = load_retrieval_catalog(
|
|
367
|
+
public_index_root,
|
|
368
|
+
expected_manifest_sha256=expected_public_manifest_sha256,
|
|
369
|
+
neural_backend=public_neural_backend,
|
|
370
|
+
)
|
|
371
|
+
public_recommendation, public_trace = _query_public_snapshot(
|
|
372
|
+
public_snapshot,
|
|
373
|
+
manifest_sha256=expected_public_manifest_sha256,
|
|
374
|
+
backend=public_backend,
|
|
375
|
+
allow_lexical_fallback=allow_lexical_fallback,
|
|
376
|
+
question=question,
|
|
377
|
+
requirement=source_requirement,
|
|
378
|
+
limit=limit,
|
|
379
|
+
)
|
|
380
|
+
private_snapshot = capture_build_index_snapshot(
|
|
381
|
+
private_index_root,
|
|
382
|
+
expected_manifest_sha256=expected_private_manifest_sha256,
|
|
383
|
+
)
|
|
384
|
+
private_retrieval = retrieve_build_candidates(
|
|
385
|
+
private_snapshot.entries,
|
|
386
|
+
filters=build_filters,
|
|
387
|
+
question=question,
|
|
388
|
+
backend=private_encoder,
|
|
389
|
+
limit=limit,
|
|
390
|
+
)
|
|
391
|
+
private_trace = build_private_query_trace(
|
|
392
|
+
private_index_manifest_sha256=expected_private_manifest_sha256,
|
|
393
|
+
entries=private_snapshot.entries,
|
|
394
|
+
retrieval=private_retrieval,
|
|
395
|
+
)
|
|
396
|
+
selection = _verify_selected_build(
|
|
397
|
+
private_snapshot,
|
|
398
|
+
selected_facts_sha256=private_trace.selected_facts_sha256,
|
|
399
|
+
build_root=build_root,
|
|
400
|
+
)
|
|
401
|
+
recommendation = (
|
|
402
|
+
_packed_recommendation(selection, answer=public_snapshot)
|
|
403
|
+
if packed
|
|
404
|
+
else _recommendation(
|
|
405
|
+
selection,
|
|
406
|
+
public_recommendation=public_recommendation,
|
|
407
|
+
public_snapshot=public_snapshot,
|
|
408
|
+
)
|
|
409
|
+
)
|
|
410
|
+
trace = CombinedLocalSearchTrace(
|
|
411
|
+
public_store_kind=store_kind,
|
|
412
|
+
public_manifest_sha256=expected_public_manifest_sha256,
|
|
413
|
+
private_manifest_sha256=expected_private_manifest_sha256,
|
|
414
|
+
public_backend_coordinate=public_trace.backend_coordinate,
|
|
415
|
+
private_backend_coordinate=private_trace.backend_coordinate,
|
|
416
|
+
question_digest=canonical_sha256({"question": question}),
|
|
417
|
+
question_sha256=sha256_bytes(question.encode("utf-8")),
|
|
418
|
+
question_binding_sha256=canonical_sha256(
|
|
419
|
+
{
|
|
420
|
+
"public_question_digest": public_trace.question_digest,
|
|
421
|
+
"private_question_sha256": private_trace.question_sha256,
|
|
422
|
+
}
|
|
423
|
+
),
|
|
424
|
+
filters_sha256=build_filters.digest,
|
|
425
|
+
public=public_trace,
|
|
426
|
+
private=private_trace,
|
|
427
|
+
build_selection=selection,
|
|
428
|
+
recommendation_kind=recommendation["kind"],
|
|
429
|
+
)
|
|
430
|
+
payload = {
|
|
431
|
+
"schema_version": LOCAL_SEARCH_PAYLOAD_VERSION,
|
|
432
|
+
"status": (
|
|
433
|
+
"local_search_incomplete"
|
|
434
|
+
if recommendation["kind"] == "public_incomplete"
|
|
435
|
+
else "local_search_complete"
|
|
436
|
+
),
|
|
437
|
+
"recommendation": recommendation,
|
|
438
|
+
"trace": trace.to_dict(),
|
|
439
|
+
}
|
|
440
|
+
return CapturedLocalSearch(
|
|
441
|
+
payload=payload,
|
|
442
|
+
trace=trace,
|
|
443
|
+
public_snapshot=public_snapshot,
|
|
444
|
+
private_snapshot=private_snapshot,
|
|
445
|
+
verified_build=selection.verified,
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _resolve_backends(
|
|
450
|
+
*,
|
|
451
|
+
public_backend: str,
|
|
452
|
+
private_backend: str,
|
|
453
|
+
neural_model_root: Path | None,
|
|
454
|
+
allow_lexical_fallback: bool,
|
|
455
|
+
) -> tuple[EmbeddingBackend | None, EmbeddingBackend]:
|
|
456
|
+
"""Resolve only built-in lexical or governed local-neural implementations."""
|
|
457
|
+
|
|
458
|
+
allowed = frozenset({"lexical", "neural"})
|
|
459
|
+
for name, value in (
|
|
460
|
+
("public_backend", public_backend),
|
|
461
|
+
("private_backend", private_backend),
|
|
462
|
+
):
|
|
463
|
+
if type(value) is not str or value not in allowed:
|
|
464
|
+
raise LocalSearchContractError(
|
|
465
|
+
"LOCAL_SEARCH_SERVICE_BACKEND",
|
|
466
|
+
name,
|
|
467
|
+
"must select the closed lexical or neural backend",
|
|
468
|
+
)
|
|
469
|
+
if type(allow_lexical_fallback) is not bool:
|
|
470
|
+
raise LocalSearchContractError(
|
|
471
|
+
"LOCAL_SEARCH_TYPE",
|
|
472
|
+
"allow_lexical_fallback",
|
|
473
|
+
"must be a boolean",
|
|
474
|
+
)
|
|
475
|
+
needs_neural = public_backend == "neural" or private_backend == "neural"
|
|
476
|
+
if not needs_neural:
|
|
477
|
+
if neural_model_root is not None:
|
|
478
|
+
raise LocalSearchContractError(
|
|
479
|
+
"LOCAL_SEARCH_SERVICE_BACKEND",
|
|
480
|
+
"neural_model_root",
|
|
481
|
+
"must be absent when neither search path selects neural",
|
|
482
|
+
)
|
|
483
|
+
return None, LexicalHashingBackend()
|
|
484
|
+
if neural_model_root is not None and not isinstance(neural_model_root, Path):
|
|
485
|
+
raise LocalSearchContractError(
|
|
486
|
+
"LOCAL_SEARCH_TYPE", "neural_model_root", "must be a pathlib.Path or null"
|
|
487
|
+
)
|
|
488
|
+
if (
|
|
489
|
+
public_backend == "neural"
|
|
490
|
+
and private_backend == "lexical"
|
|
491
|
+
and allow_lexical_fallback
|
|
492
|
+
and neural_model_root is None
|
|
493
|
+
):
|
|
494
|
+
return None, LexicalHashingBackend()
|
|
495
|
+
try:
|
|
496
|
+
neural = resolve_local_neural_backend(neural_model_root)
|
|
497
|
+
except SourceContractError as error:
|
|
498
|
+
public_only_fallback = (
|
|
499
|
+
public_backend == "neural" and private_backend == "lexical" and allow_lexical_fallback
|
|
500
|
+
)
|
|
501
|
+
typed_unavailable = error.code in {
|
|
502
|
+
"NEURAL_MODEL_ROOT_REQUIRED",
|
|
503
|
+
"NEURAL_MODEL_ROOT_UNAVAILABLE",
|
|
504
|
+
"NEURAL_RUNTIME_UNAVAILABLE",
|
|
505
|
+
"NEURAL_TUPLE_UNAVAILABLE",
|
|
506
|
+
}
|
|
507
|
+
if public_only_fallback and typed_unavailable:
|
|
508
|
+
return None, LexicalHashingBackend()
|
|
509
|
+
raise
|
|
510
|
+
return (
|
|
511
|
+
neural if public_backend == "neural" else None,
|
|
512
|
+
neural if private_backend == "neural" else LexicalHashingBackend(),
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _packed_query_backend(
|
|
517
|
+
*,
|
|
518
|
+
public_backend: str,
|
|
519
|
+
neural: EmbeddingBackend | None,
|
|
520
|
+
allow_lexical_fallback: bool,
|
|
521
|
+
) -> EmbeddingBackend:
|
|
522
|
+
"""Pick the one concrete encoder a packed search will run. Never discover one.
|
|
523
|
+
|
|
524
|
+
A packed generation publishes exactly which backend coordinates it has packs for, and the
|
|
525
|
+
reader refuses a coordinate it does not publish -- so this only has to answer which of the two
|
|
526
|
+
governed local encoders the caller selected, under the same explicit fallback rule the v2 path
|
|
527
|
+
uses.
|
|
528
|
+
"""
|
|
529
|
+
|
|
530
|
+
if public_backend == "lexical":
|
|
531
|
+
return LexicalHashingBackend()
|
|
532
|
+
if neural is not None:
|
|
533
|
+
return neural
|
|
534
|
+
if not allow_lexical_fallback:
|
|
535
|
+
raise LocalSearchContractError(
|
|
536
|
+
"LOCAL_SEARCH_SERVICE_BACKEND",
|
|
537
|
+
"public_backend",
|
|
538
|
+
"a neural packed search without an authorized fallback needs its resolved backend",
|
|
539
|
+
)
|
|
540
|
+
return LexicalHashingBackend()
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _packed_recommendation(
|
|
544
|
+
selection: BuildSelectionTrace, *, answer: PackedCatalogAnswer
|
|
545
|
+
) -> dict[str, Any]:
|
|
546
|
+
"""Precedence over a packed public store, with incompleteness surfaced rather than swallowed."""
|
|
547
|
+
|
|
548
|
+
if selection.verified is not None:
|
|
549
|
+
return {"kind": "reuse_build", **selection.verified.to_dict()}
|
|
550
|
+
if answer.status == "incomplete":
|
|
551
|
+
return {"kind": "public_incomplete", "incomplete_code": answer.incomplete_code}
|
|
552
|
+
if not answer.trace.admitted:
|
|
553
|
+
return {"kind": "no_match"}
|
|
554
|
+
coordinate = answer.trace.admitted[0]
|
|
555
|
+
entry = next(item for item in answer.entries if item.coordinate == coordinate)
|
|
556
|
+
traced = next(item for item in answer.trace.candidates if item.entry_coordinate == coordinate)
|
|
557
|
+
return {
|
|
558
|
+
"kind": "use_source",
|
|
559
|
+
"entry_coordinate": coordinate,
|
|
560
|
+
"entry_digest": entry.digest,
|
|
561
|
+
"semantic_facts_digest": traced.semantic_facts_digest,
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _query_public_snapshot(
|
|
566
|
+
snapshot: LoadedRetrievalCatalog,
|
|
567
|
+
*,
|
|
568
|
+
manifest_sha256: str,
|
|
569
|
+
backend: str,
|
|
570
|
+
allow_lexical_fallback: bool,
|
|
571
|
+
question: str,
|
|
572
|
+
requirement: CatalogRequirement,
|
|
573
|
+
limit: int,
|
|
574
|
+
):
|
|
575
|
+
entry_ids, representation, backend_coordinate, fallback_reason = _rank_retrieval(
|
|
576
|
+
snapshot,
|
|
577
|
+
question,
|
|
578
|
+
backend=backend,
|
|
579
|
+
allow_lexical_fallback=allow_lexical_fallback,
|
|
580
|
+
limit=limit,
|
|
581
|
+
)
|
|
582
|
+
recommendation = _recommend_loaded(
|
|
583
|
+
snapshot,
|
|
584
|
+
entry_ids,
|
|
585
|
+
question=question,
|
|
586
|
+
requirement=requirement,
|
|
587
|
+
)
|
|
588
|
+
representation_manifest = (
|
|
589
|
+
snapshot.manifest.neural if representation == "neural" else snapshot.manifest.lexical
|
|
590
|
+
)
|
|
591
|
+
if representation_manifest is None: # pragma: no cover - the ranker guards this state
|
|
592
|
+
raise LocalSearchContractError(
|
|
593
|
+
"LOCAL_SEARCH_SERVICE_BINDING",
|
|
594
|
+
"public.representation",
|
|
595
|
+
"selected representation is absent from the verified manifest",
|
|
596
|
+
)
|
|
597
|
+
by_coordinate = {entry.coordinate: entry for entry in snapshot.lexical.entries}
|
|
598
|
+
trace = CatalogQueryTraceV2(
|
|
599
|
+
retrieval_manifest_sha256=manifest_sha256,
|
|
600
|
+
representation=representation,
|
|
601
|
+
representation_artifact_sha256=representation_manifest.artifact_sha256,
|
|
602
|
+
backend_coordinate=backend_coordinate,
|
|
603
|
+
fallback_reason=fallback_reason,
|
|
604
|
+
requested_backend=backend,
|
|
605
|
+
lexical_fallback_allowed=allow_lexical_fallback,
|
|
606
|
+
question_digest=recommendation.question_digest,
|
|
607
|
+
requirement_digest=recommendation.requirement_digest,
|
|
608
|
+
candidates=_trace_candidates(recommendation, by_coordinate),
|
|
609
|
+
admitted=recommendation.admitted,
|
|
610
|
+
escalated=recommendation.escalated,
|
|
611
|
+
refused=recommendation.refused,
|
|
612
|
+
)
|
|
613
|
+
return recommendation, trace
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _verify_selected_build(
|
|
617
|
+
snapshot: CapturedBuildIndexSnapshot,
|
|
618
|
+
*,
|
|
619
|
+
selected_facts_sha256: str | None,
|
|
620
|
+
build_root: Path,
|
|
621
|
+
) -> BuildSelectionTrace:
|
|
622
|
+
if selected_facts_sha256 is None:
|
|
623
|
+
return BuildSelectionTrace(None, "not_selected", None, None)
|
|
624
|
+
indexed = next(
|
|
625
|
+
(entry for entry in snapshot.entries if entry.facts_sha256 == selected_facts_sha256),
|
|
626
|
+
None,
|
|
627
|
+
)
|
|
628
|
+
locations = tuple(
|
|
629
|
+
item.relative_path
|
|
630
|
+
for item in snapshot.locators.locators
|
|
631
|
+
if item.facts_sha256 == selected_facts_sha256
|
|
632
|
+
)
|
|
633
|
+
if indexed is None or len(locations) != 1:
|
|
634
|
+
return BuildSelectionTrace(selected_facts_sha256, "rejected", "build_locator_invalid", None)
|
|
635
|
+
try:
|
|
636
|
+
with open_verified_snapshot(build_root / locations[0]) as verified_snapshot:
|
|
637
|
+
observed = _entry_from_verified_snapshot(verified_snapshot)
|
|
638
|
+
verified_snapshot.validate()
|
|
639
|
+
except (BuildError, LocalSearchContractError, OSError, TypeError, ValueError):
|
|
640
|
+
return BuildSelectionTrace(
|
|
641
|
+
selected_facts_sha256, "rejected", "build_verification_failed", None
|
|
642
|
+
)
|
|
643
|
+
expected_identities = (
|
|
644
|
+
indexed.facts_sha256,
|
|
645
|
+
indexed.candidate_sha256,
|
|
646
|
+
indexed.table_sha256,
|
|
647
|
+
)
|
|
648
|
+
observed_identities = (
|
|
649
|
+
observed.facts_sha256,
|
|
650
|
+
observed.candidate_sha256,
|
|
651
|
+
observed.table_sha256,
|
|
652
|
+
)
|
|
653
|
+
if observed_identities != expected_identities:
|
|
654
|
+
return BuildSelectionTrace(
|
|
655
|
+
selected_facts_sha256, "rejected", "build_identity_mismatch", None
|
|
656
|
+
)
|
|
657
|
+
evidence = VerifiedBuildEvidence(*observed_identities)
|
|
658
|
+
return BuildSelectionTrace(selected_facts_sha256, "verified", None, evidence)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def _entry_from_verified_snapshot(snapshot: VerifiedSnapshot) -> BuildIndexEntry:
|
|
662
|
+
"""Re-derive index facts from the sole retained exact-Build verification."""
|
|
663
|
+
|
|
664
|
+
verified = snapshot.verified
|
|
665
|
+
inspection = snapshot.inspection()
|
|
666
|
+
profile_raw = snapshot.member_bytes("evidence/profile.json")
|
|
667
|
+
profile = _object(profile_raw)
|
|
668
|
+
lineage = _object(snapshot.member_bytes("evidence/lineage.json"))
|
|
669
|
+
source_values = _array(snapshot.member_bytes("evidence/sources.json"))
|
|
670
|
+
requirements_raw = _optional_member(snapshot, "requirements.json")
|
|
671
|
+
requirements = None if requirements_raw is None else _object(requirements_raw)
|
|
672
|
+
|
|
673
|
+
columns = tuple(profile["columns"])
|
|
674
|
+
types = profile["types"]
|
|
675
|
+
null_counts = profile["null_counts"]
|
|
676
|
+
sources = []
|
|
677
|
+
for value in source_values:
|
|
678
|
+
rights = value["rights"]
|
|
679
|
+
sources.append(
|
|
680
|
+
BuildSourceFacts(
|
|
681
|
+
source_id=value["source_id"],
|
|
682
|
+
content_sha256=value["sha256"],
|
|
683
|
+
rights_status=rights["status"],
|
|
684
|
+
rights_evidence_sha256=canonical_sha256(rights),
|
|
685
|
+
)
|
|
686
|
+
)
|
|
687
|
+
return BuildIndexEntry(
|
|
688
|
+
question=inspection.question,
|
|
689
|
+
question_sha256=sha256_bytes(inspection.question.encode("utf-8")),
|
|
690
|
+
requirements_sha256=(
|
|
691
|
+
verified.manifest["plan_sha256"]
|
|
692
|
+
if requirements_raw is None
|
|
693
|
+
else sha256_bytes(requirements_raw)
|
|
694
|
+
),
|
|
695
|
+
grain=inspection.grain,
|
|
696
|
+
time_coverage=_time_coverage(requirements),
|
|
697
|
+
fields=tuple(BuildField(name, str(types[name])) for name in columns),
|
|
698
|
+
sources=tuple(sorted(sources, key=lambda item: item.source_id)),
|
|
699
|
+
profile=SealedProfileFactsV1(
|
|
700
|
+
evidence_sha256=sha256_bytes(profile_raw),
|
|
701
|
+
row_count=profile["row_count"],
|
|
702
|
+
columns=columns,
|
|
703
|
+
logical_types=tuple(str(types[name]) for name in columns),
|
|
704
|
+
null_counts=tuple(null_counts[name] for name in columns),
|
|
705
|
+
),
|
|
706
|
+
recipe_lineage=_recipe_lineage(snapshot, verified.manifest_version),
|
|
707
|
+
candidate_sha256=verified.candidate_digest,
|
|
708
|
+
table_sha256=verified.table_sha256,
|
|
709
|
+
validation_policy_sha256=verified.validation_policy_digest,
|
|
710
|
+
lineage_sha256=canonical_sha256(lineage),
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _optional_member(snapshot: VerifiedSnapshot, path: str) -> bytes | None:
|
|
715
|
+
paths = {
|
|
716
|
+
member.get("path")
|
|
717
|
+
for member in snapshot.verified.manifest.get("members", ())
|
|
718
|
+
if isinstance(member, dict)
|
|
719
|
+
}
|
|
720
|
+
return snapshot.member_bytes(path) if path in paths else None
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _recipe_lineage(snapshot: VerifiedSnapshot, manifest_version: str) -> BuildRecipeLineage | None:
|
|
724
|
+
if manifest_version != RECIPE_MANIFEST_VERSION:
|
|
725
|
+
return None
|
|
726
|
+
recipe = _object(snapshot.member_bytes("recipe.json"))
|
|
727
|
+
execution = _object(snapshot.member_bytes("recipe-execution.json"))
|
|
728
|
+
return BuildRecipeLineage(
|
|
729
|
+
recipe_id=recipe["recipe_id"],
|
|
730
|
+
recipe_version=recipe["recipe_version"],
|
|
731
|
+
recipe_sha256=canonical_sha256(recipe),
|
|
732
|
+
predecessor_candidate_sha256=execution["predecessor_candidate_digest"],
|
|
733
|
+
)
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _time_coverage(requirements: dict[str, Any] | None) -> BuildTimeCoverage | None:
|
|
737
|
+
if requirements is None:
|
|
738
|
+
return None
|
|
739
|
+
value = requirements["time_range"]
|
|
740
|
+
return BuildTimeCoverage(value["start_inclusive"][:10], value["end_exclusive"][:10])
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _object(raw: bytes) -> dict[str, Any]:
|
|
744
|
+
value = json.loads(raw)
|
|
745
|
+
if not isinstance(value, dict):
|
|
746
|
+
raise ValueError("verified member must contain an object")
|
|
747
|
+
return value
|
|
748
|
+
|
|
749
|
+
|
|
750
|
+
def _array(raw: bytes) -> list[Any]:
|
|
751
|
+
value = json.loads(raw)
|
|
752
|
+
if not isinstance(value, list):
|
|
753
|
+
raise ValueError("verified member must contain an array")
|
|
754
|
+
return value
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def _recommendation(selection, *, public_recommendation, public_snapshot):
|
|
758
|
+
if selection.verified is not None:
|
|
759
|
+
return {"kind": "reuse_build", **selection.verified.to_dict()}
|
|
760
|
+
if not public_recommendation.admitted:
|
|
761
|
+
return {"kind": "no_match"}
|
|
762
|
+
coordinate = public_recommendation.admitted[0]
|
|
763
|
+
entry = next(item for item in public_snapshot.lexical.entries if item.coordinate == coordinate)
|
|
764
|
+
return {
|
|
765
|
+
"kind": "use_source",
|
|
766
|
+
"entry_coordinate": entry.coordinate,
|
|
767
|
+
"entry_digest": entry.digest,
|
|
768
|
+
"facts_digest": entry.facts_digest,
|
|
769
|
+
"publisher": entry.publisher,
|
|
770
|
+
"title": entry.title,
|
|
771
|
+
"data_formats": list(entry.data_formats),
|
|
772
|
+
"access_kind": entry.access_kind,
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
__all__ = [
|
|
777
|
+
"LOCAL_SEARCH_PAYLOAD_VERSION",
|
|
778
|
+
"LOCAL_SEARCH_TRACE_VERSION",
|
|
779
|
+
"BuildSelectionTrace",
|
|
780
|
+
"CapturedLocalSearch",
|
|
781
|
+
"CombinedLocalSearchTrace",
|
|
782
|
+
"VerifiedBuildEvidence",
|
|
783
|
+
"capture_local_search",
|
|
784
|
+
]
|