mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,2049 @@
|
|
|
1
|
+
"""Durable local coordinator for the H2 offline vertical slice."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import errno
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import stat
|
|
9
|
+
import uuid
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from copy import deepcopy
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import TYPE_CHECKING, Any
|
|
15
|
+
|
|
16
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
import fcntl
|
|
20
|
+
except ImportError: # pragma: no cover - H2 already fails closed on Windows
|
|
21
|
+
fcntl = None # type: ignore[assignment]
|
|
22
|
+
|
|
23
|
+
from mostlyright.data_harness.canonical import (
|
|
24
|
+
CanonicalJSONError,
|
|
25
|
+
canonical_json_bytes,
|
|
26
|
+
parse_canonical_json,
|
|
27
|
+
parse_json,
|
|
28
|
+
sha256_bytes,
|
|
29
|
+
)
|
|
30
|
+
from mostlyright.data_harness.contracts import ContractError, TablePlan, parse_plan
|
|
31
|
+
from mostlyright.data_harness.local_contracts import (
|
|
32
|
+
GraphTablePlan,
|
|
33
|
+
LocalQuestion,
|
|
34
|
+
LocalRequirements,
|
|
35
|
+
SourceProposal,
|
|
36
|
+
TableSemantics,
|
|
37
|
+
parse_question,
|
|
38
|
+
parse_requirements,
|
|
39
|
+
parse_source_proposal,
|
|
40
|
+
parse_table_semantics,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# The underscore-prefixed names below are private to pipeline.py by convention only; this
|
|
44
|
+
# module imports and calls each of them directly, so their signatures have to change
|
|
45
|
+
# together with the call sites here.
|
|
46
|
+
from mostlyright.data_harness.pipeline import (
|
|
47
|
+
OFFLINE_MANIFEST_VERSION,
|
|
48
|
+
OFFLINE_SEMANTICS_MANIFEST_VERSION,
|
|
49
|
+
SOURCE_PROPOSAL_SET_VERSION,
|
|
50
|
+
BuildError,
|
|
51
|
+
CandidateInspection,
|
|
52
|
+
OfflinePackageContext,
|
|
53
|
+
ResourceLimits,
|
|
54
|
+
_durability_barrier,
|
|
55
|
+
_entry_identity,
|
|
56
|
+
_open_candidate_run_handle,
|
|
57
|
+
_open_output_parent,
|
|
58
|
+
_open_source_root,
|
|
59
|
+
_read_confined_source_from_handle,
|
|
60
|
+
_remove_tree,
|
|
61
|
+
_rename_no_replace,
|
|
62
|
+
_validate_candidate_run_handle,
|
|
63
|
+
_validate_output_parent_handle,
|
|
64
|
+
_validate_root_separation,
|
|
65
|
+
build_candidate,
|
|
66
|
+
classify_plan_sources_before_persistence,
|
|
67
|
+
inspect_candidate,
|
|
68
|
+
)
|
|
69
|
+
from mostlyright.data_harness.review import (
|
|
70
|
+
ReviewResult,
|
|
71
|
+
_finalize_review,
|
|
72
|
+
_review_authority_material,
|
|
73
|
+
_verify_review_decision,
|
|
74
|
+
)
|
|
75
|
+
from mostlyright.data_harness.signing import (
|
|
76
|
+
TrustedReviewAuthorityProvider,
|
|
77
|
+
TrustedSigner,
|
|
78
|
+
)
|
|
79
|
+
from mostlyright.data_harness.ux.path_kind import UNKNOWN_KIND, name_of_kind, presence_at
|
|
80
|
+
from mostlyright.data_harness.ux.plain_file import (
|
|
81
|
+
DANGLING,
|
|
82
|
+
NOT_PLAIN,
|
|
83
|
+
TOO_LARGE,
|
|
84
|
+
PlainFileRefusal,
|
|
85
|
+
read_plain_file,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if TYPE_CHECKING:
|
|
89
|
+
from mostlyright.data_harness.visual_run.store import VisualRunStore
|
|
90
|
+
|
|
91
|
+
_OFFLINE_MANIFEST_VERSIONS = frozenset(
|
|
92
|
+
{OFFLINE_MANIFEST_VERSION, OFFLINE_SEMANTICS_MANIFEST_VERSION}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
RUN_STATE_VERSION = "offline-run-state.v1"
|
|
96
|
+
RUN_RESULT_PATH = "result"
|
|
97
|
+
INPUT_ROOT_PATH = "inputs"
|
|
98
|
+
DEFINITION_PATH = "definitions"
|
|
99
|
+
CONTROL_PATH = ".mr-data"
|
|
100
|
+
WORKSPACE_LOCK_NAME = ".lock"
|
|
101
|
+
# Retain the old private spelling for callers that reached into this module before the workspace
|
|
102
|
+
# control contract had a public lock name. New code must use ``WORKSPACE_LOCK_NAME``.
|
|
103
|
+
_LOCK_NAME = WORKSPACE_LOCK_NAME
|
|
104
|
+
_STATE_NAME = "state.json"
|
|
105
|
+
_DEFINITION_FILES = {
|
|
106
|
+
"question": "question.json",
|
|
107
|
+
"requirements": "requirements.json",
|
|
108
|
+
"source_proposals": "source-proposals.json",
|
|
109
|
+
"plan": "plan.json",
|
|
110
|
+
}
|
|
111
|
+
_SEMANTICS_DEFINITION = {"semantics": "semantics.json"}
|
|
112
|
+
_TASK_IDS = ("plan", "build", "review")
|
|
113
|
+
_PHASES = frozenset(
|
|
114
|
+
{
|
|
115
|
+
"initialized",
|
|
116
|
+
"planned",
|
|
117
|
+
"building",
|
|
118
|
+
"candidate_built",
|
|
119
|
+
"review_recorded",
|
|
120
|
+
"failed",
|
|
121
|
+
}
|
|
122
|
+
)
|
|
123
|
+
_TASK_STATUSES = frozenset({"pending", "running", "completed", "failed"})
|
|
124
|
+
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
|
|
125
|
+
_DIGEST = re.compile(r"^[0-9a-f]{64}$")
|
|
126
|
+
_ORPHAN_STAGING = re.compile(r"^\.result\.tmp-[0-9a-f]{32}$")
|
|
127
|
+
_MAX_STATE_BYTES = 8 * 1024 * 1024
|
|
128
|
+
_MAX_DEFINITION_BYTES = 4 * 1024 * 1024
|
|
129
|
+
_H2_PRODUCER_PRINCIPAL = "h2offlinecoordinatorproducer"
|
|
130
|
+
_H2_PRODUCER_ATTEMPT = "h2offlineinitialattempt"
|
|
131
|
+
_PHASE_TASK_STATUSES = {
|
|
132
|
+
"initialized": ("pending", "pending", "pending"),
|
|
133
|
+
"planned": ("completed", "pending", "pending"),
|
|
134
|
+
"building": ("completed", "running", "pending"),
|
|
135
|
+
"candidate_built": ("completed", "completed", "pending"),
|
|
136
|
+
"review_recorded": ("completed", "completed", "completed"),
|
|
137
|
+
"failed": ("completed", "failed", "pending"),
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _h2_producer_identity() -> dict[str, str]:
|
|
142
|
+
return {
|
|
143
|
+
"principal_id": _H2_PRODUCER_PRINCIPAL,
|
|
144
|
+
"attempt_id": _H2_PRODUCER_ATTEMPT,
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class OfflineRunError(RuntimeError):
|
|
149
|
+
"""A local workspace failed a closed coordinator invariant."""
|
|
150
|
+
|
|
151
|
+
def __init__(self, code: str, detail: str) -> None:
|
|
152
|
+
self.code = code
|
|
153
|
+
self.detail = detail
|
|
154
|
+
super().__init__(detail)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass(frozen=True)
|
|
158
|
+
class OfflineInputs:
|
|
159
|
+
question: LocalQuestion
|
|
160
|
+
requirements: LocalRequirements
|
|
161
|
+
source_proposals: tuple[SourceProposal, ...]
|
|
162
|
+
plan: TablePlan
|
|
163
|
+
semantics: TableSemantics | None = None
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def package_context(self) -> OfflinePackageContext:
|
|
167
|
+
return OfflinePackageContext(
|
|
168
|
+
question=self.question,
|
|
169
|
+
requirements=self.requirements,
|
|
170
|
+
source_proposals=self.source_proposals,
|
|
171
|
+
semantics=self.semantics,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
def definition_values(self) -> dict[str, Any]:
|
|
175
|
+
values = {
|
|
176
|
+
"question": self.question.to_dict(),
|
|
177
|
+
"requirements": self.requirements.to_dict(),
|
|
178
|
+
"source_proposals": {
|
|
179
|
+
"schema_version": SOURCE_PROPOSAL_SET_VERSION,
|
|
180
|
+
"source_proposals": [
|
|
181
|
+
item.to_dict()
|
|
182
|
+
for item in sorted(self.source_proposals, key=lambda item: item.source_id)
|
|
183
|
+
],
|
|
184
|
+
},
|
|
185
|
+
"plan": self.plan.to_dict(),
|
|
186
|
+
}
|
|
187
|
+
if self.semantics is not None:
|
|
188
|
+
values["semantics"] = self.semantics.to_dict()
|
|
189
|
+
return values
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@dataclass(frozen=True)
|
|
193
|
+
class OfflineExecutionResult:
|
|
194
|
+
status: str
|
|
195
|
+
run_id: str
|
|
196
|
+
candidate: CandidateInspection
|
|
197
|
+
workspace: Path
|
|
198
|
+
recovered: bool
|
|
199
|
+
idempotent: bool
|
|
200
|
+
|
|
201
|
+
def to_dict(self) -> dict[str, Any]:
|
|
202
|
+
return {
|
|
203
|
+
"status": self.status,
|
|
204
|
+
"run_id": self.run_id,
|
|
205
|
+
"workspace": str(self.workspace),
|
|
206
|
+
"recovered": self.recovered,
|
|
207
|
+
"idempotent": self.idempotent,
|
|
208
|
+
"candidate": self.candidate.to_dict(),
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@dataclass
|
|
213
|
+
class _Workspace:
|
|
214
|
+
path: Path
|
|
215
|
+
run_handle: Any
|
|
216
|
+
control_fd: int
|
|
217
|
+
control_identity: tuple[int, int, int]
|
|
218
|
+
lock_fd: int
|
|
219
|
+
lock_acquired: bool
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def root_fd(self) -> int:
|
|
223
|
+
return self.run_handle.run_fd
|
|
224
|
+
|
|
225
|
+
def validate(self) -> None:
|
|
226
|
+
_validate_candidate_run_handle(self.run_handle)
|
|
227
|
+
named = os.stat(CONTROL_PATH, dir_fd=self.root_fd, follow_symlinks=False)
|
|
228
|
+
opened = os.fstat(self.control_fd)
|
|
229
|
+
if (
|
|
230
|
+
not stat.S_ISDIR(named.st_mode)
|
|
231
|
+
or _entry_identity(named) != self.control_identity
|
|
232
|
+
or _entry_identity(opened) != self.control_identity
|
|
233
|
+
):
|
|
234
|
+
raise OfflineRunError(
|
|
235
|
+
"WORKSPACE_RACE",
|
|
236
|
+
"workspace control directory changed identity",
|
|
237
|
+
)
|
|
238
|
+
lock_named = os.stat(WORKSPACE_LOCK_NAME, dir_fd=self.control_fd, follow_symlinks=False)
|
|
239
|
+
lock_opened = os.fstat(self.lock_fd)
|
|
240
|
+
if (
|
|
241
|
+
not stat.S_ISREG(lock_named.st_mode)
|
|
242
|
+
or lock_named.st_nlink != 1
|
|
243
|
+
or _stat_signature(lock_named) != _stat_signature(lock_opened)
|
|
244
|
+
):
|
|
245
|
+
raise OfflineRunError("WORKSPACE_RACE", "workspace lock changed identity")
|
|
246
|
+
|
|
247
|
+
def close(self) -> None:
|
|
248
|
+
if self.lock_acquired and fcntl is not None:
|
|
249
|
+
fcntl.flock(self.lock_fd, fcntl.LOCK_UN)
|
|
250
|
+
os.close(self.lock_fd)
|
|
251
|
+
os.close(self.control_fd)
|
|
252
|
+
self.run_handle.close()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def load_offline_inputs(
|
|
256
|
+
*,
|
|
257
|
+
question_path: Path,
|
|
258
|
+
requirements_path: Path,
|
|
259
|
+
source_proposals_path: Path,
|
|
260
|
+
plan_path: Path,
|
|
261
|
+
semantics_path: Path | None = None,
|
|
262
|
+
) -> OfflineInputs:
|
|
263
|
+
"""Load strict JSON definitions and verify their complete H2 relationship."""
|
|
264
|
+
|
|
265
|
+
question = parse_question(_read_external_json(question_path, "question"))
|
|
266
|
+
requirements = parse_requirements(_read_external_json(requirements_path, "requirements"))
|
|
267
|
+
proposal_set = _read_external_json(source_proposals_path, "source proposals")
|
|
268
|
+
if not isinstance(proposal_set, dict) or set(proposal_set) != {
|
|
269
|
+
"schema_version",
|
|
270
|
+
"source_proposals",
|
|
271
|
+
}:
|
|
272
|
+
raise OfflineRunError(
|
|
273
|
+
"SOURCE_PROPOSAL_SET_INVALID",
|
|
274
|
+
"source proposal set fields are invalid",
|
|
275
|
+
)
|
|
276
|
+
if proposal_set["schema_version"] != SOURCE_PROPOSAL_SET_VERSION:
|
|
277
|
+
raise OfflineRunError(
|
|
278
|
+
"SOURCE_PROPOSAL_SET_INVALID",
|
|
279
|
+
"source proposal set version is unsupported",
|
|
280
|
+
)
|
|
281
|
+
proposal_values = proposal_set["source_proposals"]
|
|
282
|
+
if not isinstance(proposal_values, list):
|
|
283
|
+
raise OfflineRunError(
|
|
284
|
+
"SOURCE_PROPOSAL_SET_INVALID",
|
|
285
|
+
"source_proposals must be an array",
|
|
286
|
+
)
|
|
287
|
+
proposals = tuple(parse_source_proposal(item) for item in proposal_values)
|
|
288
|
+
plan = parse_plan(_read_external_json(plan_path, "plan"))
|
|
289
|
+
if isinstance(plan, GraphTablePlan):
|
|
290
|
+
raise OfflineRunError(
|
|
291
|
+
"GRAPH_OFFLINE_UNSUPPORTED",
|
|
292
|
+
"local-graph-table-plan.v1 must be executed through an approved Recipe or "
|
|
293
|
+
"direct plan-check",
|
|
294
|
+
)
|
|
295
|
+
semantics = (
|
|
296
|
+
parse_table_semantics(_read_external_json(semantics_path, "semantics"))
|
|
297
|
+
if semantics_path is not None
|
|
298
|
+
else None
|
|
299
|
+
)
|
|
300
|
+
inputs = OfflineInputs(
|
|
301
|
+
question=question,
|
|
302
|
+
requirements=requirements,
|
|
303
|
+
source_proposals=proposals,
|
|
304
|
+
plan=plan,
|
|
305
|
+
semantics=semantics,
|
|
306
|
+
)
|
|
307
|
+
inputs.package_context.validate_for_plan(plan)
|
|
308
|
+
return inputs
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def init_workspace(
|
|
312
|
+
workspace_path: Path,
|
|
313
|
+
*,
|
|
314
|
+
run_id: str,
|
|
315
|
+
input_root: Path,
|
|
316
|
+
inputs: OfflineInputs,
|
|
317
|
+
limits: ResourceLimits | None = None,
|
|
318
|
+
) -> dict[str, Any]:
|
|
319
|
+
"""Snapshot two sources and atomically install one resumable local workspace."""
|
|
320
|
+
|
|
321
|
+
if isinstance(inputs.plan, GraphTablePlan):
|
|
322
|
+
raise OfflineRunError(
|
|
323
|
+
"GRAPH_OFFLINE_UNSUPPORTED",
|
|
324
|
+
"local-graph-table-plan.v1 must be executed through an approved Recipe or "
|
|
325
|
+
"direct plan-check",
|
|
326
|
+
)
|
|
327
|
+
run_id = _identifier(run_id, "run_id")
|
|
328
|
+
inputs.package_context.validate_for_plan(inputs.plan)
|
|
329
|
+
limits = limits or ResourceLimits()
|
|
330
|
+
workspace = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
331
|
+
source_handle = _open_source_root(input_root)
|
|
332
|
+
try:
|
|
333
|
+
_validate_root_separation(source_handle.path, workspace)
|
|
334
|
+
source_bytes = {
|
|
335
|
+
source.source_id: _read_confined_source_from_handle(
|
|
336
|
+
source_handle,
|
|
337
|
+
source.path,
|
|
338
|
+
limits,
|
|
339
|
+
)
|
|
340
|
+
for source in inputs.plan.sources
|
|
341
|
+
}
|
|
342
|
+
finally:
|
|
343
|
+
source_handle.close()
|
|
344
|
+
classify_plan_sources_before_persistence(inputs.plan, source_bytes)
|
|
345
|
+
|
|
346
|
+
definition_values = inputs.definition_values()
|
|
347
|
+
definition_bytes = {
|
|
348
|
+
name: canonical_json_bytes(value) for name, value in definition_values.items()
|
|
349
|
+
}
|
|
350
|
+
definition_sha256s = {name: sha256_bytes(raw) for name, raw in definition_bytes.items()}
|
|
351
|
+
state = _initial_state(run_id, definition_sha256s)
|
|
352
|
+
output_parent = _open_output_parent(workspace)
|
|
353
|
+
staging = workspace.parent / f".{workspace.name}.tmp-{uuid.uuid4().hex}"
|
|
354
|
+
staging_identity: tuple[int, int, int] | None = None
|
|
355
|
+
staging_fd = -1
|
|
356
|
+
installed = False
|
|
357
|
+
try:
|
|
358
|
+
_validate_output_parent_handle(output_parent)
|
|
359
|
+
os.mkdir(staging.name, 0o700, dir_fd=output_parent.parent_fd)
|
|
360
|
+
staging_named = os.stat(
|
|
361
|
+
staging.name,
|
|
362
|
+
dir_fd=output_parent.parent_fd,
|
|
363
|
+
follow_symlinks=False,
|
|
364
|
+
)
|
|
365
|
+
if not stat.S_ISDIR(staging_named.st_mode):
|
|
366
|
+
raise OfflineRunError("WORKSPACE_INIT_RACE", "workspace staging is not a directory")
|
|
367
|
+
staging_identity = _entry_identity(staging_named)
|
|
368
|
+
staging_fd = os.open(
|
|
369
|
+
staging.name,
|
|
370
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
371
|
+
dir_fd=output_parent.parent_fd,
|
|
372
|
+
)
|
|
373
|
+
if _entry_identity(os.fstat(staging_fd)) != staging_identity:
|
|
374
|
+
raise OfflineRunError(
|
|
375
|
+
"WORKSPACE_INIT_RACE",
|
|
376
|
+
"workspace staging changed identity",
|
|
377
|
+
)
|
|
378
|
+
_populate_workspace(
|
|
379
|
+
staging_fd,
|
|
380
|
+
definition_bytes=definition_bytes,
|
|
381
|
+
source_bytes=source_bytes,
|
|
382
|
+
plan=inputs.plan,
|
|
383
|
+
state=state,
|
|
384
|
+
)
|
|
385
|
+
_validate_output_parent_handle(output_parent)
|
|
386
|
+
_rename_no_replace(
|
|
387
|
+
staging,
|
|
388
|
+
workspace,
|
|
389
|
+
expected_source_identity=staging_identity,
|
|
390
|
+
parent_fd=output_parent.parent_fd,
|
|
391
|
+
)
|
|
392
|
+
installed = True
|
|
393
|
+
os.fsync(output_parent.parent_fd)
|
|
394
|
+
_validate_output_parent_handle(output_parent)
|
|
395
|
+
except Exception:
|
|
396
|
+
if staging_identity is not None:
|
|
397
|
+
_remove_tree(
|
|
398
|
+
workspace if installed else staging,
|
|
399
|
+
expected_identity=staging_identity,
|
|
400
|
+
parent_fd=output_parent.parent_fd,
|
|
401
|
+
)
|
|
402
|
+
raise
|
|
403
|
+
finally:
|
|
404
|
+
if staging_fd >= 0:
|
|
405
|
+
os.close(staging_fd)
|
|
406
|
+
output_parent.close()
|
|
407
|
+
|
|
408
|
+
with _open_workspace(workspace) as opened:
|
|
409
|
+
observed = _read_state(opened)
|
|
410
|
+
_load_definitions(opened, observed)
|
|
411
|
+
opened.validate()
|
|
412
|
+
return _status_payload(observed, workspace)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def plan_workspace(workspace_path: Path) -> dict[str, Any]:
|
|
416
|
+
"""Validate the immutable definitions and durably complete the plan task.
|
|
417
|
+
|
|
418
|
+
Nothing here authors or edits a plan. The plan JSON is an input that was already sealed
|
|
419
|
+
into the workspace; this re-verifies it and records the plan task as completed.
|
|
420
|
+
"""
|
|
421
|
+
|
|
422
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
423
|
+
with _open_workspace(workspace_path) as workspace:
|
|
424
|
+
state = _read_state(workspace)
|
|
425
|
+
inputs = _load_definitions(workspace, state)
|
|
426
|
+
if _task(state, "plan")["status"] != "completed":
|
|
427
|
+
next_state = deepcopy(state)
|
|
428
|
+
plan_task = _task(next_state, "plan")
|
|
429
|
+
plan_task.update(
|
|
430
|
+
{
|
|
431
|
+
"status": "completed",
|
|
432
|
+
"attempts": plan_task["attempts"] + 1,
|
|
433
|
+
"result_digest": sha256_bytes(canonical_json_bytes(inputs.plan.to_dict())),
|
|
434
|
+
}
|
|
435
|
+
)
|
|
436
|
+
next_state["phase"] = "planned"
|
|
437
|
+
_append_event(next_state, "plan_validated", "plan")
|
|
438
|
+
_write_state(workspace, next_state)
|
|
439
|
+
state = next_state
|
|
440
|
+
return {
|
|
441
|
+
"status": "plan_validated",
|
|
442
|
+
"run_id": state["run_id"],
|
|
443
|
+
"workspace": str(workspace_path),
|
|
444
|
+
"question": inputs.question.to_dict(),
|
|
445
|
+
"requirements": inputs.requirements.to_dict(),
|
|
446
|
+
"source_proposals": [
|
|
447
|
+
item.to_dict()
|
|
448
|
+
for item in sorted(inputs.source_proposals, key=lambda item: item.source_id)
|
|
449
|
+
],
|
|
450
|
+
"plan": inputs.plan.to_dict(),
|
|
451
|
+
"plan_digest": _task(state, "plan")["result_digest"],
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def run_workspace(
|
|
456
|
+
workspace_path: Path,
|
|
457
|
+
*,
|
|
458
|
+
fault_hook: Callable[[str], None] | None = None,
|
|
459
|
+
visual_store: VisualRunStore | None = None,
|
|
460
|
+
) -> OfflineExecutionResult:
|
|
461
|
+
"""Build using the fixed H2 coordinator-owned producer enrollment."""
|
|
462
|
+
|
|
463
|
+
return _execute_workspace_visualized(
|
|
464
|
+
workspace_path, fault_hook=fault_hook, is_resume=False, visual_store=visual_store
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def resume_workspace(
|
|
469
|
+
workspace_path: Path,
|
|
470
|
+
*,
|
|
471
|
+
fault_hook: Callable[[str], None] | None = None,
|
|
472
|
+
visual_store: VisualRunStore | None = None,
|
|
473
|
+
) -> OfflineExecutionResult:
|
|
474
|
+
"""Resume only with the fixed coordinator-owned H2 producer identity."""
|
|
475
|
+
|
|
476
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
477
|
+
with _open_workspace(workspace_path) as workspace:
|
|
478
|
+
state = _read_state(workspace)
|
|
479
|
+
producer = state["producer"]
|
|
480
|
+
if producer is None:
|
|
481
|
+
raise OfflineRunError(
|
|
482
|
+
"RESUME_NOT_STARTED",
|
|
483
|
+
"run has no persisted producer identity; invoke run first",
|
|
484
|
+
)
|
|
485
|
+
return _execute_workspace_visualized(
|
|
486
|
+
workspace_path, fault_hook=fault_hook, is_resume=True, visual_store=visual_store
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _execute_workspace_visualized(
|
|
491
|
+
workspace_path: Path,
|
|
492
|
+
*,
|
|
493
|
+
fault_hook: Callable[[str], None] | None,
|
|
494
|
+
is_resume: bool,
|
|
495
|
+
visual_store: VisualRunStore | None,
|
|
496
|
+
) -> OfflineExecutionResult:
|
|
497
|
+
"""Include coordinator validation/recovery in one retryable visual attempt."""
|
|
498
|
+
|
|
499
|
+
work = None
|
|
500
|
+
attempt = _H2_PRODUCER_ATTEMPT
|
|
501
|
+
execution_visual_store = visual_store
|
|
502
|
+
if visual_store is not None:
|
|
503
|
+
from mostlyright.data_harness.visual_run.reducer import reduce_events
|
|
504
|
+
from mostlyright.data_harness.visual_run.sdk import (
|
|
505
|
+
VisualWork,
|
|
506
|
+
attempt_event_key,
|
|
507
|
+
next_attempt_coordinate,
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
work = VisualWork(visual_store, actor="worker")
|
|
511
|
+
if not visual_store.events():
|
|
512
|
+
work.start_run(key="offline:run-started")
|
|
513
|
+
visual_state = reduce_events(visual_store.events())
|
|
514
|
+
if visual_state.terminal:
|
|
515
|
+
if visual_state.lifecycle != "completed":
|
|
516
|
+
raise OfflineRunError(
|
|
517
|
+
"VISUAL_RUN_TERMINAL",
|
|
518
|
+
"a terminal failed visual run cannot accept a resume",
|
|
519
|
+
)
|
|
520
|
+
# The coordinator's existing-result path remains read-only and deterministic. Its
|
|
521
|
+
# already-completed visual log is evidence, not a new invocation sink.
|
|
522
|
+
work = None
|
|
523
|
+
execution_visual_store = None
|
|
524
|
+
else:
|
|
525
|
+
attempt = next_attempt_coordinate(visual_store, "offline", attempt)
|
|
526
|
+
work.attempt_started(attempt, key=attempt_event_key("offline", attempt, "started"))
|
|
527
|
+
try:
|
|
528
|
+
result = _execute_workspace(
|
|
529
|
+
workspace_path,
|
|
530
|
+
producer_principal=_H2_PRODUCER_PRINCIPAL,
|
|
531
|
+
producer_attempt=_H2_PRODUCER_ATTEMPT,
|
|
532
|
+
fault_hook=fault_hook,
|
|
533
|
+
is_resume=is_resume,
|
|
534
|
+
visual_store=execution_visual_store,
|
|
535
|
+
)
|
|
536
|
+
except BaseException as exc:
|
|
537
|
+
if work is not None:
|
|
538
|
+
from mostlyright.data_harness.visual_run.contracts import TerminalError
|
|
539
|
+
|
|
540
|
+
candidate = getattr(exc, "code", "OFFLINE_EXECUTION_FAILED")
|
|
541
|
+
code = (
|
|
542
|
+
candidate
|
|
543
|
+
if isinstance(candidate, str) and re.fullmatch(r"[A-Z][A-Z0-9_]{1,63}", candidate)
|
|
544
|
+
else "OFFLINE_EXECUTION_FAILED"
|
|
545
|
+
)
|
|
546
|
+
work.attempt_failed(
|
|
547
|
+
attempt,
|
|
548
|
+
TerminalError(code, f"Offline execution stopped with {code}.", True),
|
|
549
|
+
key=attempt_event_key("offline", attempt, "failed"),
|
|
550
|
+
)
|
|
551
|
+
raise
|
|
552
|
+
if work is not None:
|
|
553
|
+
work.attempt_completed(attempt, key=attempt_event_key("offline", attempt, "completed"))
|
|
554
|
+
return result
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def status_workspace(
|
|
558
|
+
workspace_path: Path,
|
|
559
|
+
*,
|
|
560
|
+
wait: bool = True,
|
|
561
|
+
) -> dict[str, Any]:
|
|
562
|
+
"""Return coordinator state without accepting caller-asserted review authority.
|
|
563
|
+
|
|
564
|
+
``wait=False`` gives the same reading without queueing behind a build that is holding the
|
|
565
|
+
folder: it refuses with ``WORKSPACE_BUSY`` instead. That is what a command which only reports
|
|
566
|
+
wants -- `mr-data preflight` says "check this before you build", and a check that never returns
|
|
567
|
+
while a build is running is not one. Nothing about the reading is weakened; only the waiting is.
|
|
568
|
+
"""
|
|
569
|
+
|
|
570
|
+
return _status_workspace(workspace_path, wait=wait)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _status_enrolled_workspace(
|
|
574
|
+
workspace_path: Path,
|
|
575
|
+
*,
|
|
576
|
+
review_authority_provider: TrustedReviewAuthorityProvider | None,
|
|
577
|
+
) -> dict[str, Any]:
|
|
578
|
+
"""Return reviewed state only through an inherited coordinator session."""
|
|
579
|
+
|
|
580
|
+
_authority, assignment, verifier = _review_authority_material(review_authority_provider)
|
|
581
|
+
return _status_workspace(
|
|
582
|
+
workspace_path,
|
|
583
|
+
trusted_verifier_public_key=verifier.public_key(),
|
|
584
|
+
trusted_assignment_value=assignment,
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _status_workspace(
|
|
589
|
+
workspace_path: Path,
|
|
590
|
+
*,
|
|
591
|
+
trusted_verifier_public_key: Ed25519PublicKey | None = None,
|
|
592
|
+
trusted_assignment_value: Any | None = None,
|
|
593
|
+
wait: bool = True,
|
|
594
|
+
) -> dict[str, Any]:
|
|
595
|
+
"""Return the durable coordinator record without overstating review authority."""
|
|
596
|
+
|
|
597
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
598
|
+
with _open_workspace(workspace_path, wait=wait) as workspace:
|
|
599
|
+
state = _read_state(workspace)
|
|
600
|
+
_load_definitions(workspace, state)
|
|
601
|
+
if state["candidate"] is not None:
|
|
602
|
+
if not _named_child_exists(workspace.root_fd, RUN_RESULT_PATH):
|
|
603
|
+
raise OfflineRunError(
|
|
604
|
+
"CANDIDATE_MISSING",
|
|
605
|
+
"durable state references a missing candidate",
|
|
606
|
+
)
|
|
607
|
+
if state["producer"] is None: # pragma: no cover - state validator enforces this
|
|
608
|
+
raise OfflineRunError(
|
|
609
|
+
"RUN_STATE_INVALID",
|
|
610
|
+
"candidate state is missing its producer fence",
|
|
611
|
+
)
|
|
612
|
+
_inspect_and_match_state(
|
|
613
|
+
workspace_path / RUN_RESULT_PATH,
|
|
614
|
+
state,
|
|
615
|
+
state["producer"],
|
|
616
|
+
workspace=workspace,
|
|
617
|
+
)
|
|
618
|
+
payload = _status_payload(state, workspace_path)
|
|
619
|
+
if trusted_verifier_public_key is not None:
|
|
620
|
+
verified_review = _verify_workspace_review(
|
|
621
|
+
workspace,
|
|
622
|
+
state,
|
|
623
|
+
trusted_verifier_public_key=trusted_verifier_public_key,
|
|
624
|
+
trusted_assignment_value=trusted_assignment_value,
|
|
625
|
+
)
|
|
626
|
+
payload["verified_review"] = {
|
|
627
|
+
"status": verified_review.status,
|
|
628
|
+
"candidate_digest": verified_review.candidate_digest,
|
|
629
|
+
"decision_digest": verified_review.decision_digest,
|
|
630
|
+
}
|
|
631
|
+
payload["status"] = verified_review.status
|
|
632
|
+
payload["review_authority"] = "cryptographically_verified_exact_review"
|
|
633
|
+
return payload
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def inspect_workspace(
|
|
637
|
+
workspace_path: Path,
|
|
638
|
+
) -> dict[str, Any]:
|
|
639
|
+
"""Inspect a candidate without accepting caller-asserted review authority."""
|
|
640
|
+
|
|
641
|
+
return _inspect_workspace(workspace_path)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _inspect_enrolled_workspace(
|
|
645
|
+
workspace_path: Path,
|
|
646
|
+
*,
|
|
647
|
+
review_authority_provider: TrustedReviewAuthorityProvider | None,
|
|
648
|
+
) -> dict[str, Any]:
|
|
649
|
+
"""Inspect reviewed state only through an inherited coordinator session."""
|
|
650
|
+
|
|
651
|
+
_authority, assignment, verifier = _review_authority_material(review_authority_provider)
|
|
652
|
+
return _inspect_workspace(
|
|
653
|
+
workspace_path,
|
|
654
|
+
trusted_verifier_public_key=verifier.public_key(),
|
|
655
|
+
trusted_assignment_value=assignment,
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def _inspect_workspace(
|
|
660
|
+
workspace_path: Path,
|
|
661
|
+
*,
|
|
662
|
+
trusted_verifier_public_key: Ed25519PublicKey | None = None,
|
|
663
|
+
trusted_assignment_value: Any | None = None,
|
|
664
|
+
) -> dict[str, Any]:
|
|
665
|
+
"""Return exact replay-verified candidate evidence plus coordinator state."""
|
|
666
|
+
|
|
667
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
668
|
+
with _open_workspace(workspace_path) as workspace:
|
|
669
|
+
state = _read_state(workspace)
|
|
670
|
+
_load_definitions(workspace, state)
|
|
671
|
+
result_path = workspace_path / state["paths"]["result"]
|
|
672
|
+
producer = state["producer"]
|
|
673
|
+
if state["candidate"] is None or producer is None:
|
|
674
|
+
raise OfflineRunError(
|
|
675
|
+
"CANDIDATE_NOT_COMMITTED",
|
|
676
|
+
"workspace has no durably committed candidate to inspect",
|
|
677
|
+
)
|
|
678
|
+
inspection, lease = _inspect_and_retain_matched_state(
|
|
679
|
+
result_path,
|
|
680
|
+
state,
|
|
681
|
+
producer,
|
|
682
|
+
workspace=workspace,
|
|
683
|
+
)
|
|
684
|
+
try:
|
|
685
|
+
payload = {
|
|
686
|
+
"status": "candidate_inspected",
|
|
687
|
+
"workspace": str(workspace_path),
|
|
688
|
+
"run_id": state["run_id"],
|
|
689
|
+
"state_phase": state["phase"],
|
|
690
|
+
"candidate": inspection.to_dict(),
|
|
691
|
+
"review_authority": "not_verified_without_enrolled_verifier_key",
|
|
692
|
+
"hosted_release": False,
|
|
693
|
+
}
|
|
694
|
+
if trusted_verifier_public_key is not None:
|
|
695
|
+
verified_review = _verify_workspace_review(
|
|
696
|
+
workspace,
|
|
697
|
+
state,
|
|
698
|
+
trusted_verifier_public_key=trusted_verifier_public_key,
|
|
699
|
+
trusted_assignment_value=trusted_assignment_value,
|
|
700
|
+
)
|
|
701
|
+
if verified_review.candidate_digest != inspection.candidate_digest:
|
|
702
|
+
raise OfflineRunError(
|
|
703
|
+
"REVIEW_CANDIDATE_MISMATCH",
|
|
704
|
+
"verified review does not match the retained candidate snapshot",
|
|
705
|
+
)
|
|
706
|
+
payload["verified_review"] = {
|
|
707
|
+
"status": verified_review.status,
|
|
708
|
+
"candidate_digest": verified_review.candidate_digest,
|
|
709
|
+
"decision_digest": verified_review.decision_digest,
|
|
710
|
+
}
|
|
711
|
+
payload["status"] = verified_review.status
|
|
712
|
+
payload["review_authority"] = "cryptographically_verified_exact_review"
|
|
713
|
+
lease.validate()
|
|
714
|
+
_match_candidate_inspection(inspection, state, producer)
|
|
715
|
+
workspace.validate()
|
|
716
|
+
lease.validate()
|
|
717
|
+
return payload
|
|
718
|
+
finally:
|
|
719
|
+
lease.close()
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def workspace_result_path(workspace_path: Path) -> Path:
|
|
723
|
+
"""Resolve the fixed descriptor-validated result child for review/verification."""
|
|
724
|
+
|
|
725
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
726
|
+
with _open_workspace(workspace_path) as workspace:
|
|
727
|
+
state = _read_state(workspace)
|
|
728
|
+
_load_definitions(workspace, state)
|
|
729
|
+
return workspace_path / state["paths"]["result"]
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def review_target_path(workspace_path: Path) -> Path:
|
|
733
|
+
"""Resolve the run directory review enrollment acts on: a workspace or a sealed run dir.
|
|
734
|
+
|
|
735
|
+
A workspace is marked by its ``.mr-data`` control directory and keeps its run under the
|
|
736
|
+
recorded result child, resolved through the locked, replayed reader above. Any other path is
|
|
737
|
+
returned absolutized and unjudged, mirroring the CLI's own run-dir-or-workspace dispatch:
|
|
738
|
+
every caller hands the result to ``verify_candidate``, which is the gate that decides whether
|
|
739
|
+
a sealed Build is actually there. Recipe executions seal run directories without any
|
|
740
|
+
workspace, and review enrollment accepts them exactly the way ``mr-data verify`` and
|
|
741
|
+
``mr-data review --local`` already do.
|
|
742
|
+
"""
|
|
743
|
+
|
|
744
|
+
absolute = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
745
|
+
try:
|
|
746
|
+
control = os.lstat(absolute / CONTROL_PATH)
|
|
747
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
748
|
+
return absolute
|
|
749
|
+
except OSError:
|
|
750
|
+
return absolute
|
|
751
|
+
if not stat.S_ISDIR(control.st_mode):
|
|
752
|
+
return absolute
|
|
753
|
+
return workspace_result_path(absolute)
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def recorded_result_path(workspace_path: Path) -> Path:
|
|
757
|
+
"""Where a workspace records its run, for a reading that verifies nothing and must not wait.
|
|
758
|
+
|
|
759
|
+
This is not the resolver a command that acts on a run uses. That one is
|
|
760
|
+
:func:`workspace_result_path`, which takes the durable lock, *waits* for it, and replays the
|
|
761
|
+
sealed definitions before it answers: exactly right for a command that is about to open,
|
|
762
|
+
resume, review, or verify the run, and exactly wrong for a listing. A listing walks folders
|
|
763
|
+
nobody named, says in every row that it verified nothing, and must not stall behind a build
|
|
764
|
+
that is running in one of them.
|
|
765
|
+
|
|
766
|
+
So this one never waits. It takes a shared lock without blocking and refuses with
|
|
767
|
+
``WORKSPACE_BUSY`` when a build holds the exclusive one, and it reads the recorded result path
|
|
768
|
+
out of the run state and nothing else. Nothing is weakened by the shorter reading, because
|
|
769
|
+
nothing is trusted from it: the value it returns is validated by :func:`_validate_state` to be
|
|
770
|
+
exactly ``RUN_RESULT_PATH``, and whatever a caller then does with that path goes through the
|
|
771
|
+
same verified reader every other caller uses.
|
|
772
|
+
"""
|
|
773
|
+
|
|
774
|
+
if os.name == "nt" or fcntl is None:
|
|
775
|
+
raise OfflineRunError(
|
|
776
|
+
"WORKSPACE_PLATFORM_UNSUPPORTED",
|
|
777
|
+
"descriptor-relative durable workspaces are unavailable on this platform",
|
|
778
|
+
)
|
|
779
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
780
|
+
try:
|
|
781
|
+
control_fd = os.open(
|
|
782
|
+
workspace_path / CONTROL_PATH,
|
|
783
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
784
|
+
)
|
|
785
|
+
except FileNotFoundError as exc:
|
|
786
|
+
raise OfflineRunError(
|
|
787
|
+
"WORKSPACE_ABSENT",
|
|
788
|
+
f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
|
|
789
|
+
"and a workbench has to be a folder this harness set up",
|
|
790
|
+
) from exc
|
|
791
|
+
except NotADirectoryError as exc:
|
|
792
|
+
raise OfflineRunError("WORKSPACE_INVALID", "control path is not a directory") from exc
|
|
793
|
+
try:
|
|
794
|
+
state = _read_state_without_waiting(control_fd, workspace_path)
|
|
795
|
+
finally:
|
|
796
|
+
os.close(control_fd)
|
|
797
|
+
return workspace_path / state["paths"]["result"]
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _read_state_without_waiting(control_fd: int, workspace_path: Path) -> dict[str, Any]:
|
|
801
|
+
"""The run state, read under a shared lock that is never waited for.
|
|
802
|
+
|
|
803
|
+
The shared lock is taken for one reason: a build holds the exclusive one for its whole length,
|
|
804
|
+
so failing to take this one immediately is how a reader learns a build is running without
|
|
805
|
+
joining the queue behind it. It is released before anything is parsed, and a build that starts
|
|
806
|
+
a microsecond later waits on nothing.
|
|
807
|
+
"""
|
|
808
|
+
|
|
809
|
+
try:
|
|
810
|
+
lock_fd = os.open(
|
|
811
|
+
WORKSPACE_LOCK_NAME,
|
|
812
|
+
os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
|
|
813
|
+
dir_fd=control_fd,
|
|
814
|
+
)
|
|
815
|
+
except FileNotFoundError as exc:
|
|
816
|
+
raise OfflineRunError(
|
|
817
|
+
"WORKSPACE_ABSENT",
|
|
818
|
+
f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
|
|
819
|
+
"and a workbench has to be a folder this harness set up",
|
|
820
|
+
) from exc
|
|
821
|
+
try:
|
|
822
|
+
try:
|
|
823
|
+
fcntl.flock(lock_fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
824
|
+
except OSError as exc:
|
|
825
|
+
if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
|
|
826
|
+
raise
|
|
827
|
+
raise OfflineRunError(
|
|
828
|
+
"WORKSPACE_BUSY",
|
|
829
|
+
f"a build is running in {workspace_path}",
|
|
830
|
+
) from exc
|
|
831
|
+
try:
|
|
832
|
+
raw = _read_regular_file_at(
|
|
833
|
+
control_fd,
|
|
834
|
+
_STATE_NAME,
|
|
835
|
+
maximum=_MAX_STATE_BYTES,
|
|
836
|
+
required_mode=0o600,
|
|
837
|
+
)
|
|
838
|
+
except FileNotFoundError as exc:
|
|
839
|
+
# The folder was set up but no run was ever recorded in it. That is not a damaged
|
|
840
|
+
# record; it is the absence of one, and it names a different fix.
|
|
841
|
+
raise OfflineRunError(
|
|
842
|
+
"WORKSPACE_ABSENT",
|
|
843
|
+
f"there is {presence_at(workspace_path) or UNKNOWN_KIND} at {workspace_path}, "
|
|
844
|
+
"and a workbench has to be a folder this harness set up",
|
|
845
|
+
) from exc
|
|
846
|
+
finally:
|
|
847
|
+
os.close(lock_fd)
|
|
848
|
+
try:
|
|
849
|
+
value = parse_canonical_json(raw)
|
|
850
|
+
except ValueError as exc:
|
|
851
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run state is not canonical JSON") from exc
|
|
852
|
+
return _validate_state(value)
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def _record_review(
|
|
856
|
+
workspace_path: Path,
|
|
857
|
+
*,
|
|
858
|
+
status: str,
|
|
859
|
+
candidate_digest: str,
|
|
860
|
+
decision_digest: str,
|
|
861
|
+
) -> dict[str, Any]:
|
|
862
|
+
"""Record a verified review receipt without making it the verification authority."""
|
|
863
|
+
|
|
864
|
+
if status not in {"reviewed_candidate", "fixes_required"}:
|
|
865
|
+
raise OfflineRunError("REVIEW_STATUS_INVALID", "review status is unsupported in H2")
|
|
866
|
+
candidate_digest = _digest(candidate_digest, "candidate_digest")
|
|
867
|
+
decision_digest = _digest(decision_digest, "decision_digest")
|
|
868
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
869
|
+
with _open_workspace(workspace_path) as workspace:
|
|
870
|
+
state = _read_state(workspace)
|
|
871
|
+
next_state = _record_review_in_open_workspace(
|
|
872
|
+
workspace,
|
|
873
|
+
state,
|
|
874
|
+
status=status,
|
|
875
|
+
candidate_digest=candidate_digest,
|
|
876
|
+
decision_digest=decision_digest,
|
|
877
|
+
)
|
|
878
|
+
return _status_payload(next_state, workspace_path)
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _finalize_enrolled_workspace_review(
|
|
882
|
+
workspace_path: Path,
|
|
883
|
+
*,
|
|
884
|
+
report_values: list[Any],
|
|
885
|
+
review_authority_provider: TrustedReviewAuthorityProvider | None,
|
|
886
|
+
) -> ReviewResult:
|
|
887
|
+
"""Finalize workspace review only through an inherited coordinator session."""
|
|
888
|
+
|
|
889
|
+
_authority, assignment, verifier = _review_authority_material(review_authority_provider)
|
|
890
|
+
return _finalize_workspace_review(
|
|
891
|
+
workspace_path,
|
|
892
|
+
assignment_value=assignment,
|
|
893
|
+
report_values=report_values,
|
|
894
|
+
verifier_private_key=verifier,
|
|
895
|
+
)
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def _finalize_workspace_review(
|
|
899
|
+
workspace_path: Path,
|
|
900
|
+
*,
|
|
901
|
+
assignment_value: Any,
|
|
902
|
+
report_values: list[Any],
|
|
903
|
+
verifier_private_key: TrustedSigner,
|
|
904
|
+
) -> ReviewResult:
|
|
905
|
+
"""Finalize and record one review against a single retained workspace snapshot."""
|
|
906
|
+
|
|
907
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
908
|
+
with _open_workspace(workspace_path) as workspace:
|
|
909
|
+
state = _read_state(workspace)
|
|
910
|
+
_load_definitions(workspace, state)
|
|
911
|
+
if state["candidate"] is None:
|
|
912
|
+
raise OfflineRunError(
|
|
913
|
+
"CANDIDATE_NOT_COMMITTED",
|
|
914
|
+
"workspace has no durably committed candidate to review",
|
|
915
|
+
)
|
|
916
|
+
result_path = workspace_path / state["paths"]["result"]
|
|
917
|
+
result = _finalize_review(
|
|
918
|
+
run_dir=result_path,
|
|
919
|
+
assignment_value=assignment_value,
|
|
920
|
+
report_values=report_values,
|
|
921
|
+
verifier_private_key=verifier_private_key,
|
|
922
|
+
_run_parent_fd=workspace.root_fd,
|
|
923
|
+
_transaction_validator=workspace.validate,
|
|
924
|
+
)
|
|
925
|
+
if result.candidate_digest != state["candidate"]["candidate_digest"]:
|
|
926
|
+
raise OfflineRunError(
|
|
927
|
+
"REVIEW_CANDIDATE_MISMATCH",
|
|
928
|
+
"review candidate does not match durable run state",
|
|
929
|
+
)
|
|
930
|
+
_record_review_in_open_workspace(
|
|
931
|
+
workspace,
|
|
932
|
+
state,
|
|
933
|
+
status=result.status,
|
|
934
|
+
candidate_digest=result.candidate_digest,
|
|
935
|
+
decision_digest=result.decision_digest,
|
|
936
|
+
)
|
|
937
|
+
workspace.validate()
|
|
938
|
+
return result
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def _record_review_in_open_workspace(
|
|
942
|
+
workspace: _Workspace,
|
|
943
|
+
state: dict[str, Any],
|
|
944
|
+
*,
|
|
945
|
+
status: str,
|
|
946
|
+
candidate_digest: str,
|
|
947
|
+
decision_digest: str,
|
|
948
|
+
) -> dict[str, Any]:
|
|
949
|
+
if status not in {"reviewed_candidate", "fixes_required"}:
|
|
950
|
+
raise OfflineRunError("REVIEW_STATUS_INVALID", "review status is unsupported in H2")
|
|
951
|
+
candidate_digest = _digest(candidate_digest, "candidate_digest")
|
|
952
|
+
decision_digest = _digest(decision_digest, "decision_digest")
|
|
953
|
+
if state["candidate"] is None or state["candidate"]["candidate_digest"] != candidate_digest:
|
|
954
|
+
raise OfflineRunError(
|
|
955
|
+
"REVIEW_CANDIDATE_MISMATCH",
|
|
956
|
+
"review candidate does not match durable run state",
|
|
957
|
+
)
|
|
958
|
+
review_task = _task(state, "review")
|
|
959
|
+
if (
|
|
960
|
+
review_task["status"] == "completed"
|
|
961
|
+
and state["review"] is not None
|
|
962
|
+
and state["review"]["decision_digest"] == decision_digest
|
|
963
|
+
):
|
|
964
|
+
return state
|
|
965
|
+
next_state = deepcopy(state)
|
|
966
|
+
next_review_task = _task(next_state, "review")
|
|
967
|
+
next_review_task.update(
|
|
968
|
+
{
|
|
969
|
+
"status": "completed",
|
|
970
|
+
"attempts": next_review_task["attempts"] + 1,
|
|
971
|
+
"result_digest": decision_digest,
|
|
972
|
+
}
|
|
973
|
+
)
|
|
974
|
+
next_state["review"] = {
|
|
975
|
+
"candidate_digest": candidate_digest,
|
|
976
|
+
"decision_digest": decision_digest,
|
|
977
|
+
"verified_status": status,
|
|
978
|
+
}
|
|
979
|
+
next_state["phase"] = "review_recorded"
|
|
980
|
+
_append_event(next_state, "exact_review_recorded", "review")
|
|
981
|
+
_write_state(workspace, next_state)
|
|
982
|
+
return next_state
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def _verify_workspace_review(
|
|
986
|
+
workspace: _Workspace,
|
|
987
|
+
state: dict[str, Any],
|
|
988
|
+
*,
|
|
989
|
+
trusted_verifier_public_key: Ed25519PublicKey,
|
|
990
|
+
trusted_assignment_value: Any | None,
|
|
991
|
+
) -> ReviewResult:
|
|
992
|
+
if state["candidate"] is None:
|
|
993
|
+
raise OfflineRunError(
|
|
994
|
+
"CANDIDATE_NOT_COMMITTED",
|
|
995
|
+
"workspace has no durably committed candidate to verify",
|
|
996
|
+
)
|
|
997
|
+
result_path = workspace.path / state["paths"]["result"]
|
|
998
|
+
result = _verify_review_decision(
|
|
999
|
+
result_path,
|
|
1000
|
+
trusted_verifier_public_key=trusted_verifier_public_key,
|
|
1001
|
+
trusted_assignment_value=trusted_assignment_value,
|
|
1002
|
+
_run_parent_fd=workspace.root_fd,
|
|
1003
|
+
_ancestor_validator=workspace.validate,
|
|
1004
|
+
)
|
|
1005
|
+
if result.candidate_digest != state["candidate"]["candidate_digest"]:
|
|
1006
|
+
raise OfflineRunError(
|
|
1007
|
+
"REVIEW_CANDIDATE_MISMATCH",
|
|
1008
|
+
"verified review does not match durable run state",
|
|
1009
|
+
)
|
|
1010
|
+
workspace.validate()
|
|
1011
|
+
return result
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _execute_workspace(
|
|
1015
|
+
workspace_path: Path,
|
|
1016
|
+
*,
|
|
1017
|
+
producer_principal: str,
|
|
1018
|
+
producer_attempt: str,
|
|
1019
|
+
fault_hook: Callable[[str], None] | None,
|
|
1020
|
+
is_resume: bool,
|
|
1021
|
+
visual_store: VisualRunStore | None = None,
|
|
1022
|
+
) -> OfflineExecutionResult:
|
|
1023
|
+
workspace_path = Path(os.path.abspath(os.fspath(workspace_path)))
|
|
1024
|
+
trusted_producer = _h2_producer_identity()
|
|
1025
|
+
if {
|
|
1026
|
+
"principal_id": producer_principal,
|
|
1027
|
+
"attempt_id": producer_attempt,
|
|
1028
|
+
} != trusted_producer:
|
|
1029
|
+
raise OfflineRunError(
|
|
1030
|
+
"PRODUCER_MISMATCH",
|
|
1031
|
+
"offline execution requires the fixed coordinator-owned H2 producer",
|
|
1032
|
+
)
|
|
1033
|
+
with _open_workspace(workspace_path) as workspace:
|
|
1034
|
+
state = _read_state(workspace)
|
|
1035
|
+
inputs = _load_definitions(workspace, state)
|
|
1036
|
+
producer = trusted_producer
|
|
1037
|
+
if state["producer"] is not None and state["producer"] != producer:
|
|
1038
|
+
raise OfflineRunError(
|
|
1039
|
+
"PRODUCER_MISMATCH",
|
|
1040
|
+
"resume/retry must use the persisted producer principal and attempt",
|
|
1041
|
+
)
|
|
1042
|
+
result_path = workspace_path / state["paths"]["result"]
|
|
1043
|
+
if _named_child_exists(workspace.root_fd, RUN_RESULT_PATH):
|
|
1044
|
+
# A process may have died after rename but before the output-parent
|
|
1045
|
+
# barrier. Retry it before adopting or committing an existing result.
|
|
1046
|
+
_durability_barrier(workspace.root_fd, "output-parent")
|
|
1047
|
+
inspection = _inspect_and_match_state(
|
|
1048
|
+
result_path,
|
|
1049
|
+
state,
|
|
1050
|
+
producer,
|
|
1051
|
+
workspace=workspace,
|
|
1052
|
+
)
|
|
1053
|
+
if _task(state, "build")["status"] == "completed":
|
|
1054
|
+
return OfflineExecutionResult(
|
|
1055
|
+
status="candidate_built",
|
|
1056
|
+
run_id=state["run_id"],
|
|
1057
|
+
candidate=inspection,
|
|
1058
|
+
workspace=workspace_path,
|
|
1059
|
+
recovered=False,
|
|
1060
|
+
idempotent=True,
|
|
1061
|
+
)
|
|
1062
|
+
next_state = _candidate_state(
|
|
1063
|
+
state,
|
|
1064
|
+
inspection,
|
|
1065
|
+
producer=producer,
|
|
1066
|
+
event="candidate_recovered",
|
|
1067
|
+
)
|
|
1068
|
+
_write_state(workspace, next_state)
|
|
1069
|
+
return OfflineExecutionResult(
|
|
1070
|
+
status="candidate_built",
|
|
1071
|
+
run_id=next_state["run_id"],
|
|
1072
|
+
candidate=inspection,
|
|
1073
|
+
workspace=workspace_path,
|
|
1074
|
+
recovered=True,
|
|
1075
|
+
idempotent=False,
|
|
1076
|
+
)
|
|
1077
|
+
|
|
1078
|
+
_remove_orphan_staging(workspace)
|
|
1079
|
+
next_state = deepcopy(state)
|
|
1080
|
+
plan_task = _task(next_state, "plan")
|
|
1081
|
+
if plan_task["status"] != "completed":
|
|
1082
|
+
plan_task.update(
|
|
1083
|
+
{
|
|
1084
|
+
"status": "completed",
|
|
1085
|
+
"attempts": plan_task["attempts"] + 1,
|
|
1086
|
+
"result_digest": sha256_bytes(canonical_json_bytes(inputs.plan.to_dict())),
|
|
1087
|
+
}
|
|
1088
|
+
)
|
|
1089
|
+
_append_event(next_state, "plan_validated", "plan")
|
|
1090
|
+
build_task = _task(next_state, "build")
|
|
1091
|
+
build_task.update(
|
|
1092
|
+
{
|
|
1093
|
+
"status": "running",
|
|
1094
|
+
"attempts": build_task["attempts"] + 1,
|
|
1095
|
+
"result_digest": None,
|
|
1096
|
+
}
|
|
1097
|
+
)
|
|
1098
|
+
next_state["phase"] = "building"
|
|
1099
|
+
next_state["producer"] = producer
|
|
1100
|
+
_append_event(next_state, "build_resumed" if is_resume else "build_started", "build")
|
|
1101
|
+
_write_state(workspace, next_state)
|
|
1102
|
+
state = next_state
|
|
1103
|
+
try:
|
|
1104
|
+
build_candidate(
|
|
1105
|
+
inputs.plan,
|
|
1106
|
+
input_root=workspace_path / state["paths"]["input_root"],
|
|
1107
|
+
output_dir=result_path,
|
|
1108
|
+
producer_principal=producer_principal,
|
|
1109
|
+
producer_attempt=producer_attempt,
|
|
1110
|
+
package_context=inputs.package_context,
|
|
1111
|
+
visual_store=visual_store,
|
|
1112
|
+
)
|
|
1113
|
+
except (BuildError, ContractError):
|
|
1114
|
+
failed = deepcopy(state)
|
|
1115
|
+
_task(failed, "build")["status"] = "failed"
|
|
1116
|
+
failed["phase"] = "failed"
|
|
1117
|
+
_append_event(failed, "build_failed", "build")
|
|
1118
|
+
_write_state(workspace, failed)
|
|
1119
|
+
raise
|
|
1120
|
+
if fault_hook is not None:
|
|
1121
|
+
fault_hook("after_candidate_sealed")
|
|
1122
|
+
inspection = _inspect_and_match_state(
|
|
1123
|
+
result_path,
|
|
1124
|
+
state,
|
|
1125
|
+
producer,
|
|
1126
|
+
workspace=workspace,
|
|
1127
|
+
)
|
|
1128
|
+
completed = _candidate_state(
|
|
1129
|
+
state,
|
|
1130
|
+
inspection,
|
|
1131
|
+
producer=producer,
|
|
1132
|
+
event="candidate_built",
|
|
1133
|
+
)
|
|
1134
|
+
_write_state(workspace, completed)
|
|
1135
|
+
return OfflineExecutionResult(
|
|
1136
|
+
status="candidate_built",
|
|
1137
|
+
run_id=completed["run_id"],
|
|
1138
|
+
candidate=inspection,
|
|
1139
|
+
workspace=workspace_path,
|
|
1140
|
+
recovered=False,
|
|
1141
|
+
idempotent=False,
|
|
1142
|
+
)
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def _candidate_state(
|
|
1146
|
+
state: dict[str, Any],
|
|
1147
|
+
inspection: CandidateInspection,
|
|
1148
|
+
*,
|
|
1149
|
+
producer: dict[str, str],
|
|
1150
|
+
event: str,
|
|
1151
|
+
) -> dict[str, Any]:
|
|
1152
|
+
if inspection.manifest_version not in _OFFLINE_MANIFEST_VERSIONS:
|
|
1153
|
+
raise OfflineRunError(
|
|
1154
|
+
"CANDIDATE_VERSION_INVALID",
|
|
1155
|
+
"workspace result is not a complete H2 offline package",
|
|
1156
|
+
)
|
|
1157
|
+
next_state = deepcopy(state)
|
|
1158
|
+
next_state["producer"] = producer
|
|
1159
|
+
next_state["candidate"] = {
|
|
1160
|
+
"candidate_digest": inspection.candidate_digest,
|
|
1161
|
+
"table_sha256": inspection.table_sha256,
|
|
1162
|
+
"manifest_version": inspection.manifest_version,
|
|
1163
|
+
"row_count": inspection.row_count,
|
|
1164
|
+
}
|
|
1165
|
+
next_state["phase"] = "candidate_built"
|
|
1166
|
+
_task(next_state, "build").update(
|
|
1167
|
+
{
|
|
1168
|
+
"status": "completed",
|
|
1169
|
+
"result_digest": inspection.candidate_digest,
|
|
1170
|
+
}
|
|
1171
|
+
)
|
|
1172
|
+
_append_event(next_state, event, "build")
|
|
1173
|
+
return next_state
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
def _inspect_and_match_state(
|
|
1177
|
+
result_path: Path,
|
|
1178
|
+
state: dict[str, Any],
|
|
1179
|
+
producer: dict[str, str],
|
|
1180
|
+
*,
|
|
1181
|
+
workspace: _Workspace | None = None,
|
|
1182
|
+
) -> CandidateInspection:
|
|
1183
|
+
inspection, lease = _inspect_and_retain_matched_state(
|
|
1184
|
+
result_path,
|
|
1185
|
+
state,
|
|
1186
|
+
producer,
|
|
1187
|
+
workspace=workspace,
|
|
1188
|
+
)
|
|
1189
|
+
try:
|
|
1190
|
+
return inspection
|
|
1191
|
+
finally:
|
|
1192
|
+
lease.close()
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _inspect_and_retain_matched_state(
|
|
1196
|
+
result_path: Path,
|
|
1197
|
+
state: dict[str, Any],
|
|
1198
|
+
producer: dict[str, str],
|
|
1199
|
+
*,
|
|
1200
|
+
workspace: _Workspace | None = None,
|
|
1201
|
+
) -> tuple[CandidateInspection, Any]:
|
|
1202
|
+
retained_snapshots: list[Any] = []
|
|
1203
|
+
inspection = inspect_candidate(
|
|
1204
|
+
result_path,
|
|
1205
|
+
_run_parent_fd=workspace.root_fd if workspace is not None else None,
|
|
1206
|
+
_ancestor_validator=workspace.validate if workspace is not None else None,
|
|
1207
|
+
_retained_snapshots=retained_snapshots,
|
|
1208
|
+
)
|
|
1209
|
+
if len(retained_snapshots) != 1:
|
|
1210
|
+
for snapshot in retained_snapshots:
|
|
1211
|
+
snapshot.close()
|
|
1212
|
+
raise OfflineRunError(
|
|
1213
|
+
"CANDIDATE_SNAPSHOT_INVALID",
|
|
1214
|
+
"candidate inspection did not retain its exact bound snapshot",
|
|
1215
|
+
)
|
|
1216
|
+
verified_snapshot = retained_snapshots[0]
|
|
1217
|
+
try:
|
|
1218
|
+
# Deliberate repeat, not a duplicate. The comparison runs once before the workspace
|
|
1219
|
+
# and lease identity checks and once between two rounds of them, so the accepted
|
|
1220
|
+
# comparison is bracketed by identity checks on both sides: a workspace or candidate
|
|
1221
|
+
# root swapped in during either pass fails the surrounding check.
|
|
1222
|
+
_match_candidate_inspection(inspection, state, producer)
|
|
1223
|
+
if workspace is not None:
|
|
1224
|
+
workspace.validate()
|
|
1225
|
+
verified_snapshot.validate()
|
|
1226
|
+
_match_candidate_inspection(inspection, state, producer)
|
|
1227
|
+
if workspace is not None:
|
|
1228
|
+
workspace.validate()
|
|
1229
|
+
verified_snapshot.validate()
|
|
1230
|
+
return inspection, verified_snapshot
|
|
1231
|
+
except Exception:
|
|
1232
|
+
verified_snapshot.close()
|
|
1233
|
+
raise
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def _match_candidate_inspection(
|
|
1237
|
+
inspection: CandidateInspection,
|
|
1238
|
+
state: dict[str, Any],
|
|
1239
|
+
producer: dict[str, str],
|
|
1240
|
+
) -> None:
|
|
1241
|
+
if (
|
|
1242
|
+
inspection.producer_principal != producer["principal_id"]
|
|
1243
|
+
or inspection.producer_attempt != producer["attempt_id"]
|
|
1244
|
+
or inspection.manifest_version not in _OFFLINE_MANIFEST_VERSIONS
|
|
1245
|
+
):
|
|
1246
|
+
raise OfflineRunError(
|
|
1247
|
+
"CANDIDATE_PRODUCER_MISMATCH",
|
|
1248
|
+
"existing candidate is not the expected H2 producer result",
|
|
1249
|
+
)
|
|
1250
|
+
recorded = state["candidate"]
|
|
1251
|
+
if recorded is not None and (
|
|
1252
|
+
recorded["candidate_digest"] != inspection.candidate_digest
|
|
1253
|
+
or recorded["table_sha256"] != inspection.table_sha256
|
|
1254
|
+
or recorded["row_count"] != inspection.row_count
|
|
1255
|
+
):
|
|
1256
|
+
raise OfflineRunError(
|
|
1257
|
+
"CANDIDATE_STATE_MISMATCH",
|
|
1258
|
+
"verified candidate does not match durable run state",
|
|
1259
|
+
)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def _populate_workspace(
|
|
1263
|
+
staging_fd: int,
|
|
1264
|
+
*,
|
|
1265
|
+
definition_bytes: dict[str, bytes],
|
|
1266
|
+
source_bytes: dict[str, bytes],
|
|
1267
|
+
plan: TablePlan,
|
|
1268
|
+
state: dict[str, Any],
|
|
1269
|
+
) -> None:
|
|
1270
|
+
directories: dict[str, int] = {"": staging_fd}
|
|
1271
|
+
opened_fds: list[int] = []
|
|
1272
|
+
try:
|
|
1273
|
+
for path in (CONTROL_PATH, DEFINITION_PATH, INPUT_ROOT_PATH):
|
|
1274
|
+
descriptor = _create_directory_at(staging_fd, path)
|
|
1275
|
+
directories[path] = descriptor
|
|
1276
|
+
opened_fds.append(descriptor)
|
|
1277
|
+
_create_file_at(
|
|
1278
|
+
directories[CONTROL_PATH],
|
|
1279
|
+
WORKSPACE_LOCK_NAME,
|
|
1280
|
+
b"",
|
|
1281
|
+
mode=0o600,
|
|
1282
|
+
)
|
|
1283
|
+
_create_file_at(
|
|
1284
|
+
directories[CONTROL_PATH],
|
|
1285
|
+
_STATE_NAME,
|
|
1286
|
+
canonical_json_bytes(state),
|
|
1287
|
+
mode=0o600,
|
|
1288
|
+
)
|
|
1289
|
+
definition_files = dict(_DEFINITION_FILES)
|
|
1290
|
+
if "semantics" in definition_bytes:
|
|
1291
|
+
definition_files.update(_SEMANTICS_DEFINITION)
|
|
1292
|
+
for logical_name, file_name in definition_files.items():
|
|
1293
|
+
_create_file_at(
|
|
1294
|
+
directories[DEFINITION_PATH],
|
|
1295
|
+
file_name,
|
|
1296
|
+
definition_bytes[logical_name],
|
|
1297
|
+
mode=0o444,
|
|
1298
|
+
)
|
|
1299
|
+
source_by_id = {source.source_id: source for source in plan.sources}
|
|
1300
|
+
for source_id in sorted(source_bytes):
|
|
1301
|
+
source = source_by_id[source_id]
|
|
1302
|
+
parent_fd, newly_opened = _create_relative_parents(
|
|
1303
|
+
directories[INPUT_ROOT_PATH],
|
|
1304
|
+
PurePosixPath(source.path).parts[:-1],
|
|
1305
|
+
)
|
|
1306
|
+
opened_fds.extend(newly_opened)
|
|
1307
|
+
_create_file_at(
|
|
1308
|
+
parent_fd,
|
|
1309
|
+
PurePosixPath(source.path).name,
|
|
1310
|
+
source_bytes[source_id],
|
|
1311
|
+
mode=0o444,
|
|
1312
|
+
)
|
|
1313
|
+
for descriptor in reversed(opened_fds):
|
|
1314
|
+
if descriptor not in {
|
|
1315
|
+
directories[CONTROL_PATH],
|
|
1316
|
+
directories[DEFINITION_PATH],
|
|
1317
|
+
directories[INPUT_ROOT_PATH],
|
|
1318
|
+
}:
|
|
1319
|
+
os.fchmod(descriptor, 0o555)
|
|
1320
|
+
os.fsync(descriptor)
|
|
1321
|
+
os.fchmod(directories[DEFINITION_PATH], 0o555)
|
|
1322
|
+
os.fchmod(directories[INPUT_ROOT_PATH], 0o555)
|
|
1323
|
+
for descriptor in {
|
|
1324
|
+
staging_fd,
|
|
1325
|
+
directories[CONTROL_PATH],
|
|
1326
|
+
directories[DEFINITION_PATH],
|
|
1327
|
+
directories[INPUT_ROOT_PATH],
|
|
1328
|
+
}:
|
|
1329
|
+
os.fsync(descriptor)
|
|
1330
|
+
finally:
|
|
1331
|
+
for descriptor in reversed(opened_fds):
|
|
1332
|
+
os.close(descriptor)
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def _create_relative_parents(
|
|
1336
|
+
root_fd: int,
|
|
1337
|
+
parts: tuple[str, ...],
|
|
1338
|
+
) -> tuple[int, list[int]]:
|
|
1339
|
+
parent_fd = root_fd
|
|
1340
|
+
opened: list[int] = []
|
|
1341
|
+
for part in parts:
|
|
1342
|
+
if part in {"", ".", ".."}:
|
|
1343
|
+
raise OfflineRunError("SOURCE_PATH_INVALID", "source path is not canonical")
|
|
1344
|
+
try:
|
|
1345
|
+
os.mkdir(part, 0o700, dir_fd=parent_fd)
|
|
1346
|
+
except FileExistsError:
|
|
1347
|
+
pass
|
|
1348
|
+
descriptor = os.open(
|
|
1349
|
+
part,
|
|
1350
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
1351
|
+
dir_fd=parent_fd,
|
|
1352
|
+
)
|
|
1353
|
+
opened.append(descriptor)
|
|
1354
|
+
parent_fd = descriptor
|
|
1355
|
+
return parent_fd, opened
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def _create_directory_at(parent_fd: int, name: str) -> int:
|
|
1359
|
+
os.mkdir(name, 0o700, dir_fd=parent_fd)
|
|
1360
|
+
named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
1361
|
+
descriptor = os.open(
|
|
1362
|
+
name,
|
|
1363
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
1364
|
+
dir_fd=parent_fd,
|
|
1365
|
+
)
|
|
1366
|
+
opened = os.fstat(descriptor)
|
|
1367
|
+
if not stat.S_ISDIR(named.st_mode) or _entry_identity(named) != _entry_identity(opened):
|
|
1368
|
+
os.close(descriptor)
|
|
1369
|
+
raise OfflineRunError("WORKSPACE_INIT_RACE", f"directory changed identity: {name}")
|
|
1370
|
+
return descriptor
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _create_file_at(parent_fd: int, name: str, raw: bytes, *, mode: int) -> None:
|
|
1374
|
+
descriptor = os.open(
|
|
1375
|
+
name,
|
|
1376
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
1377
|
+
0o600,
|
|
1378
|
+
dir_fd=parent_fd,
|
|
1379
|
+
)
|
|
1380
|
+
try:
|
|
1381
|
+
offset = 0
|
|
1382
|
+
while offset < len(raw):
|
|
1383
|
+
written = os.write(descriptor, raw[offset:])
|
|
1384
|
+
if written < 1:
|
|
1385
|
+
raise OfflineRunError("WORKSPACE_WRITE_FAILED", f"write stalled: {name}")
|
|
1386
|
+
offset += written
|
|
1387
|
+
os.fsync(descriptor)
|
|
1388
|
+
os.fchmod(descriptor, mode)
|
|
1389
|
+
os.fsync(descriptor)
|
|
1390
|
+
opened = os.fstat(descriptor)
|
|
1391
|
+
named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
1392
|
+
if (
|
|
1393
|
+
not stat.S_ISREG(opened.st_mode)
|
|
1394
|
+
or opened.st_nlink != 1
|
|
1395
|
+
or _stat_signature(opened) != _stat_signature(named)
|
|
1396
|
+
):
|
|
1397
|
+
raise OfflineRunError("WORKSPACE_INIT_RACE", f"file changed identity: {name}")
|
|
1398
|
+
finally:
|
|
1399
|
+
os.close(descriptor)
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
class _WorkspaceContext:
|
|
1403
|
+
def __init__(self, workspace: _Workspace) -> None:
|
|
1404
|
+
self.workspace = workspace
|
|
1405
|
+
|
|
1406
|
+
def __enter__(self) -> _Workspace:
|
|
1407
|
+
return self.workspace
|
|
1408
|
+
|
|
1409
|
+
def __exit__(self, _exc_type: Any, _exc: Any, _traceback: Any) -> None:
|
|
1410
|
+
self.workspace.close()
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def validate_workspace_structure(path: Path) -> None:
|
|
1414
|
+
"""Validate a Harness workspace control tree without acquiring its execution lock.
|
|
1415
|
+
|
|
1416
|
+
A successful return proves that the lexical path and its ancestors stayed descriptor-bound,
|
|
1417
|
+
the control directory and single-link ``0600`` lock kept their identities, and one bounded,
|
|
1418
|
+
single-link ``0600`` state file was stable canonical JSON satisfying the current run-state
|
|
1419
|
+
schema. It does not claim the state is current after this function returns, replay immutable
|
|
1420
|
+
definitions, or verify a Build. Callers that need those guarantees must use a locked offline
|
|
1421
|
+
operation instead.
|
|
1422
|
+
|
|
1423
|
+
``OfflineRunError`` or ``BuildError`` reports a refusal. The function never waits for, takes,
|
|
1424
|
+
or releases the workspace execution lock, so a viewer may use it while a build is running.
|
|
1425
|
+
"""
|
|
1426
|
+
|
|
1427
|
+
workspace_path = Path(os.path.abspath(os.fspath(path)))
|
|
1428
|
+
with _open_workspace(workspace_path, acquire_lock=False) as workspace:
|
|
1429
|
+
_read_state(workspace)
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def _open_workspace(
|
|
1433
|
+
path: Path,
|
|
1434
|
+
*,
|
|
1435
|
+
wait: bool = True,
|
|
1436
|
+
acquire_lock: bool = True,
|
|
1437
|
+
) -> _WorkspaceContext:
|
|
1438
|
+
"""Open and validate a Workbench folder, optionally under its durable lock.
|
|
1439
|
+
|
|
1440
|
+
``wait`` is about the lock and nothing else. Every command that acts on a run waits, because
|
|
1441
|
+
joining the queue behind a build is exactly right for a command that is about to change what
|
|
1442
|
+
the build is producing. A command that only *reports* must not wait: a build holds this lock
|
|
1443
|
+
for its whole length, and a reporting command that queues behind one returns no answer at all
|
|
1444
|
+
for as long as the build runs. Nothing else about the reading changes -- the same validation,
|
|
1445
|
+
the same definitions replay, the same candidate match -- so a non-waiting caller is never told
|
|
1446
|
+
less than a waiting one, only told sooner that it could not look.
|
|
1447
|
+
|
|
1448
|
+
``acquire_lock=False`` is the read-only structural-validation mode used only by
|
|
1449
|
+
:func:`validate_workspace_structure`; in that mode ``wait`` has no effect.
|
|
1450
|
+
"""
|
|
1451
|
+
|
|
1452
|
+
if os.name == "nt" or fcntl is None:
|
|
1453
|
+
raise OfflineRunError(
|
|
1454
|
+
"WORKSPACE_PLATFORM_UNSUPPORTED",
|
|
1455
|
+
"descriptor-relative durable workspaces are unavailable on this platform",
|
|
1456
|
+
)
|
|
1457
|
+
run_handle = _open_candidate_run_handle(path)
|
|
1458
|
+
control_fd = -1
|
|
1459
|
+
lock_fd = -1
|
|
1460
|
+
try:
|
|
1461
|
+
try:
|
|
1462
|
+
control_named = os.stat(
|
|
1463
|
+
CONTROL_PATH,
|
|
1464
|
+
dir_fd=run_handle.run_fd,
|
|
1465
|
+
follow_symlinks=False,
|
|
1466
|
+
)
|
|
1467
|
+
except FileNotFoundError as exc:
|
|
1468
|
+
# Pointing a command at a folder that was never initialized is the commonest way to
|
|
1469
|
+
# reach this function, and it is not a fault in the durable record: there is no record.
|
|
1470
|
+
# Letting the raw `FileNotFoundError` out named the control directory -- a path the
|
|
1471
|
+
# caller never typed -- and carried no typed code, so nothing downstream could say what
|
|
1472
|
+
# to do about it.
|
|
1473
|
+
raise OfflineRunError(
|
|
1474
|
+
"WORKSPACE_ABSENT",
|
|
1475
|
+
f"there is {presence_at(path) or UNKNOWN_KIND} at {path}, "
|
|
1476
|
+
"and a workbench has to be a folder this harness set up",
|
|
1477
|
+
) from exc
|
|
1478
|
+
if not stat.S_ISDIR(control_named.st_mode):
|
|
1479
|
+
raise OfflineRunError("WORKSPACE_INVALID", "control path is not a directory")
|
|
1480
|
+
control_fd = os.open(
|
|
1481
|
+
CONTROL_PATH,
|
|
1482
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
|
|
1483
|
+
dir_fd=run_handle.run_fd,
|
|
1484
|
+
)
|
|
1485
|
+
control_identity = _entry_identity(os.fstat(control_fd))
|
|
1486
|
+
if control_identity != _entry_identity(control_named):
|
|
1487
|
+
raise OfflineRunError("WORKSPACE_RACE", "control path changed identity")
|
|
1488
|
+
lock_fd = os.open(
|
|
1489
|
+
WORKSPACE_LOCK_NAME,
|
|
1490
|
+
os.O_RDWR | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
|
|
1491
|
+
dir_fd=control_fd,
|
|
1492
|
+
)
|
|
1493
|
+
lock_info = os.fstat(lock_fd)
|
|
1494
|
+
if (
|
|
1495
|
+
not stat.S_ISREG(lock_info.st_mode)
|
|
1496
|
+
or lock_info.st_nlink != 1
|
|
1497
|
+
or stat.S_IMODE(lock_info.st_mode) != 0o600
|
|
1498
|
+
):
|
|
1499
|
+
raise OfflineRunError("WORKSPACE_INVALID", "workspace lock is invalid")
|
|
1500
|
+
if acquire_lock:
|
|
1501
|
+
if wait:
|
|
1502
|
+
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
|
1503
|
+
else:
|
|
1504
|
+
try:
|
|
1505
|
+
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1506
|
+
except OSError as exc:
|
|
1507
|
+
if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
|
|
1508
|
+
raise
|
|
1509
|
+
raise OfflineRunError(
|
|
1510
|
+
"WORKSPACE_BUSY",
|
|
1511
|
+
f"a build is running in {path}",
|
|
1512
|
+
) from exc
|
|
1513
|
+
workspace = _Workspace(
|
|
1514
|
+
path=path,
|
|
1515
|
+
run_handle=run_handle,
|
|
1516
|
+
control_fd=control_fd,
|
|
1517
|
+
control_identity=control_identity,
|
|
1518
|
+
lock_fd=lock_fd,
|
|
1519
|
+
lock_acquired=acquire_lock,
|
|
1520
|
+
)
|
|
1521
|
+
workspace.validate()
|
|
1522
|
+
return _WorkspaceContext(workspace)
|
|
1523
|
+
except Exception:
|
|
1524
|
+
if lock_fd >= 0:
|
|
1525
|
+
os.close(lock_fd)
|
|
1526
|
+
if control_fd >= 0:
|
|
1527
|
+
os.close(control_fd)
|
|
1528
|
+
run_handle.close()
|
|
1529
|
+
raise
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def _read_state(workspace: _Workspace) -> dict[str, Any]:
|
|
1533
|
+
workspace.validate()
|
|
1534
|
+
raw = _read_regular_file_at(
|
|
1535
|
+
workspace.control_fd,
|
|
1536
|
+
_STATE_NAME,
|
|
1537
|
+
maximum=_MAX_STATE_BYTES,
|
|
1538
|
+
required_mode=0o600,
|
|
1539
|
+
)
|
|
1540
|
+
try:
|
|
1541
|
+
value = parse_canonical_json(raw)
|
|
1542
|
+
except ValueError as exc:
|
|
1543
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run state is not canonical JSON") from exc
|
|
1544
|
+
state = _validate_state(value)
|
|
1545
|
+
workspace.validate()
|
|
1546
|
+
return state
|
|
1547
|
+
|
|
1548
|
+
|
|
1549
|
+
def _write_state(workspace: _Workspace, state: dict[str, Any]) -> None:
|
|
1550
|
+
state["revision"] += 1
|
|
1551
|
+
_validate_state(state)
|
|
1552
|
+
raw = canonical_json_bytes(state)
|
|
1553
|
+
temporary = f".state.tmp-{uuid.uuid4().hex}"
|
|
1554
|
+
workspace.validate()
|
|
1555
|
+
try:
|
|
1556
|
+
_create_file_at(workspace.control_fd, temporary, raw, mode=0o600)
|
|
1557
|
+
os.replace(
|
|
1558
|
+
temporary,
|
|
1559
|
+
_STATE_NAME,
|
|
1560
|
+
src_dir_fd=workspace.control_fd,
|
|
1561
|
+
dst_dir_fd=workspace.control_fd,
|
|
1562
|
+
)
|
|
1563
|
+
os.fsync(workspace.control_fd)
|
|
1564
|
+
workspace.validate()
|
|
1565
|
+
observed = _read_state(workspace)
|
|
1566
|
+
if observed != state:
|
|
1567
|
+
raise OfflineRunError(
|
|
1568
|
+
"RUN_STATE_DURABILITY",
|
|
1569
|
+
"atomic run state does not match the committed successor",
|
|
1570
|
+
)
|
|
1571
|
+
except Exception:
|
|
1572
|
+
try:
|
|
1573
|
+
os.unlink(temporary, dir_fd=workspace.control_fd)
|
|
1574
|
+
except FileNotFoundError:
|
|
1575
|
+
pass
|
|
1576
|
+
raise
|
|
1577
|
+
|
|
1578
|
+
|
|
1579
|
+
def _load_definitions(workspace: _Workspace, state: dict[str, Any]) -> OfflineInputs:
|
|
1580
|
+
workspace.validate()
|
|
1581
|
+
definitions_fd = os.open(
|
|
1582
|
+
DEFINITION_PATH,
|
|
1583
|
+
os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW,
|
|
1584
|
+
dir_fd=workspace.root_fd,
|
|
1585
|
+
)
|
|
1586
|
+
try:
|
|
1587
|
+
info = os.fstat(definitions_fd)
|
|
1588
|
+
if not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o555:
|
|
1589
|
+
raise OfflineRunError("DEFINITIONS_INVALID", "definitions directory is not sealed")
|
|
1590
|
+
definition_files = dict(_DEFINITION_FILES)
|
|
1591
|
+
if "semantics" in state["definition_sha256s"]:
|
|
1592
|
+
definition_files.update(_SEMANTICS_DEFINITION)
|
|
1593
|
+
if set(os.listdir(definitions_fd)) != set(definition_files.values()):
|
|
1594
|
+
raise OfflineRunError("DEFINITIONS_INVALID", "definition member set changed")
|
|
1595
|
+
values: dict[str, Any] = {}
|
|
1596
|
+
for logical_name, file_name in definition_files.items():
|
|
1597
|
+
raw = _read_regular_file_at(
|
|
1598
|
+
definitions_fd,
|
|
1599
|
+
file_name,
|
|
1600
|
+
maximum=_MAX_DEFINITION_BYTES,
|
|
1601
|
+
required_mode=0o444,
|
|
1602
|
+
)
|
|
1603
|
+
if sha256_bytes(raw) != state["definition_sha256s"][logical_name]:
|
|
1604
|
+
raise OfflineRunError(
|
|
1605
|
+
"DEFINITION_DIGEST_MISMATCH",
|
|
1606
|
+
f"definition changed: {logical_name}",
|
|
1607
|
+
)
|
|
1608
|
+
try:
|
|
1609
|
+
values[logical_name] = parse_canonical_json(raw)
|
|
1610
|
+
except ValueError as exc:
|
|
1611
|
+
raise OfflineRunError(
|
|
1612
|
+
"DEFINITIONS_INVALID",
|
|
1613
|
+
f"definition is not canonical: {logical_name}",
|
|
1614
|
+
) from exc
|
|
1615
|
+
finally:
|
|
1616
|
+
os.close(definitions_fd)
|
|
1617
|
+
proposal_set = values["source_proposals"]
|
|
1618
|
+
if not isinstance(proposal_set, dict) or set(proposal_set) != {
|
|
1619
|
+
"schema_version",
|
|
1620
|
+
"source_proposals",
|
|
1621
|
+
}:
|
|
1622
|
+
raise OfflineRunError("DEFINITIONS_INVALID", "source proposal set is invalid")
|
|
1623
|
+
if proposal_set["schema_version"] != SOURCE_PROPOSAL_SET_VERSION:
|
|
1624
|
+
raise OfflineRunError("DEFINITIONS_INVALID", "source proposal set version is invalid")
|
|
1625
|
+
inputs = OfflineInputs(
|
|
1626
|
+
question=parse_question(values["question"]),
|
|
1627
|
+
requirements=parse_requirements(values["requirements"]),
|
|
1628
|
+
source_proposals=tuple(
|
|
1629
|
+
parse_source_proposal(item) for item in proposal_set["source_proposals"]
|
|
1630
|
+
),
|
|
1631
|
+
plan=parse_plan(values["plan"]),
|
|
1632
|
+
semantics=(parse_table_semantics(values["semantics"]) if "semantics" in values else None),
|
|
1633
|
+
)
|
|
1634
|
+
inputs.package_context.validate_for_plan(inputs.plan)
|
|
1635
|
+
workspace.validate()
|
|
1636
|
+
return inputs
|
|
1637
|
+
|
|
1638
|
+
|
|
1639
|
+
def _read_regular_file_at(
|
|
1640
|
+
parent_fd: int,
|
|
1641
|
+
name: str,
|
|
1642
|
+
*,
|
|
1643
|
+
maximum: int,
|
|
1644
|
+
required_mode: int,
|
|
1645
|
+
) -> bytes:
|
|
1646
|
+
named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
1647
|
+
if (
|
|
1648
|
+
not stat.S_ISREG(named.st_mode)
|
|
1649
|
+
or named.st_nlink != 1
|
|
1650
|
+
or stat.S_IMODE(named.st_mode) != required_mode
|
|
1651
|
+
or named.st_size > maximum
|
|
1652
|
+
):
|
|
1653
|
+
raise OfflineRunError("WORKSPACE_FILE_INVALID", f"workspace file is invalid: {name}")
|
|
1654
|
+
descriptor = os.open(
|
|
1655
|
+
name,
|
|
1656
|
+
os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
|
|
1657
|
+
dir_fd=parent_fd,
|
|
1658
|
+
)
|
|
1659
|
+
try:
|
|
1660
|
+
opened = os.fstat(descriptor)
|
|
1661
|
+
if _stat_signature(opened) != _stat_signature(named):
|
|
1662
|
+
raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
|
|
1663
|
+
raw = bytearray()
|
|
1664
|
+
while True:
|
|
1665
|
+
chunk = os.read(descriptor, min(1024 * 1024, maximum + 1 - len(raw)))
|
|
1666
|
+
if not chunk:
|
|
1667
|
+
break
|
|
1668
|
+
raw.extend(chunk)
|
|
1669
|
+
if len(raw) > maximum:
|
|
1670
|
+
raise OfflineRunError("WORKSPACE_FILE_INVALID", f"workspace file too large: {name}")
|
|
1671
|
+
if len(raw) != opened.st_size or _stat_signature(os.fstat(descriptor)) != _stat_signature(
|
|
1672
|
+
opened
|
|
1673
|
+
):
|
|
1674
|
+
raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
|
|
1675
|
+
finally:
|
|
1676
|
+
os.close(descriptor)
|
|
1677
|
+
after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
1678
|
+
if _stat_signature(after) != _stat_signature(named):
|
|
1679
|
+
raise OfflineRunError("WORKSPACE_RACE", f"workspace file changed: {name}")
|
|
1680
|
+
return bytes(raw)
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
def _initial_state(run_id: str, definition_sha256s: dict[str, str]) -> dict[str, Any]:
|
|
1684
|
+
state = {
|
|
1685
|
+
"schema_version": RUN_STATE_VERSION,
|
|
1686
|
+
"run_id": run_id,
|
|
1687
|
+
"revision": 0,
|
|
1688
|
+
"phase": "initialized",
|
|
1689
|
+
"paths": {
|
|
1690
|
+
"input_root": INPUT_ROOT_PATH,
|
|
1691
|
+
"result": RUN_RESULT_PATH,
|
|
1692
|
+
},
|
|
1693
|
+
"definition_sha256s": definition_sha256s,
|
|
1694
|
+
"producer": None,
|
|
1695
|
+
"candidate": None,
|
|
1696
|
+
"review": None,
|
|
1697
|
+
"tasks": [
|
|
1698
|
+
{
|
|
1699
|
+
"task_id": task_id,
|
|
1700
|
+
"status": "pending",
|
|
1701
|
+
"attempts": 0,
|
|
1702
|
+
"result_digest": None,
|
|
1703
|
+
}
|
|
1704
|
+
for task_id in _TASK_IDS
|
|
1705
|
+
],
|
|
1706
|
+
"events": [],
|
|
1707
|
+
}
|
|
1708
|
+
_append_event(state, "run_initialized", None)
|
|
1709
|
+
return _validate_state(state)
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
def _validate_state(value: Any) -> dict[str, Any]:
|
|
1713
|
+
expected = {
|
|
1714
|
+
"schema_version",
|
|
1715
|
+
"run_id",
|
|
1716
|
+
"revision",
|
|
1717
|
+
"phase",
|
|
1718
|
+
"paths",
|
|
1719
|
+
"definition_sha256s",
|
|
1720
|
+
"producer",
|
|
1721
|
+
"candidate",
|
|
1722
|
+
"review",
|
|
1723
|
+
"tasks",
|
|
1724
|
+
"events",
|
|
1725
|
+
}
|
|
1726
|
+
if not isinstance(value, dict) or set(value) != expected:
|
|
1727
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run state fields are invalid")
|
|
1728
|
+
if value["schema_version"] != RUN_STATE_VERSION:
|
|
1729
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run state version is unsupported")
|
|
1730
|
+
_identifier(value["run_id"], "run_id")
|
|
1731
|
+
if type(value["revision"]) is not int or value["revision"] < 0:
|
|
1732
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run revision is invalid")
|
|
1733
|
+
if value["phase"] not in _PHASES:
|
|
1734
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run phase is invalid")
|
|
1735
|
+
if value["paths"] != {"input_root": INPUT_ROOT_PATH, "result": RUN_RESULT_PATH}:
|
|
1736
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run paths are not the closed H2 paths")
|
|
1737
|
+
digests = value["definition_sha256s"]
|
|
1738
|
+
valid_definition_sets = {
|
|
1739
|
+
frozenset(_DEFINITION_FILES),
|
|
1740
|
+
frozenset({*_DEFINITION_FILES, *_SEMANTICS_DEFINITION}),
|
|
1741
|
+
}
|
|
1742
|
+
if not isinstance(digests, dict) or frozenset(digests) not in valid_definition_sets:
|
|
1743
|
+
raise OfflineRunError("RUN_STATE_INVALID", "definition digests are invalid")
|
|
1744
|
+
for name, digest in digests.items():
|
|
1745
|
+
_digest(digest, f"definition_sha256s.{name}")
|
|
1746
|
+
producer = value["producer"]
|
|
1747
|
+
if producer is not None:
|
|
1748
|
+
if not isinstance(producer, dict) or set(producer) != {"principal_id", "attempt_id"}:
|
|
1749
|
+
raise OfflineRunError("RUN_STATE_INVALID", "producer record is invalid")
|
|
1750
|
+
_identifier(producer["principal_id"], "producer.principal_id")
|
|
1751
|
+
_identifier(producer["attempt_id"], "producer.attempt_id")
|
|
1752
|
+
if producer != _h2_producer_identity():
|
|
1753
|
+
raise OfflineRunError(
|
|
1754
|
+
"RUN_STATE_INVALID",
|
|
1755
|
+
"run producer is not the fixed coordinator-owned H2 identity",
|
|
1756
|
+
)
|
|
1757
|
+
candidate = value["candidate"]
|
|
1758
|
+
if candidate is not None:
|
|
1759
|
+
if not isinstance(candidate, dict) or set(candidate) != {
|
|
1760
|
+
"candidate_digest",
|
|
1761
|
+
"table_sha256",
|
|
1762
|
+
"manifest_version",
|
|
1763
|
+
"row_count",
|
|
1764
|
+
}:
|
|
1765
|
+
raise OfflineRunError("RUN_STATE_INVALID", "candidate record is invalid")
|
|
1766
|
+
_digest(candidate["candidate_digest"], "candidate.candidate_digest")
|
|
1767
|
+
_digest(candidate["table_sha256"], "candidate.table_sha256")
|
|
1768
|
+
if candidate["manifest_version"] not in _OFFLINE_MANIFEST_VERSIONS:
|
|
1769
|
+
raise OfflineRunError("RUN_STATE_INVALID", "candidate manifest version is invalid")
|
|
1770
|
+
if type(candidate["row_count"]) is not int or candidate["row_count"] < 1:
|
|
1771
|
+
raise OfflineRunError("RUN_STATE_INVALID", "candidate row count is invalid")
|
|
1772
|
+
review = value["review"]
|
|
1773
|
+
if review is not None:
|
|
1774
|
+
if not isinstance(review, dict) or set(review) != {
|
|
1775
|
+
"candidate_digest",
|
|
1776
|
+
"decision_digest",
|
|
1777
|
+
"verified_status",
|
|
1778
|
+
}:
|
|
1779
|
+
raise OfflineRunError("RUN_STATE_INVALID", "review record is invalid")
|
|
1780
|
+
_digest(review["candidate_digest"], "review.candidate_digest")
|
|
1781
|
+
_digest(review["decision_digest"], "review.decision_digest")
|
|
1782
|
+
if review["verified_status"] not in {"reviewed_candidate", "fixes_required"}:
|
|
1783
|
+
raise OfflineRunError("RUN_STATE_INVALID", "review status is invalid")
|
|
1784
|
+
tasks = value["tasks"]
|
|
1785
|
+
if not isinstance(tasks, list) or len(tasks) != len(_TASK_IDS):
|
|
1786
|
+
raise OfflineRunError("RUN_STATE_INVALID", "run tasks are invalid")
|
|
1787
|
+
for index, task in enumerate(tasks):
|
|
1788
|
+
if not isinstance(task, dict) or set(task) != {
|
|
1789
|
+
"task_id",
|
|
1790
|
+
"status",
|
|
1791
|
+
"attempts",
|
|
1792
|
+
"result_digest",
|
|
1793
|
+
}:
|
|
1794
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"task {index} is invalid")
|
|
1795
|
+
if task["task_id"] != _TASK_IDS[index] or task["status"] not in _TASK_STATUSES:
|
|
1796
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"task {index} identity/status is invalid")
|
|
1797
|
+
if type(task["attempts"]) is not int or task["attempts"] < 0:
|
|
1798
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"task {index} attempts are invalid")
|
|
1799
|
+
if task["result_digest"] is not None:
|
|
1800
|
+
_digest(task["result_digest"], f"tasks[{index}].result_digest")
|
|
1801
|
+
if task["status"] == "pending" and (
|
|
1802
|
+
task["attempts"] != 0 or task["result_digest"] is not None
|
|
1803
|
+
):
|
|
1804
|
+
raise OfflineRunError(
|
|
1805
|
+
"RUN_STATE_INVALID",
|
|
1806
|
+
f"pending task {index} has execution state",
|
|
1807
|
+
)
|
|
1808
|
+
if task["status"] in {"running", "failed"} and (
|
|
1809
|
+
task["attempts"] < 1 or task["result_digest"] is not None
|
|
1810
|
+
):
|
|
1811
|
+
raise OfflineRunError(
|
|
1812
|
+
"RUN_STATE_INVALID",
|
|
1813
|
+
f"active task {index} has invalid execution state",
|
|
1814
|
+
)
|
|
1815
|
+
if task["status"] == "completed" and (
|
|
1816
|
+
task["attempts"] < 1 or task["result_digest"] is None
|
|
1817
|
+
):
|
|
1818
|
+
raise OfflineRunError(
|
|
1819
|
+
"RUN_STATE_INVALID",
|
|
1820
|
+
f"completed task {index} lacks its result",
|
|
1821
|
+
)
|
|
1822
|
+
task_statuses = tuple(task["status"] for task in tasks)
|
|
1823
|
+
if task_statuses != _PHASE_TASK_STATUSES[value["phase"]]:
|
|
1824
|
+
raise OfflineRunError(
|
|
1825
|
+
"RUN_STATE_INVALID",
|
|
1826
|
+
"run phase and task statuses are inconsistent",
|
|
1827
|
+
)
|
|
1828
|
+
events = value["events"]
|
|
1829
|
+
if not isinstance(events, list):
|
|
1830
|
+
raise OfflineRunError("RUN_STATE_INVALID", "events must be an array")
|
|
1831
|
+
for index, event in enumerate(events, start=1):
|
|
1832
|
+
if not isinstance(event, dict) or set(event) != {
|
|
1833
|
+
"sequence",
|
|
1834
|
+
"event",
|
|
1835
|
+
"task_id",
|
|
1836
|
+
"phase",
|
|
1837
|
+
}:
|
|
1838
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"event {index} is invalid")
|
|
1839
|
+
if event["sequence"] != index or event["phase"] not in _PHASES:
|
|
1840
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"event {index} ordering is invalid")
|
|
1841
|
+
_identifier(event["event"], f"events[{index}].event")
|
|
1842
|
+
if event["task_id"] is not None and event["task_id"] not in _TASK_IDS:
|
|
1843
|
+
raise OfflineRunError("RUN_STATE_INVALID", f"event {index} task is invalid")
|
|
1844
|
+
build_task = tasks[_TASK_IDS.index("build")]
|
|
1845
|
+
review_task = tasks[_TASK_IDS.index("review")]
|
|
1846
|
+
producer_required = value["phase"] not in {"initialized", "planned"}
|
|
1847
|
+
if (producer is not None) != producer_required:
|
|
1848
|
+
raise OfflineRunError(
|
|
1849
|
+
"RUN_STATE_INVALID",
|
|
1850
|
+
"run phase and producer enrollment are inconsistent",
|
|
1851
|
+
)
|
|
1852
|
+
candidate_required = value["phase"] in {"candidate_built", "review_recorded"}
|
|
1853
|
+
if (candidate is not None) != candidate_required:
|
|
1854
|
+
raise OfflineRunError(
|
|
1855
|
+
"RUN_STATE_INVALID",
|
|
1856
|
+
"run phase and candidate state are inconsistent",
|
|
1857
|
+
)
|
|
1858
|
+
review_required = value["phase"] == "review_recorded"
|
|
1859
|
+
if (review is not None) != review_required:
|
|
1860
|
+
raise OfflineRunError(
|
|
1861
|
+
"RUN_STATE_INVALID",
|
|
1862
|
+
"run phase and review state are inconsistent",
|
|
1863
|
+
)
|
|
1864
|
+
if candidate is not None and producer is None:
|
|
1865
|
+
raise OfflineRunError(
|
|
1866
|
+
"RUN_STATE_INVALID",
|
|
1867
|
+
"candidate state requires a producer fence",
|
|
1868
|
+
)
|
|
1869
|
+
if build_task["status"] == "completed" and candidate is None:
|
|
1870
|
+
raise OfflineRunError(
|
|
1871
|
+
"RUN_STATE_INVALID",
|
|
1872
|
+
"completed build task requires candidate state",
|
|
1873
|
+
)
|
|
1874
|
+
if value["phase"] in {"candidate_built", "review_recorded"} and candidate is None:
|
|
1875
|
+
raise OfflineRunError(
|
|
1876
|
+
"RUN_STATE_INVALID",
|
|
1877
|
+
"terminal candidate phase requires candidate state",
|
|
1878
|
+
)
|
|
1879
|
+
if candidate is not None and build_task["result_digest"] != candidate["candidate_digest"]:
|
|
1880
|
+
raise OfflineRunError(
|
|
1881
|
+
"RUN_STATE_INVALID",
|
|
1882
|
+
"completed build result does not match candidate state",
|
|
1883
|
+
)
|
|
1884
|
+
if review is not None:
|
|
1885
|
+
if candidate is None or review["candidate_digest"] != candidate["candidate_digest"]:
|
|
1886
|
+
raise OfflineRunError(
|
|
1887
|
+
"RUN_STATE_INVALID",
|
|
1888
|
+
"review record must reference the durable candidate",
|
|
1889
|
+
)
|
|
1890
|
+
if review_task["status"] != "completed":
|
|
1891
|
+
raise OfflineRunError(
|
|
1892
|
+
"RUN_STATE_INVALID",
|
|
1893
|
+
"review record requires a completed review task",
|
|
1894
|
+
)
|
|
1895
|
+
if value["phase"] == "review_recorded" and review is None:
|
|
1896
|
+
raise OfflineRunError(
|
|
1897
|
+
"RUN_STATE_INVALID",
|
|
1898
|
+
"review_recorded phase requires a review record",
|
|
1899
|
+
)
|
|
1900
|
+
if review is not None and review_task["result_digest"] != review["decision_digest"]:
|
|
1901
|
+
raise OfflineRunError(
|
|
1902
|
+
"RUN_STATE_INVALID",
|
|
1903
|
+
"completed review result does not match review state",
|
|
1904
|
+
)
|
|
1905
|
+
return value
|
|
1906
|
+
|
|
1907
|
+
|
|
1908
|
+
def _append_event(state: dict[str, Any], event: str, task_id: str | None) -> None:
|
|
1909
|
+
state["events"].append(
|
|
1910
|
+
{
|
|
1911
|
+
"sequence": len(state["events"]) + 1,
|
|
1912
|
+
"event": _identifier(event, "event"),
|
|
1913
|
+
"task_id": task_id,
|
|
1914
|
+
"phase": state["phase"],
|
|
1915
|
+
}
|
|
1916
|
+
)
|
|
1917
|
+
|
|
1918
|
+
|
|
1919
|
+
def _task(state: dict[str, Any], task_id: str) -> dict[str, Any]:
|
|
1920
|
+
return state["tasks"][_TASK_IDS.index(task_id)]
|
|
1921
|
+
|
|
1922
|
+
|
|
1923
|
+
def _status_payload(state: dict[str, Any], workspace: Path) -> dict[str, Any]:
|
|
1924
|
+
review = state["review"]
|
|
1925
|
+
return {
|
|
1926
|
+
"status": state["phase"],
|
|
1927
|
+
"run_id": state["run_id"],
|
|
1928
|
+
"workspace": str(workspace),
|
|
1929
|
+
"revision": state["revision"],
|
|
1930
|
+
"tasks": deepcopy(state["tasks"]),
|
|
1931
|
+
"events": deepcopy(state["events"]),
|
|
1932
|
+
"candidate": deepcopy(state["candidate"]),
|
|
1933
|
+
"review_record": deepcopy(review),
|
|
1934
|
+
"review_authority": (
|
|
1935
|
+
"record_only_reverify_with_enrolled_public_key"
|
|
1936
|
+
if review is not None
|
|
1937
|
+
else "not_reviewed"
|
|
1938
|
+
),
|
|
1939
|
+
# The local coordinator cannot authorize hosted release or verify a repaired successor.
|
|
1940
|
+
"hosted_release": False,
|
|
1941
|
+
"successor_repair_closure": False,
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def _remove_orphan_staging(workspace: _Workspace) -> None:
|
|
1946
|
+
workspace.validate()
|
|
1947
|
+
for child in os.listdir(workspace.root_fd):
|
|
1948
|
+
if not _ORPHAN_STAGING.fullmatch(child):
|
|
1949
|
+
continue
|
|
1950
|
+
named = os.stat(child, dir_fd=workspace.root_fd, follow_symlinks=False)
|
|
1951
|
+
if stat.S_ISDIR(named.st_mode):
|
|
1952
|
+
_remove_tree(
|
|
1953
|
+
workspace.path / child,
|
|
1954
|
+
expected_identity=_entry_identity(named),
|
|
1955
|
+
parent_fd=workspace.root_fd,
|
|
1956
|
+
)
|
|
1957
|
+
workspace.validate()
|
|
1958
|
+
|
|
1959
|
+
|
|
1960
|
+
def _named_child_exists(parent_fd: int, name: str) -> bool:
|
|
1961
|
+
try:
|
|
1962
|
+
os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
1963
|
+
except FileNotFoundError:
|
|
1964
|
+
return False
|
|
1965
|
+
return True
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
def _read_external_json(path: Path, label: str) -> Any:
|
|
1969
|
+
"""One of the four files a job is described in, read strictly.
|
|
1970
|
+
|
|
1971
|
+
Each refusal names the file twice over: by what it is for (`question`, `plan`) and by where it
|
|
1972
|
+
is, because a caller passes four paths on one command line and "one of them is not valid JSON"
|
|
1973
|
+
is not an answer. The strict reader's own detail -- the line and column -- is carried through
|
|
1974
|
+
for the same reason: it is the half that says where to look.
|
|
1975
|
+
|
|
1976
|
+
The read goes through :mod:`ux.plain_file` -- open once, ask the descriptor what it is, read
|
|
1977
|
+
that descriptor under a bound -- because ``path.read_bytes()`` on a path somebody typed is the
|
|
1978
|
+
recurring defect class shared with the authoring, preview, and command-line document readers.
|
|
1979
|
+
Every one of the four paths ``mr-data init`` takes comes through this function, so an unguarded
|
|
1980
|
+
``--question <a fifo>`` would open the pipe
|
|
1981
|
+
and waited for a writer that never came: no output, no exit, and nothing created. The bound was
|
|
1982
|
+
real but arrived too late -- ``read_bytes`` had already read the whole file before anything
|
|
1983
|
+
measured it -- so a file far past the limit was held in memory to be refused.
|
|
1984
|
+
"""
|
|
1985
|
+
|
|
1986
|
+
try:
|
|
1987
|
+
raw = read_plain_file(path, max_bytes=_MAX_DEFINITION_BYTES)
|
|
1988
|
+
except PlainFileRefusal as refusal:
|
|
1989
|
+
if refusal.reason == TOO_LARGE:
|
|
1990
|
+
raise OfflineRunError(
|
|
1991
|
+
"DEFINITION_TOO_LARGE", f"{label} exceeds the byte limit: {path}"
|
|
1992
|
+
) from None
|
|
1993
|
+
# A pipe, a socket or a device is the one refusal `DEFINITION_UNREADABLE` has no words for:
|
|
1994
|
+
# its remediation says to check the file is there, is a plain file rather than a folder,
|
|
1995
|
+
# and is readable, and all three of those are true of a FIFO nobody is writing to. A folder
|
|
1996
|
+
# and a missing path keep that code, because that is the sentence written for them.
|
|
1997
|
+
if refusal.reason == DANGLING:
|
|
1998
|
+
# A dangling link needs its own refusal: generic readability advice would claim the
|
|
1999
|
+
# named entry is absent or unreadable without identifying the link. The noun comes
|
|
2000
|
+
# from the mode this rule observed.
|
|
2001
|
+
raise OfflineRunError(
|
|
2002
|
+
"DEFINITION_TARGET_DANGLING",
|
|
2003
|
+
f"{label} is at {refusal.path}, which is "
|
|
2004
|
+
f"{name_of_kind(refusal.mode) or UNKNOWN_KIND} "
|
|
2005
|
+
"that leads nowhere",
|
|
2006
|
+
) from None
|
|
2007
|
+
if refusal.reason == NOT_PLAIN and not stat.S_ISDIR(refusal.mode or 0):
|
|
2008
|
+
raise OfflineRunError(
|
|
2009
|
+
"DEFINITION_TARGET_NOT_A_FILE",
|
|
2010
|
+
f"{label} is not a file: {path}",
|
|
2011
|
+
) from None
|
|
2012
|
+
raise OfflineRunError("DEFINITION_UNREADABLE", f"{label} cannot be read: {path}") from None
|
|
2013
|
+
except OSError as exc:
|
|
2014
|
+
raise OfflineRunError("DEFINITION_UNREADABLE", f"{label} cannot be read: {path}") from exc
|
|
2015
|
+
try:
|
|
2016
|
+
return parse_json(raw)
|
|
2017
|
+
except CanonicalJSONError as exc:
|
|
2018
|
+
raise OfflineRunError(
|
|
2019
|
+
"DEFINITION_INVALID", f"{label} is not strict JSON ({exc.detail}): {path}"
|
|
2020
|
+
) from exc
|
|
2021
|
+
except ValueError as exc:
|
|
2022
|
+
raise OfflineRunError("DEFINITION_INVALID", f"{label} is not strict JSON: {path}") from exc
|
|
2023
|
+
|
|
2024
|
+
|
|
2025
|
+
def _identifier(value: Any, name: str) -> str:
|
|
2026
|
+
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
|
|
2027
|
+
raise OfflineRunError(
|
|
2028
|
+
"IDENTIFIER_INVALID",
|
|
2029
|
+
f"{name} must be a bounded alphanumeric, '-' or '_' identifier",
|
|
2030
|
+
)
|
|
2031
|
+
return value
|
|
2032
|
+
|
|
2033
|
+
|
|
2034
|
+
def _digest(value: Any, name: str) -> str:
|
|
2035
|
+
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
|
2036
|
+
raise OfflineRunError("DIGEST_INVALID", f"{name} must be a lowercase SHA-256")
|
|
2037
|
+
return value
|
|
2038
|
+
|
|
2039
|
+
|
|
2040
|
+
def _stat_signature(info: os.stat_result) -> tuple[int, ...]:
|
|
2041
|
+
return (
|
|
2042
|
+
info.st_dev,
|
|
2043
|
+
info.st_ino,
|
|
2044
|
+
info.st_mode,
|
|
2045
|
+
info.st_nlink,
|
|
2046
|
+
info.st_size,
|
|
2047
|
+
info.st_mtime_ns,
|
|
2048
|
+
info.st_ctime_ns,
|
|
2049
|
+
)
|