mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,3713 @@
|
|
|
1
|
+
"""Local, streaming notebook workbench for ``mr-data view``.
|
|
2
|
+
|
|
3
|
+
The viewer is deliberately a spectator. It reads ``research.ipynb`` and ``table.ipynb`` from a
|
|
4
|
+
local run directory, renders them through the stdlib-only ``nbrender``, and streams file changes to
|
|
5
|
+
the browser over SSE. It never starts a kernel and never executes notebook code.
|
|
6
|
+
|
|
7
|
+
Each valid on-disk notebook state is retained in a bounded in-memory journal. The authoritative
|
|
8
|
+
files remain on disk beside the candidate.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
# Embedded browser assets are kept as readable source. Their physical line lengths are unrelated
|
|
12
|
+
# to Python layout and are intentionally exempt from the Python line-width rule.
|
|
13
|
+
# ruff: noqa: E501
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import errno
|
|
18
|
+
import fcntl
|
|
19
|
+
import hashlib
|
|
20
|
+
import html
|
|
21
|
+
import ipaddress
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import secrets
|
|
26
|
+
import shlex
|
|
27
|
+
import socket
|
|
28
|
+
import stat
|
|
29
|
+
import subprocess
|
|
30
|
+
import sys
|
|
31
|
+
import tempfile
|
|
32
|
+
import threading
|
|
33
|
+
import time
|
|
34
|
+
import webbrowser
|
|
35
|
+
from collections.abc import Callable, Mapping
|
|
36
|
+
from dataclasses import dataclass, replace
|
|
37
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
38
|
+
from importlib.resources import files
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from urllib.parse import parse_qs, urlsplit
|
|
41
|
+
|
|
42
|
+
from mostlyright.data_harness import events
|
|
43
|
+
from mostlyright.data_harness.event_presentation import (
|
|
44
|
+
actual_run_started_record,
|
|
45
|
+
coherent_build_seal_candidate,
|
|
46
|
+
elapsed_seconds,
|
|
47
|
+
format_elapsed,
|
|
48
|
+
reduce_lifecycle,
|
|
49
|
+
)
|
|
50
|
+
from mostlyright.data_harness.hosted_handoff import read_handoff_receipt
|
|
51
|
+
from mostlyright.data_harness.nbrender import render_notebook
|
|
52
|
+
from mostlyright.data_harness.notebook import (
|
|
53
|
+
_build_notebook_from_snapshot,
|
|
54
|
+
render_table_notebook,
|
|
55
|
+
)
|
|
56
|
+
from mostlyright.data_harness.offline import (
|
|
57
|
+
CONTROL_PATH,
|
|
58
|
+
WORKSPACE_LOCK_NAME,
|
|
59
|
+
validate_workspace_structure,
|
|
60
|
+
)
|
|
61
|
+
from mostlyright.data_harness.pipeline import (
|
|
62
|
+
RUN_BUILD_ACTIVITY_LOCK,
|
|
63
|
+
_CandidateRunHandle,
|
|
64
|
+
_entry_identity,
|
|
65
|
+
_open_candidate_run_handle,
|
|
66
|
+
_validate_candidate_run_handle,
|
|
67
|
+
open_verified_snapshot,
|
|
68
|
+
verify_candidate,
|
|
69
|
+
)
|
|
70
|
+
from mostlyright.data_harness.ux.remediation import remediation_for
|
|
71
|
+
from mostlyright.data_harness.visual_run import (
|
|
72
|
+
ReducedRun,
|
|
73
|
+
VisualRunError,
|
|
74
|
+
VisualRunEvent,
|
|
75
|
+
VisualRunTransport,
|
|
76
|
+
apply_event,
|
|
77
|
+
discover_visual_transport,
|
|
78
|
+
materialize_notebook,
|
|
79
|
+
notebook_bytes,
|
|
80
|
+
parse_last_event_id,
|
|
81
|
+
visual_authority_lock,
|
|
82
|
+
)
|
|
83
|
+
from mostlyright.data_harness.visual_run.query import (
|
|
84
|
+
QueryError,
|
|
85
|
+
QueryEvidence,
|
|
86
|
+
query_sealed_parquet,
|
|
87
|
+
sealed_parquet_columns,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
HEALTH_SIDECAR_UNAVAILABLE = "VIEWER_DATASET_SIDECAR_UNAVAILABLE"
|
|
91
|
+
HEALTH_WATCHER_FAILED = "VIEWER_WATCHER_FAILED"
|
|
92
|
+
# The typed codes `mr-data view` reports while it keeps serving. They reach a person through the
|
|
93
|
+
# page, /health and /snapshot rather than through a raise site, so the remediation completeness
|
|
94
|
+
# gate reads them from here rather than from the walk over refusals.
|
|
95
|
+
HEALTH_CODES: tuple[str, ...] = (HEALTH_SIDECAR_UNAVAILABLE, HEALTH_WATCHER_FAILED)
|
|
96
|
+
|
|
97
|
+
_SIDECAR_NAME = "table.ipynb"
|
|
98
|
+
_DOCUMENTS = {"research": "research.ipynb", "table": _SIDECAR_NAME}
|
|
99
|
+
_INSTALL_HINT = 'install the viewer extra: pip install "mostlyright-data[viewer]"'
|
|
100
|
+
_POLL_SECONDS = 0.25
|
|
101
|
+
_WATCH_POLL_SECONDS = 0.05
|
|
102
|
+
_WATCHER_STOP_SECONDS = 2.0
|
|
103
|
+
_MAX_HISTORY = 240
|
|
104
|
+
_MAX_NOTEBOOK_BYTES = 8 * 1024 * 1024
|
|
105
|
+
_MAX_SNAPSHOT_BYTES = 12 * 1024 * 1024
|
|
106
|
+
_MAX_JOURNAL_BYTES = 32 * 1024 * 1024
|
|
107
|
+
_READ_CHUNK_BYTES = 1024 * 1024
|
|
108
|
+
_RESEARCH_ACTIVITY_SCHEMA = "mostlyright.research-cell-activity.v1"
|
|
109
|
+
_RESEARCH_CELL_ID = re.compile(r"[A-Za-z0-9_-]{1,64}")
|
|
110
|
+
|
|
111
|
+
_FileSignature = tuple[int, int, int, int, int, int, int]
|
|
112
|
+
_DirectoryIdentity = tuple[int, int, int]
|
|
113
|
+
|
|
114
|
+
_DIRECTORY_FLAGS = (
|
|
115
|
+
os.O_RDONLY
|
|
116
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
117
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
118
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class ViewerError(RuntimeError):
|
|
123
|
+
"""A typed pre-bind refusal for an unsafe or ambiguous viewer target."""
|
|
124
|
+
|
|
125
|
+
def __init__(self, code: str, detail: str) -> None:
|
|
126
|
+
self.code = code
|
|
127
|
+
super().__init__(detail)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
# Vendored from mostly-right-landing/src/assets/logo-ink.svg and its Logo.astro component.
|
|
131
|
+
# The self-contained viewer inlines the vectors and has no landing-site or font dependency.
|
|
132
|
+
_LOGO = (
|
|
133
|
+
files("mostlyright.data_harness").joinpath("assets/logo-ink.svg").read_text(encoding="utf-8")
|
|
134
|
+
)
|
|
135
|
+
_MARK_ONLY = _LOGO
|
|
136
|
+
|
|
137
|
+
_WORKBENCH_CSS = r"""
|
|
138
|
+
:root{color-scheme:light;--mr-ink:#17160f;--mr-logo-neutral:#17160f;--mr-cobalt:#2b5fe3;
|
|
139
|
+
--mr-orange-brand:#f15b22;--mr-yellow:#ffc613;--mr-body:#57534a;--mr-muted:#6e6959;
|
|
140
|
+
--mr-line:#e2ddd0;--mr-line-strong:#d8d2c4;--mr-canvas:#f5f2eb;--mr-white:#fff;
|
|
141
|
+
--mr-blue:#2b5fe3;--mr-blue-soft:#e8edfc;--mr-green:#187a5a;--mr-orange:#f15b22;
|
|
142
|
+
--mr-topbar:56px;--mr-sans:"Space Grotesk","Avenir Next",Avenir,"Helvetica Neue",sans-serif;
|
|
143
|
+
--mr-mono:"JetBrains Mono","SFMono-Regular",Consolas,"Liberation Mono",monospace}
|
|
144
|
+
*,*:before,*:after{box-sizing:border-box}
|
|
145
|
+
html{scroll-behavior:auto}body{margin:0;background:var(--mr-canvas);color:var(--mr-ink);
|
|
146
|
+
font-family:var(--mr-sans);font-size:13px;-webkit-font-smoothing:antialiased}.mr-topbar{position:fixed;z-index:50;inset:0 0 auto 0;
|
|
147
|
+
height:var(--mr-topbar);display:flex;align-items:center;gap:14px;padding:0 14px 0 16px;
|
|
148
|
+
background:var(--mr-canvas);border-bottom:1px solid var(--mr-line)}
|
|
149
|
+
[hidden]{display:none!important}
|
|
150
|
+
.mr-brand{display:flex;width:165px;min-height:32px;flex:0 0 165px;align-items:center;padding-right:16px;border-right:1px solid var(--mr-line);color:var(--mr-ink);text-decoration:none}
|
|
151
|
+
.mr-logo{display:block;width:auto;height:20px}.mr-session{display:flex;min-width:0;max-width:330px;
|
|
152
|
+
align-items:center}
|
|
153
|
+
.mr-session-copy{display:flex;min-width:0;align-items:center;gap:7px;white-space:nowrap}.mr-session-copy small{overflow:hidden;color:var(--mr-body);
|
|
154
|
+
font:500 10px/1 var(--mr-mono);text-overflow:ellipsis}.mr-session-copy strong{display:flex;align-items:center;gap:7px;overflow:hidden;color:var(--mr-muted);
|
|
155
|
+
font:500 10px/1 var(--mr-mono);text-overflow:ellipsis;white-space:nowrap}.mr-session-copy strong:before{color:#aaa294;content:"/"}
|
|
156
|
+
.mr-tabs{height:36px;display:flex;align-items:center;gap:2px;margin-left:4px;padding:2px;background:#f1ede3;border:1px solid var(--mr-line);border-radius:9px}
|
|
157
|
+
.mr-tab{display:flex;height:32px;align-items:center;gap:0;padding:0 9px;background:transparent;border:1px solid transparent;border-radius:7px;
|
|
158
|
+
color:var(--mr-muted);font:400 10.5px/1 var(--mr-mono);cursor:pointer}.mr-tab:hover{background:#e8e2d6;color:var(--mr-body)}
|
|
159
|
+
.mr-tab.is-active{background:#fff;border-color:var(--mr-line-strong);color:var(--mr-ink);font-weight:600;box-shadow:0 1px 2px rgba(31,29,23,.08)}
|
|
160
|
+
.mr-tabs.is-single{margin-left:4px;padding:0 0 0 16px;background:transparent;border:0;border-left:1px solid var(--mr-line);border-radius:0}.mr-tabs.is-single .mr-tab{padding:0 4px;background:transparent;border:0;box-shadow:none;cursor:default}
|
|
161
|
+
.mr-tabs.is-single .mr-tab:hover{background:transparent}.mr-tabs.is-single .mr-tab-icon{color:var(--mr-orange-brand)}
|
|
162
|
+
.mr-tab-icon{width:13px;height:13px;margin-right:6px;fill:none;stroke:currentColor;stroke-linecap:round;stroke-linejoin:round;stroke-width:1.25}
|
|
163
|
+
.mr-tab-name{white-space:nowrap}.mr-tab-ext{margin-left:1px;color:var(--mr-muted);font-family:var(--mr-mono);font-size:9px}.mr-actions{display:flex;
|
|
164
|
+
margin-left:auto;align-items:center;gap:6px}.mr-live{display:flex;align-items:center;gap:7px;margin-right:4px;padding-right:10px;border-right:1px solid var(--mr-line);
|
|
165
|
+
color:var(--mr-muted);font-size:10px}.mr-live i{width:6px;height:6px;background:var(--mr-green);border-radius:50%}
|
|
166
|
+
.mr-live.is-reconnecting i{background:var(--mr-orange)}.mr-live.is-unavailable i,.mr-live.is-stopped i{background:#b42318}
|
|
167
|
+
.mr-button,.mr-download{height:34px;display:inline-flex;align-items:center;justify-content:center;gap:6px;padding:0 11px;
|
|
168
|
+
background:#fff;border:1px solid var(--mr-line-strong);border-radius:7px;color:var(--mr-body);font:600 10.5px/1 var(--mr-sans);
|
|
169
|
+
text-decoration:none;cursor:pointer}.mr-button:hover,.mr-download:hover{background:#f8f9fb;border-color:#aeb5bf}
|
|
170
|
+
.mr-button--primary{background:var(--mr-ink);border-color:var(--mr-ink);color:#fff}.mr-button--primary:hover{background:#30353c}
|
|
171
|
+
.mr-download[aria-disabled="true"],.mr-button[data-explore]:disabled{display:none}.mr-button[data-files]{width:34px;padding:0}.mr-files-label{position:absolute;width:1px;height:1px;margin:-1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}.mr-run-clock{min-width:44px;color:var(--mr-muted);
|
|
172
|
+
font:600 11.5px/1 var(--mr-mono);font-variant-numeric:tabular-nums;text-align:right}.mr-overall{position:absolute;width:1px;height:1px;margin:-1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap}.mr-overall span{display:none}
|
|
173
|
+
.mr-button:disabled{opacity:.45;cursor:not-allowed}.mr-workbench{min-height:100vh;padding-top:var(--mr-topbar)}
|
|
174
|
+
.mr-explorer{width:min(920px,calc(100vw - 32px));max-height:80vh;border:1px solid var(--mr-line);border-radius:12px;padding:0}.mr-explorer::backdrop{background:#17202a66}.mr-explorer-head{display:flex;justify-content:space-between;align-items:center;padding:14px 16px;border-bottom:1px solid var(--mr-line)}.mr-explorer-body{overflow:auto;padding:12px 16px 18px}.mr-explorer table{border-collapse:collapse;width:100%;font-size:12px}.mr-explorer th,.mr-explorer td{border-bottom:1px solid var(--mr-line);padding:7px;text-align:left;white-space:nowrap}.mr-explorer td{font-family:var(--mr-mono)}
|
|
175
|
+
.mr-explorer-tools{display:flex;flex-wrap:wrap;gap:8px;align-items:end;margin:0 0 12px}.mr-explorer-tools label{display:grid;gap:4px;font-size:11px;color:var(--mr-muted)}.mr-explorer-tools select,.mr-explorer-tools input{font:12px var(--mr-sans);border:1px solid var(--mr-line);border-radius:6px;background:var(--mr-surface);color:var(--mr-ink);padding:6px}.mr-explorer-tools select[multiple]{min-width:180px;max-height:94px}.mr-explorer-status{margin-left:auto;color:var(--mr-muted);font-size:12px}
|
|
176
|
+
.mr-explorer-evidence{display:grid;gap:10px;margin:0 0 16px}.mr-explorer-evidence>h3{margin:0;font-size:14px}.mr-explorer-evidence .nb-stage-observation{max-height:420px;overflow:auto}
|
|
177
|
+
.mr-stage{max-width:940px;margin:0 auto;padding:30px 28px 55vh}.mr-notebook-root{min-height:420px}
|
|
178
|
+
.mr-build-progress{margin:0 0 18px;background:var(--mr-white);border:1px solid var(--mr-line);border-radius:7px;
|
|
179
|
+
box-shadow:0 1px 3px rgba(31,35,41,.025)}.mr-build-progress>summary{list-style:none;cursor:pointer;padding:11px 14px}
|
|
180
|
+
.mr-build-progress>summary::-webkit-details-marker{display:none}.mr-build-progress-head{display:flex;align-items:baseline;gap:12px;margin:0}
|
|
181
|
+
.mr-build-progress-head strong{font-size:12px}.mr-build-progress-state{color:var(--mr-green);font:700 8.5px/1 var(--mr-mono);
|
|
182
|
+
letter-spacing:.08em;text-transform:uppercase}.mr-build-progress.is-failed .mr-build-progress-state{color:#b42318}
|
|
183
|
+
.mr-build-progress ol{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:7px 22px;margin:0;padding:4px 14px 14px;list-style:none}
|
|
184
|
+
.mr-build-line{display:grid;grid-template-columns:18px minmax(0,1fr);column-gap:4px;align-items:baseline;color:var(--mr-body)}
|
|
185
|
+
.mr-build-mark{color:var(--mr-orange);font:700 12px/1 var(--mr-mono)}.mr-build-line.is-done .mr-build-mark{color:var(--mr-green)}
|
|
186
|
+
.mr-build-line.is-failed .mr-build-mark{color:#b42318}.mr-build-copy{display:flex;min-width:0;gap:7px;align-items:baseline}
|
|
187
|
+
.mr-build-label{font-size:12.5px;font-weight:650}.mr-build-detail{overflow:hidden;color:var(--mr-muted);font:11px/1.35 var(--mr-mono);
|
|
188
|
+
text-overflow:ellipsis;white-space:nowrap}
|
|
189
|
+
.mr-health-detail{display:grid;grid-template-columns:2px minmax(0,1fr);column-gap:14px;margin:0 0 24px;padding:3px 0;
|
|
190
|
+
background:transparent;border:0;color:var(--mr-ink);line-height:1.5;overflow-wrap:anywhere}.mr-health-detail:before{grid-row:1/4;width:2px;
|
|
191
|
+
background:#a34718;border-radius:1px;content:""}.mr-health-detail strong{grid-column:2;color:#8b3f20;font:700 9.5px/1.4 var(--mr-mono);
|
|
192
|
+
letter-spacing:.06em;text-transform:uppercase}.mr-health-detail span{grid-column:2;margin-top:5px;color:var(--mr-body);font-size:13px}
|
|
193
|
+
.mr-health-detail code{grid-column:2;display:block;margin-top:8px;overflow-wrap:anywhere;color:var(--mr-muted);font:600 11px/1.5 var(--mr-mono)}
|
|
194
|
+
.mr-research-terminal{margin:24px 0 0;padding:3px 0 3px 16px;background:transparent;border:0;border-left:2px solid #a34718;
|
|
195
|
+
border-radius:0;color:var(--mr-ink);overflow-wrap:anywhere}.mr-research-terminal strong{display:block;font-size:15px}.mr-research-terminal p{margin:7px 0 0;
|
|
196
|
+
color:var(--mr-body);font-size:14px;line-height:1.5}.mr-research-terminal small{display:block;margin-top:9px;color:var(--mr-muted);font:600 11px/1.4 var(--mr-mono)}
|
|
197
|
+
.mr-workbench .nb-shell{overflow:visible;border:0;border-radius:0;background:transparent;box-shadow:none;font-family:var(--mr-sans)}
|
|
198
|
+
.mr-workbench .nb-body{display:block}.mr-workbench .nb-document{padding:0;gap:20px}.mr-workbench .nb-cell{
|
|
199
|
+
grid-template-columns:56px minmax(0,1fr);column-gap:16px}.mr-workbench .nb-content{font-size:13px}
|
|
200
|
+
/* The output row re-enters the gutter this cell states: 72px = the 56px gutter + the 16px gap.
|
|
201
|
+
Restacking lives in @container below, not @media -- the shell is narrower than the viewport. */
|
|
202
|
+
.mr-workbench .nb-io{margin-left:-72px;grid-template-columns:56px minmax(0,1fr);column-gap:16px}
|
|
203
|
+
.mr-workbench .nb-prompt{padding-top:8px;color:var(--mr-muted);font-size:9px;text-align:right}.mr-workbench .nb-prompt--md{padding-top:4px}
|
|
204
|
+
.mr-workbench .nb-md{overflow-x:auto}.mr-workbench .nb-md p,.mr-workbench .nb-md li{font-size:12.5px;line-height:1.55}.mr-workbench .nb-md p{max-width:none}
|
|
205
|
+
.mr-workbench .nb-md code{padding:1px 4px;font-size:10.5px;line-height:1.35}
|
|
206
|
+
.mr-workbench .nb-md h1{max-width:32ch;font-size:24px;line-height:1.12;letter-spacing:-.025em}
|
|
207
|
+
.mr-workbench .nb-md h2{font-size:17px;line-height:1.2;letter-spacing:-.015em}.mr-workbench .nb-md h3{font-size:13px;letter-spacing:0}
|
|
208
|
+
.mr-workbench .nb-cell:has(.nb-md h2):not(:has(.nb-md h1)){margin-top:18px;padding-top:22px;border-top:1px solid var(--mr-line)}
|
|
209
|
+
.mr-workbench .nb-document>.nb-cell:first-child{margin-top:0;padding-top:0;border-top:0}
|
|
210
|
+
.mr-workbench .nb-cell:has(.nb-md h1)+.nb-cell:has(.nb-md h2){margin-top:8px;padding-top:0;border-top:0}
|
|
211
|
+
.mr-workbench .nb-md h2+*{margin-top:4px}.mr-workbench .nb-md h3{margin-top:18px}.mr-workbench .nb-md-table{min-width:640px;background:var(--mr-white)}
|
|
212
|
+
.mr-workbench .nb-md-table thead th{padding:8px 12px;font-size:9px}.mr-workbench .nb-md-table tbody td{padding:8px 12px;background:var(--mr-white);font-size:12px}
|
|
213
|
+
.mr-workbench .nb-df{background:var(--mr-white);border-radius:6px}.mr-workbench .nb-df table{font-size:11.5px}.mr-workbench .nb-df th{padding:8px 12px;font-size:9px}.mr-workbench .nb-df td{padding:8px 12px;background:var(--mr-white)}
|
|
214
|
+
.mr-workbench .nb-result-plain{padding:10px 12px;background:var(--mr-white);border:1px solid var(--mr-line);border-radius:6px;color:var(--mr-ink);font-size:11.5px;line-height:1.55}
|
|
215
|
+
.mr-workbench .nb-rich{background:var(--mr-white);border-radius:6px}.mr-workbench .nb-rich-head{display:none}.mr-workbench .nb-rich-body{padding:0}
|
|
216
|
+
.mr-workbench .nb-rich-body table{width:100%;border-collapse:collapse;background:var(--mr-white);font-size:11.5px}.mr-workbench .nb-md-table thead th,.mr-workbench .nb-df thead th,.mr-workbench .nb-rich-body thead th{padding:8px 12px;background:var(--nb-sub-paper);border-bottom:1px solid var(--nb-border);font-family:var(--fm);font-size:9px;font-weight:600;letter-spacing:.1em;text-transform:uppercase;color:var(--nb-dim);text-align:left}.mr-workbench .nb-md-table tbody td,.mr-workbench .nb-df tbody td,.mr-workbench .nb-rich-body tbody td{background:var(--mr-white);border-bottom:1px solid var(--nb-row-rule)}.mr-workbench .nb-md-table tbody td:first-child,.mr-workbench .nb-df tbody td:first-child,.mr-workbench .nb-rich-body tbody td:first-child{background:var(--mr-white);border-left:0 solid var(--nb-row-rule);border-right:0 solid var(--nb-row-rule)}.mr-workbench .nb-md-table tbody tr:last-child td,.mr-workbench .nb-df tbody tr:last-child td,.mr-workbench .nb-rich-body tbody tr:last-child td{border-bottom:0 solid var(--nb-row-rule)}.mr-workbench .nb-rich-body td{padding:8px 12px}
|
|
217
|
+
.mr-workbench .nb-panel.is-selected,.mr-workbench .nb-cell.is-editing .nb-gutter{box-shadow:none}
|
|
218
|
+
.mr-workbench .nb-prov{display:none}
|
|
219
|
+
.mr-workbench button,.mr-workbench a{min-height:32px}.mr-workbench a{display:inline-flex;align-items:center}
|
|
220
|
+
.mr-workbench .nb-code{border-radius:7px}.mr-workbench .nb-code .nb-lineno{display:none}
|
|
221
|
+
.mr-workbench .nb-code pre[tabindex]{overflow-x:hidden;
|
|
222
|
+
overflow-wrap:anywhere;white-space:pre-wrap;font-size:11.5px;line-height:1.62}
|
|
223
|
+
.mr-workbench [data-research-activity]{position:relative}
|
|
224
|
+
.mr-workbench [data-research-activity]:before{position:absolute;top:0;bottom:0;left:-16px;width:2px;background:var(--mr-blue);content:""}
|
|
225
|
+
.mr-workbench [data-research-activity="executing"]:before{background:var(--mr-orange)}
|
|
226
|
+
.mr-workbench .nb-activity{gap:5px;margin:0 0 10px;padding:0;background:transparent;border-radius:0;color:#8cadff;
|
|
227
|
+
font:500 9px/1 var(--mr-mono);text-transform:none;letter-spacing:0}.mr-workbench .nb-activity--executing{background:transparent;color:#f08a61}
|
|
228
|
+
.mr-workbench .nb-panel-head>.nb-activity{margin:0 7px}
|
|
229
|
+
.mr-workbench .nb-activity:before{width:5px;height:5px;background:currentColor;border-radius:50%;content:""}
|
|
230
|
+
.mr-workbench .nb-activity-caret{width:1px;height:10px}
|
|
231
|
+
.mr-waiting{min-height:460px;display:flex;align-items:center;justify-content:center;flex-direction:column;gap:12px;
|
|
232
|
+
color:var(--mr-muted);text-align:center}.mr-waiting .mr-logo{width:auto;height:26px}
|
|
233
|
+
.mr-waiting strong{color:var(--mr-ink);font-size:13px}.mr-waiting span{max-width:390px;font-size:11px;line-height:1.5}
|
|
234
|
+
.mr-stream-enter{animation:mr-enter .28s ease-out both}.mr-stream-update{animation:mr-update .26s ease-out both}
|
|
235
|
+
.mr-sr-status{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}
|
|
236
|
+
.mr-toast{position:fixed;z-index:60;right:20px;bottom:20px;padding:11px 14px;background:var(--mr-ink);border-radius:8px;
|
|
237
|
+
color:#fff;font-size:12.5px;box-shadow:0 8px 30px rgba(31,35,41,.2)}
|
|
238
|
+
@keyframes mr-enter{from{opacity:0;transform:translateY(7px)}to{opacity:1;transform:none}}
|
|
239
|
+
@keyframes mr-update{from{opacity:.72}to{opacity:1}}
|
|
240
|
+
@keyframes mr-logo-f1{0%,100%{fill:var(--mr-logo-neutral)}35%{fill:var(--mr-cobalt)}50%{fill:var(--mr-orange-brand)}85%{fill:var(--mr-yellow)}}
|
|
241
|
+
@keyframes mr-logo-f2{0%,100%{fill:var(--mr-orange-brand)}35%{fill:var(--mr-yellow)}50%{fill:var(--mr-logo-neutral)}85%{fill:var(--mr-cobalt)}}
|
|
242
|
+
@keyframes mr-logo-f3{0%,100%{fill:var(--mr-yellow)}35%{fill:var(--mr-logo-neutral)}50%{fill:var(--mr-cobalt)}85%{fill:var(--mr-orange-brand)}}
|
|
243
|
+
@keyframes mr-logo-f4{0%,100%{fill:var(--mr-cobalt)}35%{fill:var(--mr-orange-brand)}50%{fill:var(--mr-yellow)}85%{fill:var(--mr-logo-neutral)}}
|
|
244
|
+
@keyframes mr-logo-f5{0%,100%{fill:var(--mr-orange-brand)}35%{fill:var(--mr-yellow)}50%{fill:var(--mr-cobalt)}85%{fill:var(--mr-logo-neutral)}}
|
|
245
|
+
@keyframes mr-logo-f6{0%,100%{fill:var(--mr-logo-neutral)}35%{fill:var(--mr-cobalt)}50%{fill:var(--mr-orange-brand)}85%{fill:var(--mr-yellow)}}
|
|
246
|
+
@keyframes mr-logo-f7{0%,100%{fill:var(--mr-cobalt)}35%{fill:var(--mr-orange-brand)}50%{fill:var(--mr-yellow)}85%{fill:var(--mr-cobalt)}}
|
|
247
|
+
@keyframes mr-logo-f8{0%,100%{fill:var(--mr-yellow)}35%{fill:var(--mr-logo-neutral)}50%{fill:var(--mr-cobalt)}85%{fill:var(--mr-orange-brand)}}
|
|
248
|
+
@keyframes mr-logo-f9{0%,100%{fill:var(--mr-yellow)}35%{fill:var(--mr-cobalt)}50%{fill:var(--mr-logo-neutral)}85%{fill:var(--mr-yellow)}}
|
|
249
|
+
.mr-logo-facet--1{--mr-logo-animation:mr-logo-f1}.mr-logo-facet--2{--mr-logo-animation:mr-logo-f2}
|
|
250
|
+
.mr-logo-facet--3{--mr-logo-animation:mr-logo-f3}.mr-logo-facet--4{--mr-logo-animation:mr-logo-f4}
|
|
251
|
+
.mr-logo-facet--5{--mr-logo-animation:mr-logo-f5}.mr-logo-facet--6{--mr-logo-animation:mr-logo-f6}
|
|
252
|
+
.mr-logo-facet--7{--mr-logo-animation:mr-logo-f7}.mr-logo-facet--8{--mr-logo-animation:mr-logo-f8}
|
|
253
|
+
.mr-logo-facet--9{--mr-logo-animation:mr-logo-f9}
|
|
254
|
+
@media(hover:hover){.mr-logo:hover .mr-logo-facet{animation-name:var(--mr-logo-animation);animation-duration:680ms;
|
|
255
|
+
animation-timing-function:steps(1,end);animation-iteration-count:3;animation-fill-mode:both}}
|
|
256
|
+
.mr-waiting .mr-logo-facet{animation-name:var(--mr-logo-animation);animation-duration:680ms;
|
|
257
|
+
animation-timing-function:steps(1,end);animation-iteration-count:infinite;animation-fill-mode:both}
|
|
258
|
+
@media(prefers-reduced-motion:reduce){.mr-stream-enter,.mr-stream-update,.mr-logo:hover .mr-logo-facet,
|
|
259
|
+
.mr-waiting .mr-logo-facet{animation:none}}
|
|
260
|
+
@media(max-width:1100px){.mr-session{max-width:230px}.mr-live{display:none}.mr-stage{padding-right:24px;padding-left:24px}}
|
|
261
|
+
@media(max-width:760px){.mr-topbar{gap:6px;padding:0 9px;overflow-x:auto}.mr-brand{display:none}.mr-live{display:none}
|
|
262
|
+
.mr-session{display:flex;max-width:190px}.mr-tabs{min-width:0}.mr-tab{padding:0 7px}.mr-tab-ext{display:none}.mr-actions{flex-shrink:0}
|
|
263
|
+
.mr-button,.mr-download{padding:0 10px}.mr-stage{padding:28px 12px 45vh}}
|
|
264
|
+
@media(max-width:559px){:root{--mr-topbar:56px}.mr-topbar{height:var(--mr-topbar);display:flex;overflow:visible}.mr-session{display:none}.mr-tabs{height:36px;margin-left:0}.mr-actions{min-width:0;margin-left:auto;gap:5px}.mr-actions [data-explore],.mr-actions .mr-download{display:none}.mr-button{height:34px;padding:0 8px;font-size:11.5px}}
|
|
265
|
+
@media(max-width:620px){.mr-build-progress ol{grid-template-columns:minmax(0,1fr)}.mr-build-progress-head{align-items:flex-start;flex-direction:column;gap:5px}}
|
|
266
|
+
@container (max-width:559px){.mr-workbench .nb-cell{grid-template-columns:minmax(0,1fr);column-gap:0}
|
|
267
|
+
.mr-workbench .nb-gutter{grid-column:1;display:flex;gap:10px;padding:0 2px 6px;text-align:left}
|
|
268
|
+
.mr-workbench .nb-content{grid-column:1;min-width:0}.mr-workbench .nb-io{margin-left:0;grid-template-columns:minmax(0,1fr);column-gap:0}.mr-workbench .nb-io-gutter,.mr-workbench .nb-io-body{grid-column:1}}
|
|
269
|
+
""".strip()
|
|
270
|
+
|
|
271
|
+
_RELOAD_SCRIPT = r"""
|
|
272
|
+
<script>
|
|
273
|
+
(() => {
|
|
274
|
+
const state = {doc: document.body.dataset.doc || '', version: Number(document.body.dataset.version || 0),
|
|
275
|
+
cursor: Number(document.body.dataset.cursor || 0),
|
|
276
|
+
epoch: document.body.dataset.epoch || '', health: document.body.dataset.health || 'running',
|
|
277
|
+
handoff: document.body.dataset.handoff === 'true', lifecycle: null,
|
|
278
|
+
requestSerial: 0, navigationSerial: 0, applyQueue: Promise.resolve(), pendingSnapshots: [], streamWorker: false,
|
|
279
|
+
followedPhases: new Set(), followHeld: false, incoming: null};
|
|
280
|
+
const root = document.querySelector('[data-notebook-root]');
|
|
281
|
+
const toast = message => { const node=document.createElement('div'); node.className='mr-toast';
|
|
282
|
+
node.setAttribute('role','status'); node.textContent=message; document.body.append(node); setTimeout(()=>node.remove(),2400); };
|
|
283
|
+
function activateInspector(select) { const inspector=select.closest('[data-column-inspector]'); if(!inspector)return;
|
|
284
|
+
const name=select.value; inspector.querySelectorAll('[data-column-profile]').forEach(panel => {
|
|
285
|
+
const active=panel.dataset.columnProfile===name; panel.hidden=!active; panel.classList.toggle('is-active',active);
|
|
286
|
+
}); const title=inspector.querySelector('[data-column-title]'); if(title) title.textContent=name;
|
|
287
|
+
}
|
|
288
|
+
function setStatus(label) { const badge=document.querySelector('[data-live-status]'); if(!badge)return;
|
|
289
|
+
badge.classList.remove('is-reconnecting','is-unavailable','is-stopped');
|
|
290
|
+
badge.classList.add('is-'+label.toLowerCase()); badge.querySelector('span').textContent=label; }
|
|
291
|
+
function healthLabel(status) { return status==='stopped'?'Stopped':status==='unavailable'?'Unavailable':'Live'; }
|
|
292
|
+
function applyHealth(health) { state.health=health.status; document.body.dataset.health=health.status;
|
|
293
|
+
setStatus(healthLabel(health.status)); const detail=document.querySelector('[data-health-detail]');if(!detail)return;
|
|
294
|
+
const failed=(health.status==='unavailable'||health.status==='stopped')&&health.code;if(!failed){detail.hidden=true;return;}
|
|
295
|
+
const code=String(health.code),message=health.detail||'',command=health.remediation||'',strong=detail.querySelector('strong'),span=detail.querySelector('span'),remediation=detail.querySelector('code');
|
|
296
|
+
const changed=strong.textContent!==code||span.textContent!==message||remediation.textContent!==command;if(!changed){detail.hidden=false;return;}detail.hidden=true;
|
|
297
|
+
strong.textContent=code;span.textContent=message;remediation.textContent=command;detail.hidden=false; }
|
|
298
|
+
function updateTabs(docs, active=state.doc) { document.querySelectorAll('.mr-tab[data-doc]').forEach(tab => { tab.hidden=!docs.includes(tab.dataset.doc);
|
|
299
|
+
tab.classList.toggle('is-active',tab.dataset.doc===active); tab.setAttribute('aria-selected',String(tab.dataset.doc===active)); }); }
|
|
300
|
+
function clockText(seconds) { seconds=Math.max(0,Math.floor(seconds));const hours=Math.floor(seconds/3600), minutes=Math.floor(seconds%3600/60), secs=seconds%60;
|
|
301
|
+
return hours?hours+':'+String(minutes).padStart(2,'0')+':'+String(secs).padStart(2,'0'):String(minutes).padStart(2,'0')+':'+String(secs).padStart(2,'0'); }
|
|
302
|
+
function tickClock() { const node=document.querySelector('[data-run-clock]'), lifecycle=state.lifecycle;if(!node||!lifecycle||lifecycle.started_at==null){if(node)node.hidden=true;return;}
|
|
303
|
+
const stop=lifecycle.ended_at==null?Date.now()/1000:Number(lifecycle.ended_at);node.textContent=clockText(stop-Number(lifecycle.started_at));node.hidden=false; }
|
|
304
|
+
function applyLifecycle(lifecycle) { if(!lifecycle)return;state.lifecycle=lifecycle;state.handoff=Boolean(lifecycle.handoff);document.body.dataset.handoff=String(state.handoff);
|
|
305
|
+
const phase=document.querySelector('[data-lifecycle-phase]');if(phase){const label=lifecycle.phase||'Research notebook';phase.textContent=label==='Research notebook'?'Research':label;}
|
|
306
|
+
const overall=document.querySelector('[data-overall-progress]'), progress=Math.max(0,Math.min(100,Number(lifecycle.progress)||0));
|
|
307
|
+
if(overall){overall.setAttribute('aria-valuenow',String(progress));const fill=overall.querySelector('span');if(fill)fill.style.width=progress+'%';}
|
|
308
|
+
const block=document.querySelector('[data-build-progress]');if(block){block.hidden=lifecycle.status==='idle';block.open=lifecycle.status!=='completed'&&lifecycle.status!=='idle';block.classList.toggle('is-failed',lifecycle.status==='failed');
|
|
309
|
+
const status=block.querySelector('[data-build-state]');if(status)status.textContent=lifecycle.status==='failed'?'Stopped':lifecycle.status==='completed'?'Complete':'Running';
|
|
310
|
+
const heading=block.querySelector('[data-build-phase]');if(heading)heading.textContent=lifecycle.phase||'';const list=block.querySelector('[data-build-lines]');
|
|
311
|
+
if(list){const nodes=(lifecycle.lines||[]).map(line=>{const item=document.createElement('li');item.className='mr-build-line is-'+line.status;item.dataset.buildSpan=line.key;
|
|
312
|
+
const mark=document.createElement('span');mark.className='mr-build-mark';mark.setAttribute('aria-hidden','true');mark.textContent=line.status==='failed'?'!':line.status==='done'?'ok':'.';
|
|
313
|
+
const copy=document.createElement('span');copy.className='mr-build-copy';const label=document.createElement('span');label.className='mr-build-label';label.textContent=line.label;
|
|
314
|
+
const detail=document.createElement('span');detail.className='mr-build-detail';detail.textContent=line.detail;copy.append(label,detail);item.append(mark,copy);return item;});list.replaceChildren(...nodes);}}
|
|
315
|
+
const files=document.querySelector('[data-files]'), filesLabel=state.handoff?'Show table in Finder':'Show run in Finder';if(files){files.setAttribute('aria-label',filesLabel);const label=files.querySelector('.mr-files-label');if(label)label.textContent=filesLabel;}
|
|
316
|
+
const download=document.querySelector('[data-download]');if(download){download.setAttribute('aria-disabled',String(!state.handoff));if(state.handoff){download.href='/download/table.parquet';download.removeAttribute('tabindex');}else{download.removeAttribute('href');download.setAttribute('tabindex','-1');}}
|
|
317
|
+
const explore=document.querySelector('[data-explore]');if(explore)explore.disabled=!state.handoff;
|
|
318
|
+
applyResearchTerminal(lifecycle);tickClock(); }
|
|
319
|
+
function applyResearchTerminal(lifecycle) { const node=document.querySelector('[data-research-terminal]'), terminal=lifecycle&&lifecycle.terminal;if(!node)return;
|
|
320
|
+
const visible=Boolean(terminal)&&state.doc!=='table';if(!visible){node.hidden=true;return;}const event=terminal.event||'',title=terminal.title||'Build stopped',message=terminal.detail||'';
|
|
321
|
+
const meta=node.querySelector('small'),parts=[terminal.severity,terminal.finding_id].filter(Boolean),metadata=parts.join(' · '),strong=node.querySelector('strong'),paragraph=node.querySelector('p');
|
|
322
|
+
const changed=node.dataset.terminalEvent!==event||strong.textContent!==title||paragraph.textContent!==message||meta.textContent!==metadata;if(!changed){node.hidden=false;return;}node.hidden=true;
|
|
323
|
+
node.dataset.terminalEvent=event;strong.textContent=title;paragraph.textContent=message;meta.textContent=metadata;meta.hidden=parts.length===0;node.hidden=false; }
|
|
324
|
+
function anchor() { const cells=[...root.querySelectorAll('[data-cell-id]')]; const item=cells.find(node=>node.getBoundingClientRect().bottom>70);
|
|
325
|
+
return item?{id:item.dataset.cellId,top:item.getBoundingClientRect().top}:null; }
|
|
326
|
+
function parsedNotebook(markup) { const template=document.createElement('template'); template.innerHTML=markup.trim();
|
|
327
|
+
const next=template.content.firstElementChild; if(!next)throw new Error('snapshot contained no notebook'); return next; }
|
|
328
|
+
function inspectorChoices(scope) { return [...scope.querySelectorAll('[data-column-select]')].map(select=>select.value); }
|
|
329
|
+
function restoreInspectorChoices(scope, choices) { scope.querySelectorAll('[data-column-select]').forEach((select,index)=>{
|
|
330
|
+
if(choices[index]&&[...select.options].some(option=>option.value===choices[index]))select.value=choices[index]; activateInspector(select); }); }
|
|
331
|
+
function ownedNodes(cell, selector) { return [...cell.querySelectorAll(selector)].filter(node=>node.closest('[data-cell-id]')===cell); }
|
|
332
|
+
function sourceNode(cell) { return cell.querySelector('.nb-code > .nb-code-lines, .nb-code > pre[tabindex], .nb-md, .nb-raw'); }
|
|
333
|
+
function outputNodes(cell) { const content=cell.querySelector(':scope > .nb-content');return content?[...content.children].filter(node=>node.matches('.nb-io,.nb-well')):[]; }
|
|
334
|
+
function syncOptional(oldCell,newCell,selector) { const oldNode=ownedNodes(oldCell,selector)[0],newNode=ownedNodes(newCell,selector)[0];
|
|
335
|
+
if(oldNode&&newNode){if(oldNode.outerHTML!==newNode.outerHTML)oldNode.parentNode.replaceChild(newNode,oldNode);return;}if(oldNode)oldNode.remove();else if(newNode){const content=oldCell.querySelector('.nb-content');if(selector==='[data-cell-exec-meta]')content.append(newNode);else content.prepend(newNode);} }
|
|
336
|
+
function syncElement(oldNode,newNode){[...oldNode.attributes].forEach(attribute=>{if(!newNode.hasAttribute(attribute.name))oldNode.removeAttribute(attribute.name);});[...newNode.attributes].forEach(attribute=>oldNode.setAttribute(attribute.name,attribute.value));oldNode.replaceChildren(...[...newNode.childNodes]);}
|
|
337
|
+
function syncOutputs(oldCell,newCell) { const oldNodes=outputNodes(oldCell),newNodes=outputNodes(newCell);
|
|
338
|
+
oldNodes.slice(newNodes.length).forEach(node=>node.remove());const content=oldCell.querySelector(':scope > .nb-content');
|
|
339
|
+
newNodes.forEach((node,index)=>{const old=oldNodes[index];if(old){if(old.outerHTML!==node.outerHTML){if(old.tagName===node.tagName)syncElement(old,node);else old.parentNode.replaceChild(node,old);}return;}
|
|
340
|
+
content.append(node);}); }
|
|
341
|
+
function syncCellState(oldCell,newCell) { oldCell.className=newCell.className;oldCell.setAttribute('aria-label',newCell.getAttribute('aria-label')||'');
|
|
342
|
+
if(newCell.hasAttribute('data-research-activity'))oldCell.setAttribute('data-research-activity',newCell.getAttribute('data-research-activity'));else oldCell.removeAttribute('data-research-activity');
|
|
343
|
+
const oldGutter=oldCell.querySelector(':scope > [data-cell-gutter]'),newGutter=newCell.querySelector(':scope > [data-cell-gutter]');
|
|
344
|
+
if(oldGutter&&newGutter)oldGutter.replaceChildren(...[...newGutter.childNodes]);
|
|
345
|
+
const oldPanel=oldCell.querySelector('.nb-panel.nb-code'),newPanel=newCell.querySelector('.nb-panel.nb-code');
|
|
346
|
+
if(oldPanel&&newPanel){oldPanel.className=newPanel.className;const oldHead=oldPanel.querySelector(':scope > .nb-panel-head'),newHead=newPanel.querySelector(':scope > .nb-panel-head');if(oldHead&&newHead)oldHead.replaceChildren(...[...newHead.childNodes]);}
|
|
347
|
+
else syncOptional(oldCell,newCell,'[data-cell-activity]');syncOptional(oldCell,newCell,'[data-cell-exec-meta]'); }
|
|
348
|
+
function exactMutationPatch(next,doc,mutation,version,epoch,cursor) { if(!mutation||doc!=='research'||root.dataset.doc!==doc||epoch!==state.epoch||
|
|
349
|
+
(mutation.source==='visual-run'&&Number(mutation.from_cursor)!==state.cursor)||
|
|
350
|
+
(mutation.source!=='visual-run'&&Number(mutation.from_version)!==state.version)||
|
|
351
|
+
Number(mutation.version)!==Number(version)||mutation.doc!==doc||
|
|
352
|
+
(mutation.source==='visual-run'&&Number(mutation.cursor)!==Number(cursor)))return null;
|
|
353
|
+
const oldDoc=root.querySelector('.nb-document'),newDoc=next.querySelector('.nb-document');if(!oldDoc||!newDoc)return null;
|
|
354
|
+
const oldCells=[...oldDoc.querySelectorAll(':scope > [data-cell-id]')],newCells=[...newDoc.querySelectorAll(':scope > [data-cell-id]')];
|
|
355
|
+
const cellId=String(mutation.cell_id||''),oldCell=oldCells.find(node=>node.dataset.cellId===cellId),newCell=newCells.find(node=>node.dataset.cellId===cellId);
|
|
356
|
+
if(!newCell)return null;const idsBefore=oldCells.map(node=>node.dataset.cellId),idsAfter=newCells.map(node=>node.dataset.cellId),newIndex=idsAfter.indexOf(cellId);
|
|
357
|
+
const insertion=!oldCell&&newCells.length===oldCells.length+1&&idsAfter.filter(id=>id!==cellId).every((id,index)=>id===idsBefore[index]);
|
|
358
|
+
const sameOrder=idsBefore.every((id,index)=>id===idsAfter[index]),move=mutation.kind==='move'&&oldCell&&newCells.length===oldCells.length&&idsBefore.every(id=>idsAfter.includes(id));
|
|
359
|
+
const update=oldCell&&newCells.length===oldCells.length&&(sameOrder||move);if(!insertion&&!update)return null;
|
|
360
|
+
if(insertion){newCell.classList.add('mr-stream-enter');const anchor=oldCells[newIndex]||null;if(anchor)anchor.before(newCell);else oldDoc.append(newCell);return newCell;}
|
|
361
|
+
if(mutation.kind==='move'){const anchorId=String(mutation.after_cell_id||''),anchor=oldCells.find(node=>node.dataset.cellId===anchorId);if(!anchor)return null;anchor.after(oldCell);syncCellState(oldCell,newCell);oldCell.classList.add('mr-stream-update');return oldCell;}
|
|
362
|
+
const oldSource=sourceNode(oldCell),newSource=sourceNode(newCell);if(!oldSource||!newSource)return null;
|
|
363
|
+
const kind=mutation.kind;if(kind==='source'){if(outputNodes(oldCell).map(node=>node.outerHTML).join('')!==outputNodes(newCell).map(node=>node.outerHTML).join(''))return null;oldSource.parentNode.replaceChild(newSource,oldSource);}
|
|
364
|
+
else if(kind==='revision'){oldSource.parentNode.replaceChild(newSource,oldSource);syncOutputs(oldCell,newCell);}
|
|
365
|
+
else if(kind==='output'||kind==='state'){if(oldSource.textContent!==newSource.textContent)return null;if(newSource.hasAttribute('aria-label'))oldSource.setAttribute('aria-label',newSource.getAttribute('aria-label')||'');oldSource.querySelectorAll('[aria-label]').forEach((node,index)=>{const fresh=newSource.querySelectorAll('[aria-label]')[index];if(fresh)node.setAttribute('aria-label',fresh.getAttribute('aria-label')||'');});syncOutputs(oldCell,newCell);}
|
|
366
|
+
else return null;syncCellState(oldCell,newCell);oldCell.classList.add('mr-stream-update');return oldCell; }
|
|
367
|
+
function followPhase(cell,mutation) { const key=mutation.cell_id+':'+mutation.phase;if(state.followedPhases.has(key))return;state.followedPhases.add(key);
|
|
368
|
+
const announcement=document.querySelector('[data-stream-announcement]');if(announcement)announcement.textContent=mutation.announcement||('Research cell '+mutation.phase);
|
|
369
|
+
const follow=document.querySelector('[data-follow-agent]'),active=cell.matches('[data-research-activity]');if(state.followHeld&&active){if(follow)follow.hidden=false;return;}if(follow)follow.hidden=true;
|
|
370
|
+
const reduced=matchMedia('(prefers-reduced-motion: reduce)').matches,delta=cell.getBoundingClientRect().top-innerHeight*.32;
|
|
371
|
+
scrollBy({top:delta,left:0,behavior:reduced?'auto':'smooth'}); }
|
|
372
|
+
function patchNotebook(markup, doc) { const incoming=state.incoming||{},mutation=incoming.mutation||null,version=incoming.version??state.version,epoch=incoming.epoch||state.epoch,cursor=incoming.cursor??state.cursor;
|
|
373
|
+
const next=parsedNotebook(markup); const held=anchor(); const choices=inspectorChoices(root);
|
|
374
|
+
const streamed=exactMutationPatch(next,doc,mutation,version,epoch,cursor);if(streamed){const heldNext=held?[...root.querySelectorAll('[data-cell-id]')].find(node=>node.dataset.cellId===held.id):null;
|
|
375
|
+
if(held&&heldNext)scrollBy(0,heldNext.getBoundingClientRect().top-held.top);followPhase(streamed,mutation);return;}
|
|
376
|
+
if(root.dataset.doc===doc){const oldDoc=root.querySelector('.nb-document'), newDoc=next.querySelector('.nb-document');
|
|
377
|
+
if(oldDoc&&newDoc){restoreInspectorChoices(newDoc,choices);
|
|
378
|
+
const oldCells=new Map([...oldDoc.querySelectorAll(':scope > [data-cell-id]')].map(node=>[node.dataset.cellId,node]));const reused=new Set();
|
|
379
|
+
const placements=[...newDoc.querySelectorAll(':scope > [data-cell-id]')].map(node=>{const old=oldCells.get(node.dataset.cellId);
|
|
380
|
+
if(!old||reused.has(old)){node.classList.add('mr-stream-enter');return node;}const prior=old.cloneNode(true);prior.classList.remove('mr-stream-enter','mr-stream-update');
|
|
381
|
+
if(prior.outerHTML!==node.outerHTML){node.classList.add('mr-stream-update');return node;}reused.add(old);return old;});
|
|
382
|
+
const heldNext=held?placements.find(node=>node.dataset.cellId===held.id):null;
|
|
383
|
+
oldDoc.replaceChildren(...placements); root.dataset.doc=doc;
|
|
384
|
+
if(held&&heldNext)scrollBy(0,heldNext.getBoundingClientRect().top-held.top);const active=mutation&&placements.find(node=>node.dataset.cellId===String(mutation.cell_id||''));if(active)followPhase(active,mutation);return;}}
|
|
385
|
+
restoreInspectorChoices(next,choices); const heldNext=held?[...next.querySelectorAll('[data-cell-id]')].find(node=>node.dataset.cellId===held.id):null;
|
|
386
|
+
root.replaceChildren(next); root.dataset.doc=doc;
|
|
387
|
+
if(held&&heldNext)scrollBy(0,heldNext.getBoundingClientRect().top-held.top);
|
|
388
|
+
}
|
|
389
|
+
async function getSnapshot(doc=state.doc, version='', cursor='') { const query=new URLSearchParams({doc}); if(version!=='')query.set('version',String(version));if(cursor!=='')query.set('cursor',String(cursor));
|
|
390
|
+
const response=await fetch('/snapshot?'+query); if(!response.ok){const error=new Error('snapshot unavailable');error.status=response.status;throw error;} return response.json(); }
|
|
391
|
+
function requestContext(doc=state.doc) { return {doc,serial:++state.requestSerial,navigation:state.navigationSerial}; }
|
|
392
|
+
function requestIsCurrent(context) { return context.navigation===state.navigationSerial&&(context.serial==null||context.serial===state.requestSerial); }
|
|
393
|
+
function commitSnapshot(data, doc) { updateTabs(data.docs,doc); state.doc=doc; state.epoch=data.epoch; state.version=data.version;
|
|
394
|
+
state.cursor=Number(data.cursor||0);
|
|
395
|
+
document.body.dataset.doc=doc; document.body.dataset.epoch=data.epoch;
|
|
396
|
+
document.body.dataset.version=String(data.version);document.body.dataset.cursor=String(state.cursor);applyHealth(data.health); applyLifecycle(data.lifecycle); }
|
|
397
|
+
async function applySnapshot(data) { if(data.epoch===state.epoch&&data.version<state.version)return;const doc=data.doc||'';
|
|
398
|
+
if(data.epoch!==state.epoch)state.followedPhases.clear();state.incoming={mutation:data.mutation,version:data.version,epoch:data.epoch,cursor:data.cursor};patchNotebook(data.html,doc);state.incoming=null;commitSnapshot(data,doc); }
|
|
399
|
+
async function recoverSnapshot(error, context) { if(!requestIsCurrent(context))return;
|
|
400
|
+
if(error.status===410&&context.cursor!==''){try{const latest=await getSnapshot(context.doc);if(requestIsCurrent(context)){await applySnapshot(latest);return;}}catch(_latestError){}}
|
|
401
|
+
console.error('Mostly Right snapshot update failed',error);setStatus('Unavailable'); }
|
|
402
|
+
function queueSnapshot(doc=state.doc,version='',cursor='') { const numeric=version===''?'':Number(version),eventCursor=cursor===''?'':Number(cursor);
|
|
403
|
+
if(state.pendingSnapshots.some(item=>item.doc===doc&&item.version===numeric&&item.cursor===eventCursor))return state.applyQueue;
|
|
404
|
+
state.pendingSnapshots.push({doc,version:numeric,cursor:eventCursor,navigation:state.navigationSerial});if(state.streamWorker)return state.applyQueue;
|
|
405
|
+
state.streamWorker=true;state.applyQueue=state.applyQueue.then(async()=>{while(state.pendingSnapshots.length){const context=state.pendingSnapshots.shift();
|
|
406
|
+
if(!requestIsCurrent(context))continue;try{const data=await getSnapshot(context.doc,context.version,context.cursor);if(!requestIsCurrent(context))continue;await applySnapshot(data);}
|
|
407
|
+
catch(error){await recoverSnapshot(error,context);}}}).finally(()=>{state.streamWorker=false;if(state.pendingSnapshots.length)queueSnapshot();});return state.applyQueue; }
|
|
408
|
+
function queueCoordinate(data) { state.applyQueue=state.applyQueue.then(()=>{if(data.epoch===state.epoch&&data.version<state.version)return;
|
|
409
|
+
if(data.epoch!==state.epoch||data.version>state.version){
|
|
410
|
+
state.epoch=data.epoch;state.version=data.version;document.body.dataset.epoch=data.epoch;document.body.dataset.version=String(data.version);}
|
|
411
|
+
const visibleDocs=data.lifecycle&&data.lifecycle.handoff?data.docs:data.docs.filter(doc=>doc!=='table');
|
|
412
|
+
updateTabs(visibleDocs,state.doc);applyHealth(data.health);applyLifecycle(data.lifecycle);});return state.applyQueue; }
|
|
413
|
+
async function navigate(doc) { holdFollow();const navigation=++state.navigationSerial;const context=requestContext(doc);
|
|
414
|
+
state.pendingSnapshots=[];
|
|
415
|
+
try{const data=await getSnapshot(doc);if(!requestIsCurrent(context)||navigation!==state.navigationSerial)return;await applySnapshot(data);}
|
|
416
|
+
catch(error){await recoverSnapshot(error,context);} }
|
|
417
|
+
const explorerState={offset:0,limit:50,columns:[],allColumns:[],filters:[]};
|
|
418
|
+
function explorerValue(value){if(value==='true')return true;if(value==='false')return false;if(/^-?[0-9]+$/.test(value))return Number(value);if(/^-?[0-9]+\.[0-9]+$/.test(value))return Number(value);return value;}
|
|
419
|
+
async function loadExplorer(reset=false){const dialog=document.querySelector('[data-explorer]'),body=dialog.querySelector('[data-explorer-body]');if(reset)explorerState.offset=0;body.setAttribute('aria-busy','true');
|
|
420
|
+
const query=new URLSearchParams({offset:String(explorerState.offset),limit:String(explorerState.limit)});if(explorerState.columns.length)query.set('columns',explorerState.columns.join(','));if(explorerState.filters.length)query.set('filters',JSON.stringify(explorerState.filters));
|
|
421
|
+
try{const response=await fetch('/rows?'+query),data=await response.json();if(!response.ok)throw new Error();if(!explorerState.allColumns.length)explorerState.allColumns=[...data.columns];if(!explorerState.columns.length)explorerState.columns=[...data.columns];
|
|
422
|
+
const tools=document.createElement('div');tools.className='mr-explorer-tools';const columnLabel=document.createElement('label');columnLabel.textContent='Columns';const columnSelect=document.createElement('select');columnSelect.multiple=true;columnSelect.dataset.explorerColumns='';explorerState.allColumns.forEach(name=>{const option=document.createElement('option');option.value=name;option.textContent=name;option.selected=explorerState.columns.includes(name);columnSelect.append(option);});columnLabel.append(columnSelect);
|
|
423
|
+
const filterLabel=document.createElement('label');filterLabel.textContent='Filter column';const filterColumn=document.createElement('select');filterColumn.dataset.explorerFilterColumn='';explorerState.allColumns.forEach(name=>{const option=document.createElement('option');option.value=name;option.textContent=name;filterColumn.append(option);});filterLabel.append(filterColumn);
|
|
424
|
+
const opLabel=document.createElement('label');opLabel.textContent='Operator';const opSelect=document.createElement('select');opSelect.dataset.explorerFilterOp='';[['eq','='],['ne','not equal'],['lt','less than'],['le','at most'],['gt','greater than'],['ge','at least'],['is_null','is null'],['is_not_null','is not null']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;opSelect.append(option);});opLabel.append(opSelect);
|
|
425
|
+
const valueLabel=document.createElement('label');valueLabel.textContent='Value';const valueInput=document.createElement('input');valueInput.dataset.explorerFilterValue='';valueInput.type='text';valueInput.placeholder='bounded value';valueLabel.append(valueInput);const apply=document.createElement('button');apply.className='mr-button';apply.type='button';apply.dataset.explorerApply='';apply.textContent='Apply';
|
|
426
|
+
const previous=document.createElement('button');previous.className='mr-button';previous.type='button';previous.dataset.explorerPage='previous';previous.textContent='Previous';previous.disabled=data.offset===0;const next=document.createElement('button');next.className='mr-button';next.type='button';next.dataset.explorerPage='next';next.textContent='Next';next.disabled=!data.truncated;const status=document.createElement('span');status.className='mr-explorer-status';status.textContent=(data.rows.length?String(data.offset+1)+'-'+String(data.offset+data.rows.length):'0')+' of '+String(data.matched_row_count)+' matching rows';tools.append(columnLabel,filterLabel,opLabel,valueLabel,apply,previous,next,status);
|
|
427
|
+
const table=document.createElement('table'),head=document.createElement('thead'),hr=document.createElement('tr');data.columns.forEach(name=>{const th=document.createElement('th');th.scope='col';th.textContent=name;hr.append(th);});head.append(hr);table.append(head);const tbody=document.createElement('tbody');data.rows.forEach(row=>{const tr=document.createElement('tr');data.columns.forEach(name=>{const td=document.createElement('td'),value=row[name];td.textContent=value===null?'null':typeof value==='object'?JSON.stringify(value):String(value);tr.append(td);});tbody.append(tr);});table.append(tbody);const evidence=document.createElement('section');evidence.className='mr-explorer-evidence';evidence.setAttribute('aria-label','Verified explorer evidence');const evidenceTitle=document.createElement('h3');evidenceTitle.textContent='Verified profiles, findings & lineage';const evidenceBody=document.createElement('pre');evidenceBody.className='nb-json';evidenceBody.textContent=JSON.stringify(data.evidence,null,2);evidence.append(evidenceTitle,evidenceBody);body.replaceChildren(evidence,tools,table);
|
|
428
|
+
}catch(_error){body.textContent='Could not load the verified row window.';}finally{body.removeAttribute('aria-busy');}}
|
|
429
|
+
document.addEventListener('change', event => { const select=event.target.closest&&event.target.closest('[data-column-select]');if(select)activateInspector(select); });
|
|
430
|
+
const holdFollow=()=>{state.followHeld=true;const button=document.querySelector('[data-follow-agent]'),active=root.querySelector('[data-research-activity]');if(button)button.hidden=!active;};
|
|
431
|
+
if(window.addEventListener){window.addEventListener('wheel',holdFollow,{passive:true});window.addEventListener('touchstart',holdFollow,{passive:true});
|
|
432
|
+
window.addEventListener('keydown',event=>{if(['PageUp','PageDown','Home','End','ArrowUp','ArrowDown'].includes(event.key))holdFollow();});}
|
|
433
|
+
document.addEventListener('click', async event => {
|
|
434
|
+
const copy=event.target.closest('.nb-panel-btn');if(copy&©.textContent.trim()==='copy'){const panel=copy.closest('.nb-code');const pre=panel&&panel.querySelector('pre:not(.nb-lineno)');
|
|
435
|
+
if(pre){try{await navigator.clipboard.writeText(pre.textContent||'');const before=copy.textContent;copy.textContent='copied';toast('Code copied');setTimeout(()=>copy.textContent=before,1400);}catch(_error){toast('Could not copy code');}}return;}
|
|
436
|
+
const tab=event.target.closest('.mr-tab[data-doc]'); if(tab){await navigate(tab.dataset.doc);return;}
|
|
437
|
+
if(event.target.closest('[data-follow-agent]')){state.followHeld=false;event.target.closest('[data-follow-agent]').hidden=true;const active=root.querySelector('[data-research-activity]');if(active)active.scrollIntoView({block:'center',behavior:matchMedia('(prefers-reduced-motion: reduce)').matches?'auto':'smooth'});return;}
|
|
438
|
+
if(event.target.closest('[data-explore]')){const dialog=document.querySelector('[data-explorer]'),body=dialog.querySelector('[data-explorer-body]');body.textContent='Loading rows…';dialog.showModal();await loadExplorer(true);return;}
|
|
439
|
+
if(event.target.closest('[data-explorer-apply]')){const body=document.querySelector('[data-explorer-body]'),selected=[...body.querySelector('[data-explorer-columns]').selectedOptions].map(option=>option.value);if(!selected.length){toast('Select at least one column');return;}explorerState.columns=selected;const column=body.querySelector('[data-explorer-filter-column]').value,op=body.querySelector('[data-explorer-filter-op]').value,raw=body.querySelector('[data-explorer-filter-value]').value;explorerState.filters=raw||op==='is_null'||op==='is_not_null'?[{column,op,...(op==='is_null'||op==='is_not_null'?{}:{value:explorerValue(raw)})}]:[];await loadExplorer(true);return;}
|
|
440
|
+
const explorerPage=event.target.closest('[data-explorer-page]');if(explorerPage){explorerState.offset=Math.max(0,explorerState.offset+(explorerPage.dataset.explorerPage==='next'?explorerState.limit:-explorerState.limit));await loadExplorer();return;}
|
|
441
|
+
if(event.target.closest('[data-explorer-close]')){document.querySelector('[data-explorer]').close();return;}
|
|
442
|
+
if(event.target.closest('[data-files]')){const response=await fetch('/open-files',{method:'POST'});toast(response.ok?(state.handoff?'Revealed the table':'Opened the run location'):'Could not open the run location');return;}
|
|
443
|
+
});
|
|
444
|
+
setStatus('Connecting');
|
|
445
|
+
const source=new EventSource('/events'); source.onmessage=event=>{const data=JSON.parse(event.data);
|
|
446
|
+
if(data.visual_event){if(Number(data.cursor)<=state.cursor)return;queueSnapshot(state.doc,'',data.cursor);return;}
|
|
447
|
+
// Only 'stopped' is terminal. 'unavailable' is a steady state the page keeps serving research
|
|
448
|
+
// in -- an appended-to table notebook stays unavailable for as long as it is appended to --
|
|
449
|
+
// so it applies the health and then goes on fetching snapshots.
|
|
450
|
+
if(data.health.status==='stopped'){if(Number(data.cursor||0)>state.cursor){queueSnapshot(state.doc,'',data.cursor);return;}applyHealth(data.health);return;}
|
|
451
|
+
if(data.health.status==='unavailable')applyHealth(data.health);
|
|
452
|
+
if(Number(data.cursor||0)>state.cursor){queueSnapshot(state.doc,'',data.cursor);return;}
|
|
453
|
+
const handoff=!state.handoff&&data.lifecycle&&data.lifecycle.handoff;
|
|
454
|
+
if(!handoff&&state.doc&&Array.isArray(data.changed_docs)&&!data.changed_docs.includes(state.doc)){
|
|
455
|
+
queueCoordinate(data);return;}
|
|
456
|
+
if(data.epoch!==state.epoch||data.version>state.version||handoff)queueSnapshot(handoff?'table':state.doc,handoff?'':data.version);else applyLifecycle(data.lifecycle);};
|
|
457
|
+
source.onopen=async()=>{setStatus('Connecting');const context=requestContext(state.doc);try{const data=await getSnapshot(context.doc);if(requestIsCurrent(context)){if(data.epoch===state.epoch&&Number(data.version)===state.version&&Number(data.cursor||0)===state.cursor)commitSnapshot(data,context.doc);else await applySnapshot(data);}}
|
|
458
|
+
catch(error){await recoverSnapshot(error,context);}};
|
|
459
|
+
source.onerror=()=>{if(state.health!=='stopped'&&state.health!=='unavailable')setStatus('Reconnecting');};
|
|
460
|
+
window.setInterval(tickClock,1000);
|
|
461
|
+
})();
|
|
462
|
+
</script>
|
|
463
|
+
""".strip()
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
class ViewerExtraMissing(RuntimeError):
|
|
467
|
+
"""Compatibility type retained for one release; the base renderer never raises it."""
|
|
468
|
+
|
|
469
|
+
def __init__(self, message: str = _INSTALL_HINT) -> None:
|
|
470
|
+
super().__init__(message)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
@dataclass(frozen=True)
|
|
474
|
+
class _JournalEntry:
|
|
475
|
+
version: int
|
|
476
|
+
documents: dict[str, bytes]
|
|
477
|
+
mutation: dict[str, object] | None = None
|
|
478
|
+
visual_cursor: int = 0
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _parsed_notebook(raw: bytes) -> dict[str, object] | None:
|
|
482
|
+
try:
|
|
483
|
+
value = json.loads(raw.decode("utf-8"))
|
|
484
|
+
except (UnicodeDecodeError, ValueError, RecursionError):
|
|
485
|
+
return None
|
|
486
|
+
if (
|
|
487
|
+
not isinstance(value, dict)
|
|
488
|
+
or value.get("nbformat") != 4
|
|
489
|
+
or not isinstance(value.get("metadata"), dict)
|
|
490
|
+
or not isinstance(value.get("cells"), list)
|
|
491
|
+
):
|
|
492
|
+
return None
|
|
493
|
+
return value
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _declared_activity(cell: object) -> str | None | bool:
|
|
497
|
+
"""Return writing/executing, None for complete, or False for malformed metadata."""
|
|
498
|
+
|
|
499
|
+
if not isinstance(cell, dict) or not isinstance(cell.get("metadata"), dict):
|
|
500
|
+
return False
|
|
501
|
+
metadata = cell["metadata"]
|
|
502
|
+
mostlyright = metadata.get("mostlyright")
|
|
503
|
+
if mostlyright is None:
|
|
504
|
+
return None
|
|
505
|
+
if not isinstance(mostlyright, dict):
|
|
506
|
+
return False
|
|
507
|
+
activity = mostlyright.get("activity")
|
|
508
|
+
if activity is None:
|
|
509
|
+
return None
|
|
510
|
+
if (
|
|
511
|
+
not isinstance(activity, dict)
|
|
512
|
+
or set(activity) != {"schema_version", "state"}
|
|
513
|
+
or activity.get("schema_version") != _RESEARCH_ACTIVITY_SCHEMA
|
|
514
|
+
or activity.get("state") not in {"writing", "executing"}
|
|
515
|
+
):
|
|
516
|
+
return False
|
|
517
|
+
return str(activity["state"])
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _metadata_without_activity(cell: dict[str, object]) -> dict[str, object] | None:
|
|
521
|
+
metadata = cell.get("metadata")
|
|
522
|
+
if not isinstance(metadata, dict):
|
|
523
|
+
return None
|
|
524
|
+
result = dict(metadata)
|
|
525
|
+
mostlyright = result.get("mostlyright")
|
|
526
|
+
if mostlyright is not None:
|
|
527
|
+
if not isinstance(mostlyright, dict):
|
|
528
|
+
return None
|
|
529
|
+
cleaned = dict(mostlyright)
|
|
530
|
+
cleaned.pop("activity", None)
|
|
531
|
+
if cleaned:
|
|
532
|
+
result["mostlyright"] = cleaned
|
|
533
|
+
else:
|
|
534
|
+
result.pop("mostlyright", None)
|
|
535
|
+
return result
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def _standard_outputs(value: object, *, cell_count: int | None) -> bool:
|
|
539
|
+
if not isinstance(value, list):
|
|
540
|
+
return False
|
|
541
|
+
fields = {
|
|
542
|
+
"stream": {"output_type", "name", "text"},
|
|
543
|
+
"display_data": {"output_type", "data", "metadata"},
|
|
544
|
+
"execute_result": {"output_type", "data", "metadata", "execution_count"},
|
|
545
|
+
"error": {"output_type", "ename", "evalue", "traceback"},
|
|
546
|
+
}
|
|
547
|
+
for output in value:
|
|
548
|
+
if not isinstance(output, dict):
|
|
549
|
+
return False
|
|
550
|
+
output_type = output.get("output_type")
|
|
551
|
+
if not isinstance(output_type, str) or set(output) != fields.get(output_type):
|
|
552
|
+
return False
|
|
553
|
+
if output_type == "stream":
|
|
554
|
+
text = output.get("text")
|
|
555
|
+
if output.get("name") not in {"stdout", "stderr"} or not (
|
|
556
|
+
isinstance(text, str)
|
|
557
|
+
or (isinstance(text, list) and all(isinstance(part, str) for part in text))
|
|
558
|
+
):
|
|
559
|
+
return False
|
|
560
|
+
elif output_type in {"display_data", "execute_result"}:
|
|
561
|
+
if not isinstance(output.get("data"), dict) or not isinstance(
|
|
562
|
+
output.get("metadata"), dict
|
|
563
|
+
):
|
|
564
|
+
return False
|
|
565
|
+
if output_type == "execute_result" and (
|
|
566
|
+
isinstance(output.get("execution_count"), bool)
|
|
567
|
+
or not isinstance(output.get("execution_count"), int)
|
|
568
|
+
or output.get("execution_count", -1) < 0
|
|
569
|
+
or (cell_count is not None and output.get("execution_count") != cell_count)
|
|
570
|
+
):
|
|
571
|
+
return False
|
|
572
|
+
elif not (
|
|
573
|
+
isinstance(output.get("ename"), str)
|
|
574
|
+
and isinstance(output.get("evalue"), str)
|
|
575
|
+
and isinstance(output.get("traceback"), list)
|
|
576
|
+
and all(isinstance(line, str) for line in output["traceback"])
|
|
577
|
+
):
|
|
578
|
+
return False
|
|
579
|
+
return True
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _helper_cell_shape(cell: object, state: str | None) -> bool:
|
|
583
|
+
"""Recognize only cells emitted by the strict --cell-file helper contract."""
|
|
584
|
+
|
|
585
|
+
if not isinstance(cell, dict):
|
|
586
|
+
return False
|
|
587
|
+
cell_id = cell.get("id")
|
|
588
|
+
cell_type = cell.get("cell_type")
|
|
589
|
+
if (
|
|
590
|
+
not isinstance(cell_id, str)
|
|
591
|
+
or _RESEARCH_CELL_ID.fullmatch(cell_id) is None
|
|
592
|
+
or cell_type not in {"markdown", "code"}
|
|
593
|
+
or not isinstance(cell.get("source"), str)
|
|
594
|
+
or _declared_activity(cell) != state
|
|
595
|
+
):
|
|
596
|
+
return False
|
|
597
|
+
expected = {"cell_type", "id", "metadata", "source"}
|
|
598
|
+
if cell_type == "code":
|
|
599
|
+
expected |= {"execution_count", "outputs"}
|
|
600
|
+
count = cell.get("execution_count")
|
|
601
|
+
outputs = cell.get("outputs")
|
|
602
|
+
if not _standard_outputs(outputs, cell_count=count if state is None else None):
|
|
603
|
+
return False
|
|
604
|
+
if state is None:
|
|
605
|
+
if isinstance(count, bool) or not isinstance(count, int) or count < 0:
|
|
606
|
+
return False
|
|
607
|
+
elif count is not None or (state == "writing" and outputs):
|
|
608
|
+
return False
|
|
609
|
+
if state == "executing" and cell_type != "code":
|
|
610
|
+
return False
|
|
611
|
+
elif state == "executing":
|
|
612
|
+
return False
|
|
613
|
+
return set(cell) == expected
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _research_mutation(
|
|
617
|
+
before_raw: bytes,
|
|
618
|
+
after_raw: bytes,
|
|
619
|
+
*,
|
|
620
|
+
from_version: int,
|
|
621
|
+
version: int,
|
|
622
|
+
) -> dict[str, object] | None:
|
|
623
|
+
"""Prove one exact trailing-cell transition from two valid notebook documents."""
|
|
624
|
+
|
|
625
|
+
before = _parsed_notebook(before_raw)
|
|
626
|
+
after = _parsed_notebook(after_raw)
|
|
627
|
+
if before is None or after is None or before["metadata"] != after["metadata"]:
|
|
628
|
+
return None
|
|
629
|
+
old_cells = before["cells"]
|
|
630
|
+
new_cells = after["cells"]
|
|
631
|
+
assert isinstance(old_cells, list) and isinstance(new_cells, list)
|
|
632
|
+
phase_start = False
|
|
633
|
+
if len(new_cells) == len(old_cells) + 1 and old_cells == new_cells[:-1]:
|
|
634
|
+
old_cell = None
|
|
635
|
+
new_cell = new_cells[-1]
|
|
636
|
+
new_state = _declared_activity(new_cell)
|
|
637
|
+
if new_state != "writing" or not _helper_cell_shape(new_cell, "writing"):
|
|
638
|
+
return None
|
|
639
|
+
kind = "source"
|
|
640
|
+
phase = "writing"
|
|
641
|
+
phase_start = True
|
|
642
|
+
elif len(new_cells) == len(old_cells) and new_cells and old_cells[:-1] == new_cells[:-1]:
|
|
643
|
+
old_cell = old_cells[-1]
|
|
644
|
+
new_cell = new_cells[-1]
|
|
645
|
+
if not isinstance(old_cell, dict) or not isinstance(new_cell, dict):
|
|
646
|
+
return None
|
|
647
|
+
if old_cell.get("id") != new_cell.get("id") or old_cell.get("cell_type") != new_cell.get(
|
|
648
|
+
"cell_type"
|
|
649
|
+
):
|
|
650
|
+
return None
|
|
651
|
+
old_state = _declared_activity(old_cell)
|
|
652
|
+
new_state = _declared_activity(new_cell)
|
|
653
|
+
if old_state not in {"writing", "executing"} or new_state is False:
|
|
654
|
+
return None
|
|
655
|
+
if not _helper_cell_shape(old_cell, str(old_state)) or not _helper_cell_shape(
|
|
656
|
+
new_cell, None if new_state is None else str(new_state)
|
|
657
|
+
):
|
|
658
|
+
return None
|
|
659
|
+
if _metadata_without_activity(old_cell) != _metadata_without_activity(new_cell):
|
|
660
|
+
return None
|
|
661
|
+
old_source = old_cell["source"]
|
|
662
|
+
new_source = new_cell["source"]
|
|
663
|
+
assert isinstance(old_source, str) and isinstance(new_source, str)
|
|
664
|
+
if old_state == "writing" and new_state == "writing":
|
|
665
|
+
if old_source == new_source or not new_source.startswith(old_source):
|
|
666
|
+
return None
|
|
667
|
+
if old_cell.get("outputs") != new_cell.get("outputs"):
|
|
668
|
+
return None
|
|
669
|
+
kind, phase = "source", "writing"
|
|
670
|
+
elif old_state == "writing" and new_state == "executing":
|
|
671
|
+
if old_cell.get("cell_type") != "code" or old_source != new_source:
|
|
672
|
+
return None
|
|
673
|
+
kind, phase, phase_start = "state", "executing", True
|
|
674
|
+
elif old_state == "executing" and new_state == "executing":
|
|
675
|
+
if old_source != new_source or old_cell.get("outputs") == new_cell.get("outputs"):
|
|
676
|
+
return None
|
|
677
|
+
kind, phase = "output", "executing"
|
|
678
|
+
elif new_state is None and (
|
|
679
|
+
old_state == "executing"
|
|
680
|
+
or (old_state == "writing" and old_cell.get("cell_type") == "markdown")
|
|
681
|
+
):
|
|
682
|
+
if old_source != new_source:
|
|
683
|
+
return None
|
|
684
|
+
kind, phase, phase_start = "state", "complete", True
|
|
685
|
+
else:
|
|
686
|
+
return None
|
|
687
|
+
else:
|
|
688
|
+
return None
|
|
689
|
+
assert isinstance(new_cell, dict)
|
|
690
|
+
return {
|
|
691
|
+
"from_version": from_version,
|
|
692
|
+
"version": version,
|
|
693
|
+
"doc": "research",
|
|
694
|
+
"cell_id": new_cell["id"],
|
|
695
|
+
"kind": kind,
|
|
696
|
+
"phase": phase,
|
|
697
|
+
"phase_start": phase_start,
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _only_research_document_changed(before: dict[str, bytes], after: dict[str, bytes]) -> bool:
|
|
702
|
+
"""Keep document-set and table changes out of the cell-mutation channel."""
|
|
703
|
+
|
|
704
|
+
if set(before) != set(after) or before.get("research") == after.get("research"):
|
|
705
|
+
return False
|
|
706
|
+
return all(before[name] == after[name] for name in before if name != "research")
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
@dataclass(frozen=True)
|
|
710
|
+
class ViewerTarget:
|
|
711
|
+
"""One explicit viewer layout selected by the caller."""
|
|
712
|
+
|
|
713
|
+
mode: str
|
|
714
|
+
selected_path: Path
|
|
715
|
+
run_dir: Path
|
|
716
|
+
research_path: Path
|
|
717
|
+
table_path: Path
|
|
718
|
+
files_dir: Path
|
|
719
|
+
# Whether the caller named the research folder. Run-directory mode watches one either way, but
|
|
720
|
+
# only a named one was asked for -- and a command this viewer prints for the reader to run has
|
|
721
|
+
# to be the command they ran, not one carrying a flag they never passed.
|
|
722
|
+
explicit_research: bool = False
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def viewer_target(
|
|
726
|
+
*,
|
|
727
|
+
workspace: Path | None = None,
|
|
728
|
+
run_dir: Path | None = None,
|
|
729
|
+
research_dir: Path | None = None,
|
|
730
|
+
) -> ViewerTarget:
|
|
731
|
+
"""Return the exact notebook paths for one and only one explicit target mode."""
|
|
732
|
+
|
|
733
|
+
if (workspace is None) == (run_dir is None):
|
|
734
|
+
raise ValueError("viewer needs exactly one of workspace or run_dir")
|
|
735
|
+
selected = Path(os.path.abspath(os.fspath(workspace if workspace is not None else run_dir)))
|
|
736
|
+
explicit_research = (
|
|
737
|
+
Path(os.path.abspath(os.fspath(research_dir))) if research_dir is not None else None
|
|
738
|
+
)
|
|
739
|
+
if workspace is not None:
|
|
740
|
+
research_root = explicit_research or selected
|
|
741
|
+
return ViewerTarget(
|
|
742
|
+
mode="workspace",
|
|
743
|
+
selected_path=selected,
|
|
744
|
+
run_dir=selected / "result",
|
|
745
|
+
research_path=research_root / "research.ipynb",
|
|
746
|
+
table_path=selected / "result" / _SIDECAR_NAME,
|
|
747
|
+
files_dir=research_root,
|
|
748
|
+
explicit_research=explicit_research is not None,
|
|
749
|
+
)
|
|
750
|
+
research_root = explicit_research or selected.parent
|
|
751
|
+
return ViewerTarget(
|
|
752
|
+
mode="run-dir",
|
|
753
|
+
selected_path=selected,
|
|
754
|
+
run_dir=selected,
|
|
755
|
+
research_path=research_root / "research.ipynb",
|
|
756
|
+
table_path=selected / _SIDECAR_NAME,
|
|
757
|
+
files_dir=research_root,
|
|
758
|
+
explicit_research=explicit_research is not None,
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def prepare_viewer_target(
|
|
763
|
+
*,
|
|
764
|
+
workspace: Path | None = None,
|
|
765
|
+
run_dir: Path | None = None,
|
|
766
|
+
research_dir: Path | None = None,
|
|
767
|
+
) -> ViewerTarget:
|
|
768
|
+
"""Validate one CLI target and install its initial research notebook before binding.
|
|
769
|
+
|
|
770
|
+
Workspace mode deliberately has no implicit research location. A future workspace must stay
|
|
771
|
+
absent so ``mr-data init`` can install it atomically, while an existing target must replay as a
|
|
772
|
+
real Harness workspace. Research always lives in an explicit, already-existing real folder.
|
|
773
|
+
|
|
774
|
+
A notebook is seeded only where the caller named the folder to seed. Run-directory mode still
|
|
775
|
+
watches ``research.ipynb`` beside the selected run when ``--research-dir`` is omitted, but that
|
|
776
|
+
folder was never named -- it is usually a home or working directory the caller was not asking
|
|
777
|
+
the viewer to write into -- so nothing is created there and the page waits.
|
|
778
|
+
"""
|
|
779
|
+
|
|
780
|
+
target = viewer_target(workspace=workspace, run_dir=run_dir, research_dir=research_dir)
|
|
781
|
+
if workspace is not None:
|
|
782
|
+
if research_dir is None:
|
|
783
|
+
raise ViewerError(
|
|
784
|
+
"VIEWER_RESEARCH_DIR_REQUIRED",
|
|
785
|
+
"--workspace requires an explicit --research-dir that already exists",
|
|
786
|
+
)
|
|
787
|
+
_validate_workspace_target(target.selected_path)
|
|
788
|
+
else:
|
|
789
|
+
_validate_run_target(target.selected_path)
|
|
790
|
+
_require_real_directory(target.files_dir, label="research directory")
|
|
791
|
+
if research_dir is not None:
|
|
792
|
+
_seed_research_notebook(target.files_dir)
|
|
793
|
+
return target
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _validate_workspace_target(workspace: Path) -> None:
|
|
797
|
+
"""Accept an absent future target or a descriptor-validated Harness workspace."""
|
|
798
|
+
|
|
799
|
+
try:
|
|
800
|
+
info = os.lstat(workspace)
|
|
801
|
+
except FileNotFoundError:
|
|
802
|
+
# Still reject a missing target reached through a symlinked ancestor.
|
|
803
|
+
try:
|
|
804
|
+
_canonical_viewer_root(workspace)
|
|
805
|
+
except ValueError as exc:
|
|
806
|
+
raise ViewerError(
|
|
807
|
+
"VIEWER_WORKSPACE_INVALID", f"workspace path must use real directories: {workspace}"
|
|
808
|
+
) from exc
|
|
809
|
+
return
|
|
810
|
+
except OSError as exc:
|
|
811
|
+
raise ViewerError(
|
|
812
|
+
"VIEWER_WORKSPACE_UNAVAILABLE", f"workspace is unavailable: {workspace}"
|
|
813
|
+
) from exc
|
|
814
|
+
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
|
815
|
+
raise ViewerError(
|
|
816
|
+
"VIEWER_WORKSPACE_INVALID",
|
|
817
|
+
f"an existing --workspace must be a real Harness workspace: {workspace}",
|
|
818
|
+
)
|
|
819
|
+
try:
|
|
820
|
+
_canonical_viewer_root(workspace)
|
|
821
|
+
except ValueError as exc:
|
|
822
|
+
raise ViewerError(
|
|
823
|
+
"VIEWER_WORKSPACE_INVALID", f"workspace path must use real directories: {workspace}"
|
|
824
|
+
) from exc
|
|
825
|
+
try:
|
|
826
|
+
validate_workspace_structure(workspace)
|
|
827
|
+
except Exception as exc:
|
|
828
|
+
raise ViewerError(
|
|
829
|
+
"VIEWER_WORKSPACE_INVALID",
|
|
830
|
+
f"an existing --workspace must be a valid Harness workspace: {workspace}",
|
|
831
|
+
) from exc
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
def _validate_run_target(run_dir: Path) -> None:
|
|
835
|
+
"""Accept an absent future target or an existing descriptor-verified Harness Build."""
|
|
836
|
+
|
|
837
|
+
try:
|
|
838
|
+
info = os.lstat(run_dir)
|
|
839
|
+
except FileNotFoundError:
|
|
840
|
+
try:
|
|
841
|
+
_canonical_viewer_root(run_dir)
|
|
842
|
+
except ValueError as exc:
|
|
843
|
+
raise ViewerError(
|
|
844
|
+
"VIEWER_RUN_DIR_INVALID", f"run path must use real directories: {run_dir}"
|
|
845
|
+
) from exc
|
|
846
|
+
return
|
|
847
|
+
except OSError as exc:
|
|
848
|
+
raise ViewerError(
|
|
849
|
+
"VIEWER_RUN_DIR_UNAVAILABLE", f"run path is unavailable: {run_dir}"
|
|
850
|
+
) from exc
|
|
851
|
+
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
|
852
|
+
raise ViewerError(
|
|
853
|
+
"VIEWER_RUN_DIR_INVALID",
|
|
854
|
+
f"an existing --run-dir must be a verified Harness Build: {run_dir}",
|
|
855
|
+
)
|
|
856
|
+
try:
|
|
857
|
+
_canonical_viewer_root(run_dir)
|
|
858
|
+
verify_candidate(run_dir)
|
|
859
|
+
except Exception as exc:
|
|
860
|
+
raise ViewerError(
|
|
861
|
+
"VIEWER_RUN_DIR_INVALID",
|
|
862
|
+
f"an existing --run-dir must be a verified Harness Build: {run_dir}",
|
|
863
|
+
) from exc
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _require_real_directory(path: Path, *, label: str) -> None:
|
|
867
|
+
anchor: _PinnedPath | None = None
|
|
868
|
+
try:
|
|
869
|
+
canonical, _anchors = _canonical_viewer_root(path)
|
|
870
|
+
anchor = _PinnedPath(canonical)
|
|
871
|
+
descriptor = anchor.open_directory()
|
|
872
|
+
except (OSError, ValueError) as exc:
|
|
873
|
+
raise ViewerError(
|
|
874
|
+
"VIEWER_RESEARCH_DIR_INVALID", f"{label} must use real directories: {path}"
|
|
875
|
+
) from exc
|
|
876
|
+
try:
|
|
877
|
+
if not stat.S_ISDIR(os.fstat(descriptor).st_mode):
|
|
878
|
+
raise ViewerError(
|
|
879
|
+
"VIEWER_RESEARCH_DIR_INVALID", f"{label} must be a real directory: {path}"
|
|
880
|
+
)
|
|
881
|
+
finally:
|
|
882
|
+
os.close(descriptor)
|
|
883
|
+
if anchor is not None:
|
|
884
|
+
anchor.close()
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _seed_research_notebook(research_dir: Path) -> None:
|
|
888
|
+
"""Exclusively and atomically install a minimal claim-free notebook when absent."""
|
|
889
|
+
|
|
890
|
+
notebook = {
|
|
891
|
+
"cells": [
|
|
892
|
+
{
|
|
893
|
+
"cell_type": "markdown",
|
|
894
|
+
"id": "research-ready",
|
|
895
|
+
# Marked so the bundled helper can drop it the first time a real section is
|
|
896
|
+
# written. Without the marker it survives every update and the delivered notebook
|
|
897
|
+
# opens on an empty placeholder claiming the work has not started.
|
|
898
|
+
"metadata": {"mostlyright": {"seed": True}},
|
|
899
|
+
"source": (
|
|
900
|
+
"# Research workspace\n\n"
|
|
901
|
+
"Ready for research notes. Record the question, selected sources, and "
|
|
902
|
+
"supporting evidence here as the work proceeds."
|
|
903
|
+
),
|
|
904
|
+
}
|
|
905
|
+
],
|
|
906
|
+
"metadata": {"mostlyright": {"phase": "ready_for_research"}},
|
|
907
|
+
"nbformat": 4,
|
|
908
|
+
"nbformat_minor": 5,
|
|
909
|
+
}
|
|
910
|
+
raw = (json.dumps(notebook, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
|
|
911
|
+
try:
|
|
912
|
+
anchor = _PinnedPath(research_dir)
|
|
913
|
+
directory_fd = anchor.open_directory()
|
|
914
|
+
except (OSError, ValueError) as exc:
|
|
915
|
+
raise ViewerError(
|
|
916
|
+
"VIEWER_RESEARCH_DIR_INVALID",
|
|
917
|
+
f"research directory must use real directories: {research_dir}",
|
|
918
|
+
) from exc
|
|
919
|
+
try:
|
|
920
|
+
anchor.validate()
|
|
921
|
+
try:
|
|
922
|
+
existing = os.stat("research.ipynb", dir_fd=directory_fd, follow_symlinks=False)
|
|
923
|
+
except FileNotFoundError:
|
|
924
|
+
existing = None
|
|
925
|
+
if existing is not None:
|
|
926
|
+
if not _is_regular_single_link(existing):
|
|
927
|
+
raise ViewerError(
|
|
928
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
929
|
+
"existing research.ipynb must be a regular single-link file",
|
|
930
|
+
)
|
|
931
|
+
_validate_existing_research_notebook(directory_fd, existing)
|
|
932
|
+
return
|
|
933
|
+
installed = _install_seeded_notebook(directory_fd, raw, research_dir)
|
|
934
|
+
_validate_existing_research_notebook(directory_fd, installed)
|
|
935
|
+
finally:
|
|
936
|
+
os.close(directory_fd)
|
|
937
|
+
anchor.close()
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
def _install_seeded_notebook(directory_fd: int, raw: bytes, research_dir: Path) -> os.stat_result:
|
|
941
|
+
"""Write and atomically install the seed, reporting a refused write as a typed refusal.
|
|
942
|
+
|
|
943
|
+
ENOSPC on the volume, a read-only mount, or a filesystem with no hard links -- exFAT and some
|
|
944
|
+
SMB shares -- all surface here, and a raw OSError carries no code for the remediation map to
|
|
945
|
+
answer through.
|
|
946
|
+
"""
|
|
947
|
+
|
|
948
|
+
temporary = ""
|
|
949
|
+
descriptor = -1
|
|
950
|
+
try:
|
|
951
|
+
for _attempt in range(100):
|
|
952
|
+
temporary = f".research.ipynb.{secrets.token_hex(12)}"
|
|
953
|
+
try:
|
|
954
|
+
descriptor = os.open(
|
|
955
|
+
temporary,
|
|
956
|
+
os.O_WRONLY
|
|
957
|
+
| os.O_CREAT
|
|
958
|
+
| os.O_EXCL
|
|
959
|
+
| os.O_NOFOLLOW
|
|
960
|
+
| getattr(os, "O_CLOEXEC", 0),
|
|
961
|
+
0o600,
|
|
962
|
+
dir_fd=directory_fd,
|
|
963
|
+
)
|
|
964
|
+
break
|
|
965
|
+
except FileExistsError:
|
|
966
|
+
continue
|
|
967
|
+
else: # pragma: no cover - cryptographically improbable without hostile interference
|
|
968
|
+
raise ViewerError(
|
|
969
|
+
"VIEWER_RESEARCH_SEED_FAILED",
|
|
970
|
+
"could not allocate a research notebook temporary file",
|
|
971
|
+
)
|
|
972
|
+
offset = 0
|
|
973
|
+
while offset < len(raw):
|
|
974
|
+
written = os.write(descriptor, raw[offset:])
|
|
975
|
+
if written < 1:
|
|
976
|
+
raise OSError("research notebook write did not make progress")
|
|
977
|
+
offset += written
|
|
978
|
+
os.fsync(descriptor)
|
|
979
|
+
os.fchmod(descriptor, 0o644)
|
|
980
|
+
os.fsync(descriptor)
|
|
981
|
+
os.close(descriptor)
|
|
982
|
+
descriptor = -1
|
|
983
|
+
try:
|
|
984
|
+
# Linking a complete temporary file is an atomic no-replace installation. A racing
|
|
985
|
+
# writer wins without ever being overwritten.
|
|
986
|
+
os.link(
|
|
987
|
+
temporary,
|
|
988
|
+
"research.ipynb",
|
|
989
|
+
src_dir_fd=directory_fd,
|
|
990
|
+
dst_dir_fd=directory_fd,
|
|
991
|
+
follow_symlinks=False,
|
|
992
|
+
)
|
|
993
|
+
except FileExistsError:
|
|
994
|
+
pass
|
|
995
|
+
os.unlink(temporary, dir_fd=directory_fd)
|
|
996
|
+
temporary = ""
|
|
997
|
+
os.fsync(directory_fd)
|
|
998
|
+
installed = os.stat("research.ipynb", dir_fd=directory_fd, follow_symlinks=False)
|
|
999
|
+
if not _is_regular_single_link(installed):
|
|
1000
|
+
raise ViewerError(
|
|
1001
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1002
|
+
"research.ipynb was replaced during viewer startup",
|
|
1003
|
+
)
|
|
1004
|
+
return installed
|
|
1005
|
+
except ViewerError:
|
|
1006
|
+
raise
|
|
1007
|
+
except OSError as exc:
|
|
1008
|
+
raise ViewerError(
|
|
1009
|
+
"VIEWER_RESEARCH_SEED_FAILED",
|
|
1010
|
+
f"research.ipynb could not be installed in {research_dir}: {exc}",
|
|
1011
|
+
) from exc
|
|
1012
|
+
finally:
|
|
1013
|
+
if descriptor >= 0:
|
|
1014
|
+
os.close(descriptor)
|
|
1015
|
+
if temporary:
|
|
1016
|
+
try:
|
|
1017
|
+
os.unlink(temporary, dir_fd=directory_fd)
|
|
1018
|
+
except FileNotFoundError:
|
|
1019
|
+
pass
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def _validate_existing_research_notebook(directory_fd: int, expected: os.stat_result) -> None:
|
|
1023
|
+
"""Read and render-check an existing research notebook through its pinned directory."""
|
|
1024
|
+
|
|
1025
|
+
if expected.st_size > _MAX_NOTEBOOK_BYTES:
|
|
1026
|
+
raise ViewerError(
|
|
1027
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1028
|
+
f"existing research.ipynb exceeds the {_MAX_NOTEBOOK_BYTES}-byte viewer limit",
|
|
1029
|
+
)
|
|
1030
|
+
descriptor = -1
|
|
1031
|
+
try:
|
|
1032
|
+
descriptor = os.open(
|
|
1033
|
+
"research.ipynb",
|
|
1034
|
+
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
1035
|
+
dir_fd=directory_fd,
|
|
1036
|
+
)
|
|
1037
|
+
opened = os.fstat(descriptor)
|
|
1038
|
+
if not _is_regular_single_link(opened) or _stat_signature(opened) != _stat_signature(
|
|
1039
|
+
expected
|
|
1040
|
+
):
|
|
1041
|
+
raise ViewerError(
|
|
1042
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1043
|
+
"existing research.ipynb changed during viewer startup",
|
|
1044
|
+
)
|
|
1045
|
+
chunks: list[bytes] = []
|
|
1046
|
+
remaining = _MAX_NOTEBOOK_BYTES + 1
|
|
1047
|
+
while remaining:
|
|
1048
|
+
chunk = os.read(descriptor, min(_READ_CHUNK_BYTES, remaining))
|
|
1049
|
+
if not chunk:
|
|
1050
|
+
break
|
|
1051
|
+
chunks.append(chunk)
|
|
1052
|
+
remaining -= len(chunk)
|
|
1053
|
+
raw = b"".join(chunks)
|
|
1054
|
+
if len(raw) > _MAX_NOTEBOOK_BYTES or len(raw) != opened.st_size:
|
|
1055
|
+
raise ViewerError(
|
|
1056
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1057
|
+
"existing research.ipynb changed or exceeds the viewer limit",
|
|
1058
|
+
)
|
|
1059
|
+
try:
|
|
1060
|
+
decoded = raw.decode("utf-8")
|
|
1061
|
+
parsed = json.loads(decoded)
|
|
1062
|
+
if not isinstance(parsed, dict):
|
|
1063
|
+
raise ValueError("notebook root must be an object")
|
|
1064
|
+
render_notebook(parsed, mode="embed", filename="research.ipynb")
|
|
1065
|
+
except (
|
|
1066
|
+
UnicodeDecodeError,
|
|
1067
|
+
ValueError,
|
|
1068
|
+
TypeError,
|
|
1069
|
+
KeyError,
|
|
1070
|
+
OverflowError,
|
|
1071
|
+
RecursionError,
|
|
1072
|
+
) as exc:
|
|
1073
|
+
raise ViewerError(
|
|
1074
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1075
|
+
"existing research.ipynb is not a valid renderable UTF-8 JSON notebook",
|
|
1076
|
+
) from exc
|
|
1077
|
+
current = os.fstat(descriptor)
|
|
1078
|
+
named = os.stat("research.ipynb", dir_fd=directory_fd, follow_symlinks=False)
|
|
1079
|
+
if _stat_signature(current) != _stat_signature(opened) or _stat_signature(
|
|
1080
|
+
named
|
|
1081
|
+
) != _stat_signature(opened):
|
|
1082
|
+
raise ViewerError(
|
|
1083
|
+
"VIEWER_RESEARCH_NOTEBOOK_INVALID",
|
|
1084
|
+
"existing research.ipynb changed during viewer startup",
|
|
1085
|
+
)
|
|
1086
|
+
except ViewerError:
|
|
1087
|
+
raise
|
|
1088
|
+
except OSError as exc:
|
|
1089
|
+
raise ViewerError(
|
|
1090
|
+
"VIEWER_RESEARCH_NOTEBOOK_UNAVAILABLE",
|
|
1091
|
+
"existing research.ipynb could not be read safely",
|
|
1092
|
+
) from exc
|
|
1093
|
+
finally:
|
|
1094
|
+
if descriptor >= 0:
|
|
1095
|
+
os.close(descriptor)
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def _stat_signature(info: os.stat_result) -> _FileSignature:
|
|
1099
|
+
return (
|
|
1100
|
+
info.st_dev,
|
|
1101
|
+
info.st_ino,
|
|
1102
|
+
info.st_mode,
|
|
1103
|
+
info.st_nlink,
|
|
1104
|
+
info.st_size,
|
|
1105
|
+
info.st_mtime_ns,
|
|
1106
|
+
info.st_ctime_ns,
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1109
|
+
|
|
1110
|
+
def _is_regular_single_link(info: os.stat_result) -> bool:
|
|
1111
|
+
return stat.S_ISREG(info.st_mode) and info.st_nlink == 1
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def _directory_identity(info: os.stat_result) -> _DirectoryIdentity:
|
|
1115
|
+
return (info.st_dev, info.st_ino, info.st_mode)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def _canonical_viewer_root(
|
|
1119
|
+
target: Path,
|
|
1120
|
+
) -> tuple[Path, tuple[tuple[Path, _DirectoryIdentity], ...]]:
|
|
1121
|
+
"""Canonicalize one selected root and bind every ancestor that already exists."""
|
|
1122
|
+
|
|
1123
|
+
absolute = Path(os.path.abspath(os.fspath(target)))
|
|
1124
|
+
canonical = absolute.resolve(strict=False)
|
|
1125
|
+
if canonical != absolute:
|
|
1126
|
+
raise ValueError("viewer root must not contain symlinked ancestors")
|
|
1127
|
+
chain: list[tuple[Path, _DirectoryIdentity]] = []
|
|
1128
|
+
current = Path(canonical.anchor)
|
|
1129
|
+
components = canonical.parts[1:]
|
|
1130
|
+
for component in (None, *components):
|
|
1131
|
+
if component is not None:
|
|
1132
|
+
current /= component
|
|
1133
|
+
try:
|
|
1134
|
+
info = os.lstat(current)
|
|
1135
|
+
except FileNotFoundError:
|
|
1136
|
+
break
|
|
1137
|
+
except OSError as exc:
|
|
1138
|
+
raise ValueError("viewer root ancestors are unavailable") from exc
|
|
1139
|
+
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
|
1140
|
+
raise ValueError("viewer root ancestors must be real directories")
|
|
1141
|
+
chain.append((current, _directory_identity(info)))
|
|
1142
|
+
return canonical, tuple(chain)
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
class _PinnedPath:
|
|
1146
|
+
"""Walk absolute paths from a retained filesystem-root descriptor without following links."""
|
|
1147
|
+
|
|
1148
|
+
def __init__(self, path: Path) -> None:
|
|
1149
|
+
self.path = Path(os.path.abspath(os.fspath(path)))
|
|
1150
|
+
self._root_fd = os.open(self.path.anchor, _DIRECTORY_FLAGS)
|
|
1151
|
+
root = os.fstat(self._root_fd)
|
|
1152
|
+
if not stat.S_ISDIR(root.st_mode):
|
|
1153
|
+
self.close()
|
|
1154
|
+
raise ValueError("viewer filesystem anchor is not a directory")
|
|
1155
|
+
self._expected: dict[tuple[str, ...], _DirectoryIdentity] = {(): _directory_identity(root)}
|
|
1156
|
+
descriptor = os.dup(self._root_fd)
|
|
1157
|
+
walked: list[str] = []
|
|
1158
|
+
try:
|
|
1159
|
+
for component in self.path.parts[1:]:
|
|
1160
|
+
try:
|
|
1161
|
+
child = os.open(component, _DIRECTORY_FLAGS, dir_fd=descriptor)
|
|
1162
|
+
except FileNotFoundError:
|
|
1163
|
+
break
|
|
1164
|
+
opened = os.fstat(child)
|
|
1165
|
+
if not stat.S_ISDIR(opened.st_mode):
|
|
1166
|
+
os.close(child)
|
|
1167
|
+
raise ValueError("viewer path component is not a directory")
|
|
1168
|
+
os.close(descriptor)
|
|
1169
|
+
descriptor = child
|
|
1170
|
+
walked.append(component)
|
|
1171
|
+
self._expected[tuple(walked)] = _directory_identity(opened)
|
|
1172
|
+
except BaseException:
|
|
1173
|
+
self.close()
|
|
1174
|
+
raise
|
|
1175
|
+
finally:
|
|
1176
|
+
os.close(descriptor)
|
|
1177
|
+
|
|
1178
|
+
def close(self) -> None:
|
|
1179
|
+
if getattr(self, "_root_fd", -1) >= 0:
|
|
1180
|
+
os.close(self._root_fd)
|
|
1181
|
+
self._root_fd = -1
|
|
1182
|
+
|
|
1183
|
+
def open_directory(self, path: Path | None = None) -> int:
|
|
1184
|
+
target = self.path if path is None else Path(os.path.abspath(os.fspath(path)))
|
|
1185
|
+
try:
|
|
1186
|
+
relative = target.relative_to(Path(target.anchor))
|
|
1187
|
+
except ValueError as exc:
|
|
1188
|
+
raise OSError(errno.EINVAL, "viewer path has a different filesystem anchor") from exc
|
|
1189
|
+
descriptor = os.dup(self._root_fd)
|
|
1190
|
+
walked: list[str] = []
|
|
1191
|
+
try:
|
|
1192
|
+
if _directory_identity(os.fstat(descriptor)) != self._expected[()]:
|
|
1193
|
+
raise OSError(errno.ESTALE, "viewer filesystem anchor changed")
|
|
1194
|
+
for component in relative.parts:
|
|
1195
|
+
child = os.open(component, _DIRECTORY_FLAGS, dir_fd=descriptor)
|
|
1196
|
+
opened = os.fstat(child)
|
|
1197
|
+
if not stat.S_ISDIR(opened.st_mode):
|
|
1198
|
+
os.close(child)
|
|
1199
|
+
raise OSError(errno.ENOTDIR, "viewer path component is not a directory")
|
|
1200
|
+
os.close(descriptor)
|
|
1201
|
+
descriptor = child
|
|
1202
|
+
walked.append(component)
|
|
1203
|
+
expected = self._expected.get(tuple(walked))
|
|
1204
|
+
if expected is not None and _directory_identity(opened) != expected:
|
|
1205
|
+
raise OSError(errno.ESTALE, "viewer path ancestor changed identity")
|
|
1206
|
+
return descriptor
|
|
1207
|
+
except BaseException:
|
|
1208
|
+
os.close(descriptor)
|
|
1209
|
+
raise
|
|
1210
|
+
|
|
1211
|
+
def validate(self) -> None:
|
|
1212
|
+
descriptor = self.open_directory()
|
|
1213
|
+
os.close(descriptor)
|
|
1214
|
+
|
|
1215
|
+
def open_parent(self, path: Path) -> tuple[int, str]:
|
|
1216
|
+
absolute = Path(os.path.abspath(os.fspath(path)))
|
|
1217
|
+
if not absolute.name:
|
|
1218
|
+
raise OSError(errno.EINVAL, "viewer member has no name")
|
|
1219
|
+
return self.open_directory(absolute.parent), absolute.name
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
@dataclass
|
|
1223
|
+
class _RunRecoveryLease:
|
|
1224
|
+
"""A descriptor-pinned run and its exact exclusive sidecar-writer interval."""
|
|
1225
|
+
|
|
1226
|
+
run_handle: _CandidateRunHandle
|
|
1227
|
+
lock_fd: int
|
|
1228
|
+
lock_identity: _DirectoryIdentity
|
|
1229
|
+
|
|
1230
|
+
@property
|
|
1231
|
+
def run_fd(self) -> int:
|
|
1232
|
+
return self.run_handle.run_fd
|
|
1233
|
+
|
|
1234
|
+
def validate(self) -> None:
|
|
1235
|
+
"""Require both selected path and named lock to remain the objects we acquired."""
|
|
1236
|
+
|
|
1237
|
+
_validate_candidate_run_handle(self.run_handle)
|
|
1238
|
+
opened = os.fstat(self.lock_fd)
|
|
1239
|
+
if not _is_regular_single_link(opened) or _entry_identity(opened) != self.lock_identity:
|
|
1240
|
+
raise OSError(errno.ESTALE, "run build activity lock changed identity")
|
|
1241
|
+
named = os.stat(
|
|
1242
|
+
RUN_BUILD_ACTIVITY_LOCK,
|
|
1243
|
+
dir_fd=self.run_fd,
|
|
1244
|
+
follow_symlinks=False,
|
|
1245
|
+
)
|
|
1246
|
+
if not _is_regular_single_link(named) or _entry_identity(named) != self.lock_identity:
|
|
1247
|
+
raise OSError(errno.ESTALE, "run build activity lock path changed identity")
|
|
1248
|
+
_validate_candidate_run_handle(self.run_handle)
|
|
1249
|
+
|
|
1250
|
+
def manifest_observation(self) -> _FileSignature | None:
|
|
1251
|
+
"""Read the exact pinned run's manifest identity for the one-attempt latch."""
|
|
1252
|
+
|
|
1253
|
+
candidate_fd = -1
|
|
1254
|
+
try:
|
|
1255
|
+
candidate_fd = os.open("candidate", _DIRECTORY_FLAGS, dir_fd=self.run_fd)
|
|
1256
|
+
manifest = os.stat("manifest.json", dir_fd=candidate_fd, follow_symlinks=False)
|
|
1257
|
+
except OSError:
|
|
1258
|
+
return None
|
|
1259
|
+
finally:
|
|
1260
|
+
if candidate_fd >= 0:
|
|
1261
|
+
os.close(candidate_fd)
|
|
1262
|
+
if not _is_regular_single_link(manifest):
|
|
1263
|
+
return None
|
|
1264
|
+
return _stat_signature(manifest)
|
|
1265
|
+
|
|
1266
|
+
def close(self) -> None:
|
|
1267
|
+
"""Release exclusion only after every descriptor-relative recovery step is finished."""
|
|
1268
|
+
|
|
1269
|
+
if fcntl is None:
|
|
1270
|
+
raise ViewerError(
|
|
1271
|
+
"VIEWER_PLATFORM_UNSUPPORTED",
|
|
1272
|
+
"secure viewer locking is unavailable on this platform",
|
|
1273
|
+
)
|
|
1274
|
+
try:
|
|
1275
|
+
fcntl.flock(self.lock_fd, fcntl.LOCK_UN)
|
|
1276
|
+
finally:
|
|
1277
|
+
try:
|
|
1278
|
+
os.close(self.lock_fd)
|
|
1279
|
+
finally:
|
|
1280
|
+
self.run_handle.close()
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
def _sidecar_lifecycle_grows_monotonically(
|
|
1284
|
+
previous: Mapping[str, object], lifecycle: Mapping[str, object]
|
|
1285
|
+
) -> bool:
|
|
1286
|
+
"""Allow one same-feed sidecar transition and make its first outcome sticky."""
|
|
1287
|
+
|
|
1288
|
+
before = previous.get("sidecar_state")
|
|
1289
|
+
after = lifecycle.get("sidecar_state")
|
|
1290
|
+
if before is None and after is None:
|
|
1291
|
+
# The visual projection narrates no sidecar outcome, so there is no first outcome here to
|
|
1292
|
+
# make sticky, and the caller has already established that this is structurally the same
|
|
1293
|
+
# attempt. Falling through to the refusal below instead froze a completed visual lifecycle
|
|
1294
|
+
# forever: the one same-attempt change the caller exists to allow -- independent dataset
|
|
1295
|
+
# admission turning handoff on -- was refused along with everything else, so a Dataset that
|
|
1296
|
+
# became admissible after its run had narrated completion never reached the page.
|
|
1297
|
+
return True
|
|
1298
|
+
if before == "absent":
|
|
1299
|
+
return after in {"absent", "invalid", "started"}
|
|
1300
|
+
if before == "invalid":
|
|
1301
|
+
return after == "invalid"
|
|
1302
|
+
if before == "started":
|
|
1303
|
+
return after in {"started", "ready", "failed"}
|
|
1304
|
+
if before in {"ready", "failed"}:
|
|
1305
|
+
# Ready and failed are both terminal outcomes for this attempt/path. Table admission may
|
|
1306
|
+
# still change handoff after ready, but the narrated outcome itself is immutable.
|
|
1307
|
+
return after == before
|
|
1308
|
+
return False
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def _visual_event_mutation(event: VisualRunEvent, before: ReducedRun) -> dict[str, object] | None:
|
|
1312
|
+
"""Translate one durable visual event into browser mutation intent.
|
|
1313
|
+
|
|
1314
|
+
The event sequence and object version are authoritative. Notebook diffs remain a defensive
|
|
1315
|
+
DOM-equivalence check, never the source of identity, ordering, follow behavior, or narration.
|
|
1316
|
+
"""
|
|
1317
|
+
|
|
1318
|
+
if not event.event_type.startswith("cell."):
|
|
1319
|
+
return None
|
|
1320
|
+
kind_by_event = {
|
|
1321
|
+
"cell.created": "source",
|
|
1322
|
+
"cell.source_extended": "source",
|
|
1323
|
+
"cell.source_revised": "revision",
|
|
1324
|
+
"cell.moved": "move",
|
|
1325
|
+
"cell.execution_started": "state",
|
|
1326
|
+
"cell.output_appended": "output",
|
|
1327
|
+
"cell.output_replaced": "output",
|
|
1328
|
+
"cell.completed": "state",
|
|
1329
|
+
"cell.failed": "state",
|
|
1330
|
+
}
|
|
1331
|
+
phase_by_event = {
|
|
1332
|
+
"cell.created": "writing",
|
|
1333
|
+
"cell.source_extended": "writing",
|
|
1334
|
+
"cell.source_revised": "writing",
|
|
1335
|
+
"cell.moved": "writing",
|
|
1336
|
+
"cell.execution_started": "executing",
|
|
1337
|
+
"cell.output_appended": "executing",
|
|
1338
|
+
"cell.output_replaced": "executing",
|
|
1339
|
+
"cell.completed": "complete",
|
|
1340
|
+
"cell.failed": "failed",
|
|
1341
|
+
}
|
|
1342
|
+
kind = kind_by_event.get(event.event_type)
|
|
1343
|
+
if kind is None:
|
|
1344
|
+
return None
|
|
1345
|
+
prior = before.cells.get(event.object_id)
|
|
1346
|
+
result: dict[str, object] = {
|
|
1347
|
+
"source": "visual-run",
|
|
1348
|
+
"doc": "research",
|
|
1349
|
+
"cursor": event.sequence,
|
|
1350
|
+
"from_cursor": before.cursor,
|
|
1351
|
+
"event_type": event.event_type,
|
|
1352
|
+
"cell_id": event.object_id,
|
|
1353
|
+
"object_version": event.object_version,
|
|
1354
|
+
"previous_object_version": event.previous_object_version,
|
|
1355
|
+
"kind": kind,
|
|
1356
|
+
"phase": phase_by_event[event.event_type],
|
|
1357
|
+
"announcement": f"Research cell {phase_by_event[event.event_type]}",
|
|
1358
|
+
}
|
|
1359
|
+
if prior is not None:
|
|
1360
|
+
result["previous_status"] = prior.status
|
|
1361
|
+
if event.event_type == "cell.moved":
|
|
1362
|
+
result["after_cell_id"] = str(event.payload["after_cell_id"])
|
|
1363
|
+
if event.event_type.startswith("cell.output_"):
|
|
1364
|
+
result["output_index"] = int(event.payload["output_index"])
|
|
1365
|
+
return result
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
def _visual_lifecycle(
|
|
1369
|
+
reduced: ReducedRun,
|
|
1370
|
+
*,
|
|
1371
|
+
table_ready: bool,
|
|
1372
|
+
admitted_candidate_digest: str | None,
|
|
1373
|
+
) -> dict[str, object]:
|
|
1374
|
+
"""Project browser lifecycle only from authoritative reduced protocol objects."""
|
|
1375
|
+
|
|
1376
|
+
handoff = bool(
|
|
1377
|
+
table_ready
|
|
1378
|
+
and reduced.table_ready
|
|
1379
|
+
and admitted_candidate_digest is not None
|
|
1380
|
+
and admitted_candidate_digest == reduced.candidate_digest
|
|
1381
|
+
)
|
|
1382
|
+
lines: list[dict[str, object]] = []
|
|
1383
|
+
last_stage = "Research"
|
|
1384
|
+
for item in reduced.objects.values():
|
|
1385
|
+
if item.kind in {"cell", "run", "phase"}:
|
|
1386
|
+
continue
|
|
1387
|
+
status = (
|
|
1388
|
+
"failed"
|
|
1389
|
+
if item.status in {"failed", "rejected"}
|
|
1390
|
+
else "done"
|
|
1391
|
+
if item.status == "complete"
|
|
1392
|
+
else "running"
|
|
1393
|
+
)
|
|
1394
|
+
if item.kind == "source":
|
|
1395
|
+
label = "Source evidence"
|
|
1396
|
+
detail = str(item.payload.get("observation_kind", item.payload.get("source_kind", "")))
|
|
1397
|
+
elif item.kind == "transformation":
|
|
1398
|
+
operation = str(item.payload.get("operation", "transformation"))
|
|
1399
|
+
label = f"{operation.title()} transformation"
|
|
1400
|
+
detail = f"{item.stage} · {item.payload.get('row_count', 'pending')} rows"
|
|
1401
|
+
status = "done" if item.version >= 2 and item.status != "failed" else status
|
|
1402
|
+
elif item.kind == "quality":
|
|
1403
|
+
label = "Quality evidence"
|
|
1404
|
+
detail = (
|
|
1405
|
+
f"{item.payload.get('checks_passed', 0)}/"
|
|
1406
|
+
f"{item.payload.get('checks_total', 0)} checks"
|
|
1407
|
+
)
|
|
1408
|
+
elif item.kind == "table":
|
|
1409
|
+
label = "Table written"
|
|
1410
|
+
detail = (
|
|
1411
|
+
f"{item.payload.get('row_count', 0)} rows · "
|
|
1412
|
+
f"{item.payload.get('column_count', 0)} columns"
|
|
1413
|
+
)
|
|
1414
|
+
elif item.kind == "build":
|
|
1415
|
+
digest = str(item.payload.get("candidate_digest", ""))
|
|
1416
|
+
lines.append(
|
|
1417
|
+
{
|
|
1418
|
+
"key": "build:sealed",
|
|
1419
|
+
"label": "Build sealed",
|
|
1420
|
+
"detail": digest[:16],
|
|
1421
|
+
"status": "done",
|
|
1422
|
+
}
|
|
1423
|
+
)
|
|
1424
|
+
label = "Build verified"
|
|
1425
|
+
detail = digest[:16]
|
|
1426
|
+
status = "done" if item.version >= 2 and item.status == "complete" else "running"
|
|
1427
|
+
elif item.kind == "notebook":
|
|
1428
|
+
label = f"{item.object_id.partition(':')[2].title()} notebook"
|
|
1429
|
+
detail = str(item.payload.get("notebook_digest", ""))[:16]
|
|
1430
|
+
else:
|
|
1431
|
+
continue
|
|
1432
|
+
last_stage = item.stage.replace("-", " ").title()
|
|
1433
|
+
lines.append(
|
|
1434
|
+
{
|
|
1435
|
+
"key": item.object_id,
|
|
1436
|
+
"label": label,
|
|
1437
|
+
"detail": detail,
|
|
1438
|
+
"status": status,
|
|
1439
|
+
}
|
|
1440
|
+
)
|
|
1441
|
+
if item.kind == "transformation" and item.version >= 3:
|
|
1442
|
+
lines.append(
|
|
1443
|
+
{
|
|
1444
|
+
"key": f"{item.object_id}:evidence",
|
|
1445
|
+
"label": "Stage evidence",
|
|
1446
|
+
"detail": f"{item.payload.get('operation', 'operation')} · verified observation",
|
|
1447
|
+
"status": "done",
|
|
1448
|
+
}
|
|
1449
|
+
)
|
|
1450
|
+
status = (
|
|
1451
|
+
"failed"
|
|
1452
|
+
if reduced.lifecycle in {"failed", "interrupted"}
|
|
1453
|
+
else "completed"
|
|
1454
|
+
if reduced.lifecycle == "completed"
|
|
1455
|
+
else "running"
|
|
1456
|
+
)
|
|
1457
|
+
done = sum(line["status"] == "done" for line in lines)
|
|
1458
|
+
progress = 100 if status == "completed" else int(96 * done / max(len(lines) + 1, 1))
|
|
1459
|
+
if reduced.active_object_id is not None:
|
|
1460
|
+
active = reduced.objects.get(reduced.active_object_id)
|
|
1461
|
+
if active is not None and active.kind not in {"cell", "run", "phase"}:
|
|
1462
|
+
last_stage = active.stage.replace("-", " ").title()
|
|
1463
|
+
if not lines and status == "running":
|
|
1464
|
+
status = "idle"
|
|
1465
|
+
return {
|
|
1466
|
+
"status": status,
|
|
1467
|
+
"phase": "Table ready" if handoff else last_stage,
|
|
1468
|
+
"progress": progress,
|
|
1469
|
+
"lines": lines,
|
|
1470
|
+
"handoff": handoff,
|
|
1471
|
+
"sealed": reduced.candidate_digest is not None,
|
|
1472
|
+
"legacy_handoff_allowed": False,
|
|
1473
|
+
"started_at": None,
|
|
1474
|
+
"ended_at": None,
|
|
1475
|
+
"terminal": (
|
|
1476
|
+
{
|
|
1477
|
+
"event": reduced.lifecycle,
|
|
1478
|
+
"title": reduced.terminal_error.code,
|
|
1479
|
+
"detail": reduced.terminal_error.message,
|
|
1480
|
+
}
|
|
1481
|
+
if reduced.terminal_error is not None
|
|
1482
|
+
else None
|
|
1483
|
+
),
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
class _ViewerState:
|
|
1488
|
+
"""Atomic notebook snapshots and a bounded journal of real file changes."""
|
|
1489
|
+
|
|
1490
|
+
def __init__(self, target: ViewerTarget | Path) -> None:
|
|
1491
|
+
if isinstance(target, Path):
|
|
1492
|
+
target = viewer_target(run_dir=target)
|
|
1493
|
+
canonical, _selected_anchors = _canonical_viewer_root(target.selected_path)
|
|
1494
|
+
canonical_research, _research_anchors = _canonical_viewer_root(target.research_path.parent)
|
|
1495
|
+
# Canonicalizing names the research folder whether or not the caller did, so the caller's
|
|
1496
|
+
# answer is carried across rather than re-derived from the rebuilt paths.
|
|
1497
|
+
target = replace(
|
|
1498
|
+
viewer_target(
|
|
1499
|
+
workspace=canonical if target.mode == "workspace" else None,
|
|
1500
|
+
run_dir=canonical if target.mode == "run-dir" else None,
|
|
1501
|
+
research_dir=canonical_research,
|
|
1502
|
+
),
|
|
1503
|
+
explicit_research=target.explicit_research,
|
|
1504
|
+
)
|
|
1505
|
+
self.target = target
|
|
1506
|
+
self.run_dir = target.run_dir
|
|
1507
|
+
self.research_path = target.research_path
|
|
1508
|
+
self.table_path = target.table_path
|
|
1509
|
+
self.files_dir = target.files_dir
|
|
1510
|
+
self._selected_path = _PinnedPath(target.selected_path)
|
|
1511
|
+
self._research_path = _PinnedPath(target.research_path.parent)
|
|
1512
|
+
self._visual_transport: VisualRunTransport | None = None
|
|
1513
|
+
self._visual_refresh_lock = threading.Lock()
|
|
1514
|
+
self._visual_present = False
|
|
1515
|
+
self._visual_error: VisualRunError | None = None
|
|
1516
|
+
self._visual_cursor = 0
|
|
1517
|
+
self._visual_state = ReducedRun()
|
|
1518
|
+
try:
|
|
1519
|
+
self._research_fd = self._research_path.open_directory()
|
|
1520
|
+
except BaseException:
|
|
1521
|
+
self._selected_path.close()
|
|
1522
|
+
self._research_path.close()
|
|
1523
|
+
raise
|
|
1524
|
+
self._visual_present = self._visual_boundary_exists()
|
|
1525
|
+
if self._visual_present:
|
|
1526
|
+
try:
|
|
1527
|
+
self._visual_transport = discover_visual_transport(
|
|
1528
|
+
self.research_path.parent, directory_fd=self._research_fd
|
|
1529
|
+
)
|
|
1530
|
+
except VisualRunError as exc:
|
|
1531
|
+
self._visual_error = exc
|
|
1532
|
+
if self._visual_transport is not None:
|
|
1533
|
+
self._refresh_visual_notebook()
|
|
1534
|
+
self.epoch = secrets.token_urlsafe(18)
|
|
1535
|
+
self._lock = threading.Lock()
|
|
1536
|
+
self._condition = threading.Condition(self._lock)
|
|
1537
|
+
self._health_status = "running"
|
|
1538
|
+
self._health_code: str | None = None
|
|
1539
|
+
if self._visual_error is not None:
|
|
1540
|
+
self._health_status = "stopped"
|
|
1541
|
+
self._health_code = HEALTH_WATCHER_FAILED
|
|
1542
|
+
self._health_build_present = False
|
|
1543
|
+
self._health_path_exists = False
|
|
1544
|
+
self._sidecar_manifest_observation: _FileSignature | None = None
|
|
1545
|
+
self._sidecar_candidate_identity: str | None = None
|
|
1546
|
+
self._admitted_table_key: tuple[_FileSignature, _FileSignature] | None = None
|
|
1547
|
+
self._admitted_candidate_digest: str | None = None
|
|
1548
|
+
self._rejected_table_signature: _FileSignature | None = None
|
|
1549
|
+
self._version = 0
|
|
1550
|
+
self._signature: tuple[tuple[str, _FileSignature], ...] = ()
|
|
1551
|
+
signature = self._scan_signature(names=("table",) if self._visual_present else _DOCUMENTS)
|
|
1552
|
+
snapshot = self._read_documents(signature, previous={}) if signature is not None else None
|
|
1553
|
+
if snapshot is not None:
|
|
1554
|
+
self._signature, documents = snapshot
|
|
1555
|
+
else:
|
|
1556
|
+
documents = {}
|
|
1557
|
+
with visual_authority_lock(self._research_fd, exclusive=False) as validate_authority:
|
|
1558
|
+
if not self._visual_present and self._visual_boundary_exists():
|
|
1559
|
+
# The shared research-root lock makes this absence observation and initial legacy
|
|
1560
|
+
# journal commit one linearizable interval with respect to store installation.
|
|
1561
|
+
# A visual writer either installs its boundary first and wins, or waits until the
|
|
1562
|
+
# already validated legacy snapshot is committed.
|
|
1563
|
+
self._visual_present = True
|
|
1564
|
+
self._refresh_visual_notebook()
|
|
1565
|
+
signature = self._scan_signature(names=("table",))
|
|
1566
|
+
snapshot = (
|
|
1567
|
+
self._read_documents(signature, previous={}) if signature is not None else None
|
|
1568
|
+
)
|
|
1569
|
+
if snapshot is not None:
|
|
1570
|
+
self._signature, documents = snapshot
|
|
1571
|
+
else:
|
|
1572
|
+
documents = {}
|
|
1573
|
+
if self._visual_transport is not None and self._visual_error is None:
|
|
1574
|
+
documents["research"] = notebook_bytes(
|
|
1575
|
+
self._visual_state, self._visual_transport.store.read_artifact
|
|
1576
|
+
)
|
|
1577
|
+
validate_authority()
|
|
1578
|
+
validate_authority.commit_after_validation(
|
|
1579
|
+
lambda: setattr(
|
|
1580
|
+
self,
|
|
1581
|
+
"_history",
|
|
1582
|
+
[_JournalEntry(0, documents, visual_cursor=self._visual_cursor)],
|
|
1583
|
+
)
|
|
1584
|
+
)
|
|
1585
|
+
try:
|
|
1586
|
+
self.feed_dir: Path | None = events.run_feed_dir(self.run_dir)
|
|
1587
|
+
except ValueError:
|
|
1588
|
+
# A legacy run name outside the feed's bounded path vocabulary remains viewable; it
|
|
1589
|
+
# simply cannot have a feed selected for it.
|
|
1590
|
+
self.feed_dir = None
|
|
1591
|
+
self._lifecycle_feed_path: Path | None = None
|
|
1592
|
+
self._lifecycle_run_started_at: float | None = None
|
|
1593
|
+
try:
|
|
1594
|
+
(
|
|
1595
|
+
self._lifecycle_feed_path,
|
|
1596
|
+
self._lifecycle_run_started_at,
|
|
1597
|
+
self._lifecycle,
|
|
1598
|
+
) = self._read_lifecycle(table_ready="table" in documents)
|
|
1599
|
+
except (LookupError, TypeError, ValueError):
|
|
1600
|
+
# Missing or malformed feed bytes cannot make an independently admitted table
|
|
1601
|
+
# unavailable, and a future run legitimately has no feed at startup.
|
|
1602
|
+
self._lifecycle = reduce_lifecycle([], table_ready="table" in documents)
|
|
1603
|
+
self._lifecycle["hosted"] = self._hosted_coordinate()
|
|
1604
|
+
self._lifecycle_serial = 0
|
|
1605
|
+
|
|
1606
|
+
def close(self) -> None:
|
|
1607
|
+
"""Release descriptors retained for the viewer lifetime."""
|
|
1608
|
+
|
|
1609
|
+
self._closed = True
|
|
1610
|
+
transport = getattr(self, "_visual_transport", None)
|
|
1611
|
+
if transport is not None:
|
|
1612
|
+
transport.store.close()
|
|
1613
|
+
self._visual_transport = None
|
|
1614
|
+
research_fd = getattr(self, "_research_fd", -1)
|
|
1615
|
+
if research_fd >= 0:
|
|
1616
|
+
os.close(research_fd)
|
|
1617
|
+
self._research_fd = -1
|
|
1618
|
+
research_path = getattr(self, "_research_path", None)
|
|
1619
|
+
if research_path is not None:
|
|
1620
|
+
research_path.close()
|
|
1621
|
+
selected_path = getattr(self, "_selected_path", None)
|
|
1622
|
+
if selected_path is not None:
|
|
1623
|
+
selected_path.close()
|
|
1624
|
+
|
|
1625
|
+
def __del__(self) -> None:
|
|
1626
|
+
try:
|
|
1627
|
+
if not getattr(self, "_closed", False):
|
|
1628
|
+
self.close()
|
|
1629
|
+
except (AttributeError, OSError):
|
|
1630
|
+
pass
|
|
1631
|
+
|
|
1632
|
+
def _member_stat(self, name: str) -> os.stat_result:
|
|
1633
|
+
if name == "research":
|
|
1634
|
+
self._research_path.validate()
|
|
1635
|
+
return os.stat("research.ipynb", dir_fd=self._research_fd, follow_symlinks=False)
|
|
1636
|
+
parent_fd, leaf = self._selected_path.open_parent(self.table_path)
|
|
1637
|
+
try:
|
|
1638
|
+
return os.stat(leaf, dir_fd=parent_fd, follow_symlinks=False)
|
|
1639
|
+
finally:
|
|
1640
|
+
os.close(parent_fd)
|
|
1641
|
+
|
|
1642
|
+
def _visual_boundary_exists(self) -> bool:
|
|
1643
|
+
"""Observe the fixed visual-run authority name without following it."""
|
|
1644
|
+
|
|
1645
|
+
try:
|
|
1646
|
+
os.stat("visual-run", dir_fd=self._research_fd, follow_symlinks=False)
|
|
1647
|
+
except FileNotFoundError:
|
|
1648
|
+
return False
|
|
1649
|
+
except OSError:
|
|
1650
|
+
# An unreadable or malformed authority boundary still fences legacy input.
|
|
1651
|
+
return True
|
|
1652
|
+
return True
|
|
1653
|
+
|
|
1654
|
+
def _open_member(self, name: str) -> tuple[int, int, str]:
|
|
1655
|
+
if name == "research":
|
|
1656
|
+
self._research_path.validate()
|
|
1657
|
+
parent_fd = os.dup(self._research_fd)
|
|
1658
|
+
leaf = "research.ipynb"
|
|
1659
|
+
else:
|
|
1660
|
+
parent_fd, leaf = self._selected_path.open_parent(self.table_path)
|
|
1661
|
+
try:
|
|
1662
|
+
descriptor = os.open(
|
|
1663
|
+
leaf,
|
|
1664
|
+
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
1665
|
+
dir_fd=parent_fd,
|
|
1666
|
+
)
|
|
1667
|
+
except BaseException:
|
|
1668
|
+
os.close(parent_fd)
|
|
1669
|
+
raise
|
|
1670
|
+
return descriptor, parent_fd, leaf
|
|
1671
|
+
|
|
1672
|
+
def _scan_signature(
|
|
1673
|
+
self, *, names: tuple[str, ...] = _DOCUMENTS
|
|
1674
|
+
) -> tuple[tuple[str, _FileSignature], ...] | None:
|
|
1675
|
+
"""Return no-follow identities, refusing unsafe or unverified notebook inputs."""
|
|
1676
|
+
|
|
1677
|
+
signature: list[tuple[str, _FileSignature]] = []
|
|
1678
|
+
total = 0
|
|
1679
|
+
for name in names:
|
|
1680
|
+
try:
|
|
1681
|
+
info = self._member_stat(name)
|
|
1682
|
+
except FileNotFoundError:
|
|
1683
|
+
continue
|
|
1684
|
+
except OSError:
|
|
1685
|
+
if name == "table":
|
|
1686
|
+
continue
|
|
1687
|
+
return None
|
|
1688
|
+
if not _is_regular_single_link(info) or info.st_size > _MAX_NOTEBOOK_BYTES:
|
|
1689
|
+
if name == "table":
|
|
1690
|
+
self._rejected_table_signature = _stat_signature(info)
|
|
1691
|
+
self._mark_sidecar_unavailable()
|
|
1692
|
+
continue
|
|
1693
|
+
return None
|
|
1694
|
+
stamp = _stat_signature(info)
|
|
1695
|
+
if name == "table":
|
|
1696
|
+
if self._rejected_table_signature not in (None, stamp):
|
|
1697
|
+
self._rejected_table_signature = None
|
|
1698
|
+
if self._rejected_table_signature == stamp:
|
|
1699
|
+
continue
|
|
1700
|
+
if not self._table_is_admissible(stamp):
|
|
1701
|
+
# A future target is intentionally allowed to be absent at startup. Its later
|
|
1702
|
+
# appearance is not evidence that a Build produced it: ordinary directories
|
|
1703
|
+
# must not satisfy the live viewer handoff merely by containing a renderable
|
|
1704
|
+
# file with the sidecar name.
|
|
1705
|
+
continue
|
|
1706
|
+
total += info.st_size
|
|
1707
|
+
if total > _MAX_SNAPSHOT_BYTES:
|
|
1708
|
+
return None
|
|
1709
|
+
signature.append((name, stamp))
|
|
1710
|
+
return tuple(signature)
|
|
1711
|
+
|
|
1712
|
+
def _manifest_observation(self) -> _FileSignature | None:
|
|
1713
|
+
"""Return the current descriptor-relative manifest identity, when one is safe to inspect."""
|
|
1714
|
+
|
|
1715
|
+
manifest_path = self.run_dir / "candidate" / "manifest.json"
|
|
1716
|
+
manifest_parent = -1
|
|
1717
|
+
try:
|
|
1718
|
+
manifest_parent, manifest_name = self._selected_path.open_parent(manifest_path)
|
|
1719
|
+
manifest = os.stat(manifest_name, dir_fd=manifest_parent, follow_symlinks=False)
|
|
1720
|
+
except OSError:
|
|
1721
|
+
return None
|
|
1722
|
+
finally:
|
|
1723
|
+
if manifest_parent >= 0:
|
|
1724
|
+
os.close(manifest_parent)
|
|
1725
|
+
if not _is_regular_single_link(manifest):
|
|
1726
|
+
return None
|
|
1727
|
+
return _stat_signature(manifest)
|
|
1728
|
+
|
|
1729
|
+
def _table_is_admissible(self, sidecar: _FileSignature) -> bool:
|
|
1730
|
+
"""Admit only the deterministic sidecar bytes derived from the exact verified Build."""
|
|
1731
|
+
|
|
1732
|
+
manifest = self._manifest_observation()
|
|
1733
|
+
if manifest is None:
|
|
1734
|
+
self._admitted_table_key = None
|
|
1735
|
+
self._admitted_candidate_digest = None
|
|
1736
|
+
return False
|
|
1737
|
+
key = (manifest, sidecar)
|
|
1738
|
+
if key == self._admitted_table_key:
|
|
1739
|
+
return True
|
|
1740
|
+
try:
|
|
1741
|
+
with open_verified_snapshot(self.run_dir) as verified_snapshot:
|
|
1742
|
+
candidate_digest = verified_snapshot.verified.candidate_digest
|
|
1743
|
+
notebook = _build_notebook_from_snapshot(
|
|
1744
|
+
self.run_dir / "candidate", verified_snapshot
|
|
1745
|
+
)
|
|
1746
|
+
expected = (json.dumps(notebook, indent=1, ensure_ascii=False) + "\n").encode(
|
|
1747
|
+
"utf-8"
|
|
1748
|
+
)
|
|
1749
|
+
actual = self._read_document("table", sidecar)
|
|
1750
|
+
current = self._member_stat("table")
|
|
1751
|
+
except Exception:
|
|
1752
|
+
self._admitted_table_key = None
|
|
1753
|
+
self._admitted_candidate_digest = None
|
|
1754
|
+
return False
|
|
1755
|
+
if (
|
|
1756
|
+
actual != expected
|
|
1757
|
+
or not _is_regular_single_link(current)
|
|
1758
|
+
or _stat_signature(current) != sidecar
|
|
1759
|
+
or self._manifest_observation() != manifest
|
|
1760
|
+
):
|
|
1761
|
+
self._admitted_table_key = None
|
|
1762
|
+
self._admitted_candidate_digest = None
|
|
1763
|
+
self._rejected_table_signature = sidecar
|
|
1764
|
+
self._mark_sidecar_unavailable()
|
|
1765
|
+
return False
|
|
1766
|
+
self._admitted_table_key = key
|
|
1767
|
+
self._admitted_candidate_digest = candidate_digest
|
|
1768
|
+
return True
|
|
1769
|
+
|
|
1770
|
+
def _read_documents(
|
|
1771
|
+
self,
|
|
1772
|
+
signature: tuple[tuple[str, _FileSignature], ...],
|
|
1773
|
+
*,
|
|
1774
|
+
previous: dict[str, bytes],
|
|
1775
|
+
) -> tuple[tuple[tuple[str, _FileSignature], ...], dict[str, bytes]] | None:
|
|
1776
|
+
"""Read one coherent valid state, sharing unchanged bytes with the previous entry."""
|
|
1777
|
+
|
|
1778
|
+
previous_stamps = dict(self._signature)
|
|
1779
|
+
accepted: list[tuple[str, _FileSignature]] = []
|
|
1780
|
+
documents: dict[str, bytes] = {}
|
|
1781
|
+
for name, expected in signature:
|
|
1782
|
+
if previous_stamps.get(name) == expected and name in previous:
|
|
1783
|
+
documents[name] = previous[name]
|
|
1784
|
+
accepted.append((name, expected))
|
|
1785
|
+
continue
|
|
1786
|
+
raw = self._read_document(name, expected)
|
|
1787
|
+
if raw is None:
|
|
1788
|
+
if name == "table":
|
|
1789
|
+
self._rejected_table_signature = expected
|
|
1790
|
+
self._mark_sidecar_unavailable()
|
|
1791
|
+
continue
|
|
1792
|
+
return None
|
|
1793
|
+
documents[name] = raw
|
|
1794
|
+
accepted.append((name, expected))
|
|
1795
|
+
accepted_signature = tuple(accepted)
|
|
1796
|
+
if (
|
|
1797
|
+
self._scan_signature(names=tuple(name for name, _stamp in signature))
|
|
1798
|
+
!= accepted_signature
|
|
1799
|
+
):
|
|
1800
|
+
return None
|
|
1801
|
+
return accepted_signature, documents
|
|
1802
|
+
|
|
1803
|
+
def _read_document(self, name: str, expected: _FileSignature) -> bytes | None:
|
|
1804
|
+
"""Read and render-check one descriptor-bound notebook without admitting a raced file."""
|
|
1805
|
+
|
|
1806
|
+
descriptor = -1
|
|
1807
|
+
parent_fd = -1
|
|
1808
|
+
try:
|
|
1809
|
+
descriptor, parent_fd, leaf = self._open_member(name)
|
|
1810
|
+
opened = os.fstat(descriptor)
|
|
1811
|
+
if not _is_regular_single_link(opened) or _stat_signature(opened) != expected:
|
|
1812
|
+
return None
|
|
1813
|
+
chunks: list[bytes] = []
|
|
1814
|
+
remaining = _MAX_NOTEBOOK_BYTES + 1
|
|
1815
|
+
while remaining:
|
|
1816
|
+
chunk = os.read(descriptor, min(_READ_CHUNK_BYTES, remaining))
|
|
1817
|
+
if not chunk:
|
|
1818
|
+
break
|
|
1819
|
+
chunks.append(chunk)
|
|
1820
|
+
remaining -= len(chunk)
|
|
1821
|
+
raw = b"".join(chunks)
|
|
1822
|
+
if len(raw) > _MAX_NOTEBOOK_BYTES or len(raw) != opened.st_size:
|
|
1823
|
+
return None
|
|
1824
|
+
try:
|
|
1825
|
+
decoded = raw.decode("utf-8")
|
|
1826
|
+
parsed = json.loads(decoded)
|
|
1827
|
+
except (UnicodeDecodeError, ValueError, RecursionError):
|
|
1828
|
+
return None
|
|
1829
|
+
if not isinstance(parsed, dict):
|
|
1830
|
+
return None
|
|
1831
|
+
# A notebook joins the journal only if the same stdlib renderer used by HTTP can
|
|
1832
|
+
# consume it. These are content-shaped failures; unexpected runtime/programming errors
|
|
1833
|
+
# deliberately cross the polling boundary instead of being hidden.
|
|
1834
|
+
try:
|
|
1835
|
+
render_notebook(parsed, mode="embed", filename=_DOCUMENTS[name])
|
|
1836
|
+
except (OverflowError, RecursionError):
|
|
1837
|
+
return None
|
|
1838
|
+
current = os.fstat(descriptor)
|
|
1839
|
+
named = os.stat(leaf, dir_fd=parent_fd, follow_symlinks=False)
|
|
1840
|
+
if _stat_signature(current) != expected or _stat_signature(named) != expected:
|
|
1841
|
+
return None
|
|
1842
|
+
return raw
|
|
1843
|
+
except OSError:
|
|
1844
|
+
return None
|
|
1845
|
+
finally:
|
|
1846
|
+
if descriptor >= 0:
|
|
1847
|
+
os.close(descriptor)
|
|
1848
|
+
if parent_fd >= 0:
|
|
1849
|
+
os.close(parent_fd)
|
|
1850
|
+
|
|
1851
|
+
def _journal_bytes(self) -> int:
|
|
1852
|
+
seen: set[int] = set()
|
|
1853
|
+
total = 0
|
|
1854
|
+
for entry in self._history:
|
|
1855
|
+
for raw in entry.documents.values():
|
|
1856
|
+
identity = id(raw)
|
|
1857
|
+
if identity not in seen:
|
|
1858
|
+
seen.add(identity)
|
|
1859
|
+
total += len(raw)
|
|
1860
|
+
return total
|
|
1861
|
+
|
|
1862
|
+
def _trim_history(self) -> None:
|
|
1863
|
+
while len(self._history) > 1 and (
|
|
1864
|
+
len(self._history) > _MAX_HISTORY or self._journal_bytes() > _MAX_JOURNAL_BYTES
|
|
1865
|
+
):
|
|
1866
|
+
self._history.pop(0)
|
|
1867
|
+
|
|
1868
|
+
def _append_history(
|
|
1869
|
+
self,
|
|
1870
|
+
documents: dict[str, bytes],
|
|
1871
|
+
*,
|
|
1872
|
+
mutation: dict[str, object] | None = None,
|
|
1873
|
+
visual_cursor: int | None = None,
|
|
1874
|
+
) -> None:
|
|
1875
|
+
"""Append one real version so every retained mutation has a consecutive predecessor."""
|
|
1876
|
+
|
|
1877
|
+
previous = self._history[-1]
|
|
1878
|
+
next_version = self._version + 1
|
|
1879
|
+
if mutation is None and visual_cursor is None:
|
|
1880
|
+
before = previous.documents.get("research")
|
|
1881
|
+
after = documents.get("research")
|
|
1882
|
+
if (
|
|
1883
|
+
before is not None
|
|
1884
|
+
and after is not None
|
|
1885
|
+
and _only_research_document_changed(previous.documents, documents)
|
|
1886
|
+
):
|
|
1887
|
+
mutation = _research_mutation(
|
|
1888
|
+
before,
|
|
1889
|
+
after,
|
|
1890
|
+
from_version=previous.version,
|
|
1891
|
+
version=next_version,
|
|
1892
|
+
)
|
|
1893
|
+
if mutation is not None:
|
|
1894
|
+
mutation = {**mutation, "from_version": previous.version, "version": next_version}
|
|
1895
|
+
entry = _JournalEntry(
|
|
1896
|
+
next_version,
|
|
1897
|
+
documents,
|
|
1898
|
+
mutation,
|
|
1899
|
+
previous.visual_cursor if visual_cursor is None else visual_cursor,
|
|
1900
|
+
)
|
|
1901
|
+
self._history.append(entry)
|
|
1902
|
+
self._version = next_version
|
|
1903
|
+
self._trim_history()
|
|
1904
|
+
|
|
1905
|
+
def poll(self, *, refresh_operational: bool = True) -> None:
|
|
1906
|
+
# Nothing about the table sidecar may gate this scan. Research streaming is the viewer's
|
|
1907
|
+
# first promise, and an unrecoverable sidecar is sticky per candidate identity: returning
|
|
1908
|
+
# early on one stopped research.ipynb from ever being read again, for the life of the
|
|
1909
|
+
# process, with the page still reporting a live server. The scan is already cheap in that
|
|
1910
|
+
# state -- a missing table member is skipped and a rejected signature short-circuits.
|
|
1911
|
+
# Read narration first so a current CLI sidecar write can exclude viewer recovery before
|
|
1912
|
+
# this poll touches the same absent pathname. Refresh again after document admission below,
|
|
1913
|
+
# because handoff requires both the ready record and the independently verified notebook.
|
|
1914
|
+
self._refresh_visual_notebook()
|
|
1915
|
+
if refresh_operational:
|
|
1916
|
+
self._refresh_lifecycle()
|
|
1917
|
+
self._recover_missing_table_sidecar()
|
|
1918
|
+
with self._lock:
|
|
1919
|
+
sidecar_unavailable = self._health_code == HEALTH_SIDECAR_UNAVAILABLE
|
|
1920
|
+
visual_authoritative = self._visual_present or self._visual_transport is not None
|
|
1921
|
+
signature = self._scan_signature(names=("table",) if visual_authoritative else _DOCUMENTS)
|
|
1922
|
+
if signature is not None:
|
|
1923
|
+
with self._condition:
|
|
1924
|
+
unchanged = signature == self._signature
|
|
1925
|
+
previous = self._history[-1].documents
|
|
1926
|
+
if unchanged and sidecar_unavailable and "table" in previous:
|
|
1927
|
+
self._health_status = "running"
|
|
1928
|
+
self._health_code = None
|
|
1929
|
+
self._condition.notify_all()
|
|
1930
|
+
if not unchanged:
|
|
1931
|
+
snapshot = self._read_documents(signature, previous=previous)
|
|
1932
|
+
if snapshot is not None:
|
|
1933
|
+
accepted_signature, documents = snapshot
|
|
1934
|
+
if visual_authoritative and "research" in previous:
|
|
1935
|
+
documents["research"] = previous["research"]
|
|
1936
|
+
if not visual_authoritative:
|
|
1937
|
+
with visual_authority_lock(
|
|
1938
|
+
self._research_fd, exclusive=False
|
|
1939
|
+
) as validate_authority:
|
|
1940
|
+
if self._visual_boundary_exists():
|
|
1941
|
+
# A first visual writer takes this same research-root lock
|
|
1942
|
+
# exclusively while installing the durable boundary. It therefore
|
|
1943
|
+
# cannot appear between this observation and the legacy commit.
|
|
1944
|
+
self._visual_present = True
|
|
1945
|
+
self._refresh_visual_notebook()
|
|
1946
|
+
return
|
|
1947
|
+
validate_authority()
|
|
1948
|
+
|
|
1949
|
+
def commit_legacy() -> None:
|
|
1950
|
+
with self._condition:
|
|
1951
|
+
if accepted_signature != self._signature:
|
|
1952
|
+
self._signature = accepted_signature
|
|
1953
|
+
self._append_history(documents)
|
|
1954
|
+
if sidecar_unavailable and "table" in documents:
|
|
1955
|
+
self._health_status = "running"
|
|
1956
|
+
self._health_code = None
|
|
1957
|
+
self._condition.notify_all()
|
|
1958
|
+
|
|
1959
|
+
validate_authority.commit_after_validation(commit_legacy)
|
|
1960
|
+
else:
|
|
1961
|
+
with self._condition:
|
|
1962
|
+
if accepted_signature != self._signature:
|
|
1963
|
+
self._signature = accepted_signature
|
|
1964
|
+
self._append_history(documents)
|
|
1965
|
+
if sidecar_unavailable and "table" in documents:
|
|
1966
|
+
self._health_status = "running"
|
|
1967
|
+
self._health_code = None
|
|
1968
|
+
self._condition.notify_all()
|
|
1969
|
+
if refresh_operational:
|
|
1970
|
+
self._refresh_lifecycle()
|
|
1971
|
+
|
|
1972
|
+
def _hosted_coordinate(self) -> dict[str, object] | None:
|
|
1973
|
+
"""Name the hosted run these local bytes came from, when one wrote a record here.
|
|
1974
|
+
|
|
1975
|
+
The viewer observes content fingerprints and local files; nothing in it otherwise names a
|
|
1976
|
+
run, a Dataset, a Table, or a version, so a Build that arrived from hosted execution could
|
|
1977
|
+
be shown but not attributed. This reads the record the handoff wrote beside the Build and
|
|
1978
|
+
reports the five identifiers on it.
|
|
1979
|
+
|
|
1980
|
+
It decides nothing. Admission is unchanged and does not consult this: a dataset is shown
|
|
1981
|
+
because this viewer verified the Build and re-derived the notebook itself, and it would be
|
|
1982
|
+
shown identically if the record were absent, and refused identically if the record claimed
|
|
1983
|
+
anything at all.
|
|
1984
|
+
"""
|
|
1985
|
+
|
|
1986
|
+
receipt = read_handoff_receipt(self.run_dir)
|
|
1987
|
+
if receipt is None:
|
|
1988
|
+
return None
|
|
1989
|
+
named = {
|
|
1990
|
+
key: receipt.get(key)
|
|
1991
|
+
for key in (
|
|
1992
|
+
"dataset_id",
|
|
1993
|
+
"table_id",
|
|
1994
|
+
"table_version_id",
|
|
1995
|
+
"version_number",
|
|
1996
|
+
"hosted_run_id",
|
|
1997
|
+
)
|
|
1998
|
+
}
|
|
1999
|
+
return named if all(value is not None for value in named.values()) else None
|
|
2000
|
+
|
|
2001
|
+
def _read_lifecycle(
|
|
2002
|
+
self, *, table_ready: bool
|
|
2003
|
+
) -> tuple[Path | None, float | None, dict[str, object]]:
|
|
2004
|
+
"""Read the selected live attempt and reduce it without receipt replay or inference."""
|
|
2005
|
+
|
|
2006
|
+
if self._visual_transport is not None:
|
|
2007
|
+
snapshot = self._visual_transport.snapshot()
|
|
2008
|
+
reduced = snapshot.state
|
|
2009
|
+
lifecycle = _visual_lifecycle(
|
|
2010
|
+
reduced,
|
|
2011
|
+
table_ready=table_ready,
|
|
2012
|
+
admitted_candidate_digest=self._admitted_candidate_digest,
|
|
2013
|
+
)
|
|
2014
|
+
lifecycle["hosted"] = self._hosted_coordinate()
|
|
2015
|
+
return self._visual_transport.store.log_path, None, lifecycle
|
|
2016
|
+
if self._visual_present:
|
|
2017
|
+
raise ValueError("visual run log is invalid")
|
|
2018
|
+
if self.feed_dir is None:
|
|
2019
|
+
return None, None, reduce_lifecycle([], table_ready=table_ready)
|
|
2020
|
+
path, _size, records = events.select_narrated_feed(
|
|
2021
|
+
self.feed_dir,
|
|
2022
|
+
sealed_candidate_digest=self._admitted_candidate_digest,
|
|
2023
|
+
)
|
|
2024
|
+
if path is None:
|
|
2025
|
+
raise LookupError("no narrated feed is currently selected")
|
|
2026
|
+
if not records:
|
|
2027
|
+
if not events.feed_is_readable(path):
|
|
2028
|
+
raise ValueError("selected feed is not readable")
|
|
2029
|
+
raise LookupError("selected feed has not narrated an event")
|
|
2030
|
+
start = actual_run_started_record(records)
|
|
2031
|
+
run_started_at: float | None = None
|
|
2032
|
+
if start is not None:
|
|
2033
|
+
facts = start.get("facts")
|
|
2034
|
+
moment = start.get("at")
|
|
2035
|
+
try:
|
|
2036
|
+
events.validate_facts("run_started", facts) # type: ignore[arg-type]
|
|
2037
|
+
except ValueError:
|
|
2038
|
+
pass
|
|
2039
|
+
else:
|
|
2040
|
+
if isinstance(moment, (int, float)) and not isinstance(moment, bool):
|
|
2041
|
+
run_started_at = float(moment)
|
|
2042
|
+
admitted_digest = self._admitted_candidate_digest
|
|
2043
|
+
selected_digest = coherent_build_seal_candidate(records)
|
|
2044
|
+
attempt_table_ready = (
|
|
2045
|
+
table_ready and admitted_digest is not None and selected_digest == admitted_digest
|
|
2046
|
+
)
|
|
2047
|
+
previous_path = getattr(self, "_lifecycle_feed_path", None)
|
|
2048
|
+
previous_lifecycle = getattr(self, "_lifecycle", {})
|
|
2049
|
+
allow_legacy_handoff = (
|
|
2050
|
+
bool(previous_lifecycle.get("legacy_handoff_allowed", True))
|
|
2051
|
+
if path == previous_path
|
|
2052
|
+
else previous_path is None
|
|
2053
|
+
)
|
|
2054
|
+
lifecycle = reduce_lifecycle(
|
|
2055
|
+
records,
|
|
2056
|
+
table_ready=attempt_table_ready,
|
|
2057
|
+
allow_legacy_handoff=allow_legacy_handoff,
|
|
2058
|
+
)
|
|
2059
|
+
lifecycle["hosted"] = self._hosted_coordinate()
|
|
2060
|
+
return path, run_started_at, lifecycle
|
|
2061
|
+
|
|
2062
|
+
@staticmethod
|
|
2063
|
+
def _coherent_lifecycle_successor(
|
|
2064
|
+
previous: Mapping[str, object],
|
|
2065
|
+
previous_path: Path | None,
|
|
2066
|
+
lifecycle: Mapping[str, object],
|
|
2067
|
+
path: Path | None,
|
|
2068
|
+
run_started_at: float | None,
|
|
2069
|
+
) -> bool:
|
|
2070
|
+
"""Require a distinct, later attempt before replacing a terminal state.
|
|
2071
|
+
|
|
2072
|
+
A same-path feed may only grow; replacing it with a shorter valid prefix is not evidence of
|
|
2073
|
+
a retry. The only accepted same-attempt change preserves its start and terminal timestamps,
|
|
2074
|
+
as when independent table admission changes handoff presentation. A genuine retry has its
|
|
2075
|
+
own attempt file and a ``run_started`` timestamp strictly after the prior terminal timestamp.
|
|
2076
|
+
"""
|
|
2077
|
+
|
|
2078
|
+
if previous.get("status") not in {"completed", "failed"}:
|
|
2079
|
+
return True
|
|
2080
|
+
if path is None or previous_path is None:
|
|
2081
|
+
return False
|
|
2082
|
+
if path == previous_path:
|
|
2083
|
+
structurally_same_attempt = (
|
|
2084
|
+
previous.get("status") == "completed"
|
|
2085
|
+
and lifecycle.get("status") == "completed"
|
|
2086
|
+
and previous.get("sealed") is True
|
|
2087
|
+
and lifecycle.get("sealed") is True
|
|
2088
|
+
and lifecycle.get("started_at") == previous.get("started_at")
|
|
2089
|
+
and lifecycle.get("ended_at") == previous.get("ended_at")
|
|
2090
|
+
)
|
|
2091
|
+
return structurally_same_attempt and _sidecar_lifecycle_grows_monotonically(
|
|
2092
|
+
previous, lifecycle
|
|
2093
|
+
)
|
|
2094
|
+
previous_end = previous.get("ended_at")
|
|
2095
|
+
return (
|
|
2096
|
+
isinstance(previous_end, (int, float))
|
|
2097
|
+
and not isinstance(previous_end, bool)
|
|
2098
|
+
and run_started_at is not None
|
|
2099
|
+
and run_started_at > float(previous_end)
|
|
2100
|
+
)
|
|
2101
|
+
|
|
2102
|
+
def _refresh_lifecycle(self) -> None:
|
|
2103
|
+
"""Publish a lifecycle notice only when real feed or handoff state changed."""
|
|
2104
|
+
|
|
2105
|
+
with self._lock:
|
|
2106
|
+
table_ready = "table" in self._history[-1].documents
|
|
2107
|
+
try:
|
|
2108
|
+
path, run_started_at, lifecycle = self._read_lifecycle(table_ready=table_ready)
|
|
2109
|
+
except LookupError:
|
|
2110
|
+
# A legacy/direct library build may have no narrated records at all. Only an idle
|
|
2111
|
+
# lifecycle may fall back to independently admitted document state; once any attempt
|
|
2112
|
+
# has narrated progress or a terminal, a missing/header-only replacement retains it.
|
|
2113
|
+
with self._lock:
|
|
2114
|
+
if self._lifecycle.get("status") != "idle":
|
|
2115
|
+
return
|
|
2116
|
+
path = None
|
|
2117
|
+
run_started_at = None
|
|
2118
|
+
lifecycle = reduce_lifecycle([], table_ready=table_ready)
|
|
2119
|
+
lifecycle["hosted"] = self._hosted_coordinate()
|
|
2120
|
+
except Exception:
|
|
2121
|
+
# The feed is untrusted and concurrently written. Keep the last understood lifecycle;
|
|
2122
|
+
# the next poll retries, exactly as notebook rendering keeps its last valid snapshot.
|
|
2123
|
+
return
|
|
2124
|
+
with self._condition:
|
|
2125
|
+
if lifecycle == self._lifecycle:
|
|
2126
|
+
return
|
|
2127
|
+
if not self._coherent_lifecycle_successor(
|
|
2128
|
+
self._lifecycle,
|
|
2129
|
+
self._lifecycle_feed_path,
|
|
2130
|
+
lifecycle,
|
|
2131
|
+
path,
|
|
2132
|
+
run_started_at,
|
|
2133
|
+
):
|
|
2134
|
+
return
|
|
2135
|
+
self._lifecycle = lifecycle
|
|
2136
|
+
self._lifecycle_feed_path = path
|
|
2137
|
+
self._lifecycle_run_started_at = run_started_at
|
|
2138
|
+
self._lifecycle_serial += 1
|
|
2139
|
+
self._condition.notify_all()
|
|
2140
|
+
|
|
2141
|
+
def _refresh_visual_notebook(self) -> None:
|
|
2142
|
+
"""Project the visual log when present; never combine it with legacy notebook events."""
|
|
2143
|
+
|
|
2144
|
+
self._visual_refresh_lock.acquire()
|
|
2145
|
+
try:
|
|
2146
|
+
self._refresh_visual_notebook_locked()
|
|
2147
|
+
finally:
|
|
2148
|
+
self._visual_refresh_lock.release()
|
|
2149
|
+
|
|
2150
|
+
def _refresh_visual_notebook_locked(self) -> None:
|
|
2151
|
+
transport = getattr(self, "_visual_transport", None)
|
|
2152
|
+
if transport is None:
|
|
2153
|
+
self._visual_present = self._visual_present or self._visual_boundary_exists()
|
|
2154
|
+
try:
|
|
2155
|
+
transport = discover_visual_transport(
|
|
2156
|
+
self.research_path.parent, directory_fd=self._research_fd
|
|
2157
|
+
)
|
|
2158
|
+
except VisualRunError as exc:
|
|
2159
|
+
self._visual_error = exc
|
|
2160
|
+
self._visual_present = True
|
|
2161
|
+
condition = getattr(self, "_condition", None)
|
|
2162
|
+
if condition is not None:
|
|
2163
|
+
with condition:
|
|
2164
|
+
self._health_status = "stopped"
|
|
2165
|
+
self._health_code = HEALTH_WATCHER_FAILED
|
|
2166
|
+
condition.notify_all()
|
|
2167
|
+
return
|
|
2168
|
+
if transport is None:
|
|
2169
|
+
return
|
|
2170
|
+
self._visual_transport = transport
|
|
2171
|
+
self._visual_present = True
|
|
2172
|
+
try:
|
|
2173
|
+
snapshot = transport.snapshot()
|
|
2174
|
+
if not hasattr(self, "_history"):
|
|
2175
|
+
self._visual_state = snapshot.state
|
|
2176
|
+
if snapshot.cursor == self._visual_cursor:
|
|
2177
|
+
return
|
|
2178
|
+
materialize_notebook(
|
|
2179
|
+
self.research_path,
|
|
2180
|
+
snapshot.state,
|
|
2181
|
+
transport.store.read_artifact,
|
|
2182
|
+
directory_fd=self._research_fd,
|
|
2183
|
+
)
|
|
2184
|
+
self._visual_cursor = snapshot.cursor
|
|
2185
|
+
return
|
|
2186
|
+
if snapshot.cursor == self._visual_cursor:
|
|
2187
|
+
return
|
|
2188
|
+
state = self._visual_state
|
|
2189
|
+
while self._visual_cursor < snapshot.cursor:
|
|
2190
|
+
page = transport.events_after(self._visual_cursor, limit=256)
|
|
2191
|
+
if not page.events:
|
|
2192
|
+
raise VisualRunError(
|
|
2193
|
+
"VISUAL_RUN_SEQUENCE_GAP", "visual event cursor did not advance"
|
|
2194
|
+
)
|
|
2195
|
+
for event in page.events:
|
|
2196
|
+
before = state
|
|
2197
|
+
state = apply_event(state, event)
|
|
2198
|
+
raw = notebook_bytes(state, transport.store.read_artifact)
|
|
2199
|
+
with self._condition:
|
|
2200
|
+
documents = dict(self._history[-1].documents)
|
|
2201
|
+
documents["research"] = raw
|
|
2202
|
+
self._append_history(
|
|
2203
|
+
documents,
|
|
2204
|
+
mutation=_visual_event_mutation(event, before),
|
|
2205
|
+
visual_cursor=event.sequence,
|
|
2206
|
+
)
|
|
2207
|
+
self._condition.notify_all()
|
|
2208
|
+
self._visual_cursor = event.sequence
|
|
2209
|
+
self._visual_state = state
|
|
2210
|
+
materialize_notebook(
|
|
2211
|
+
self.research_path,
|
|
2212
|
+
state,
|
|
2213
|
+
transport.store.read_artifact,
|
|
2214
|
+
directory_fd=self._research_fd,
|
|
2215
|
+
)
|
|
2216
|
+
observed = self._scan_signature(names=("table",))
|
|
2217
|
+
held = set(self._history[-1].documents)
|
|
2218
|
+
if observed is not None and all(name in held for name, _stamp in observed):
|
|
2219
|
+
# Only stamps for documents this journal actually holds may be adopted here.
|
|
2220
|
+
# `self._signature` is the identity of what the last entry contains, and adopting
|
|
2221
|
+
# the stamp of a sidecar that has just appeared but has not been read would say
|
|
2222
|
+
# the entry holds it. `poll` compares the two to decide whether to read, so that
|
|
2223
|
+
# claim is self-fulfilling: the sidecar looks unchanged from then on and is never
|
|
2224
|
+
# admitted, for the life of the process. It is reachable whenever a Build writes
|
|
2225
|
+
# the sidecar and closes its narration between two polls, which is the ordinary
|
|
2226
|
+
# case when the Build is not the local one -- and the reason it stayed invisible
|
|
2227
|
+
# is that a local Build's sidecar usually lands a poll earlier than its last
|
|
2228
|
+
# event does.
|
|
2229
|
+
self._signature = observed
|
|
2230
|
+
except (OSError, ValueError, VisualRunError) as exc:
|
|
2231
|
+
if isinstance(exc, VisualRunError):
|
|
2232
|
+
self._visual_error = exc
|
|
2233
|
+
elif isinstance(exc, ValueError):
|
|
2234
|
+
self._visual_error = VisualRunError("VISUAL_RUN_PROJECTION_LIMIT", str(exc))
|
|
2235
|
+
condition = getattr(self, "_condition", threading.Condition())
|
|
2236
|
+
with condition:
|
|
2237
|
+
if hasattr(self, "_health_status"):
|
|
2238
|
+
self._health_status = "stopped"
|
|
2239
|
+
self._health_code = HEALTH_WATCHER_FAILED
|
|
2240
|
+
condition.notify_all()
|
|
2241
|
+
|
|
2242
|
+
def _recover_missing_table_sidecar(self) -> None:
|
|
2243
|
+
"""Make at most one verified render attempt for each observed candidate identity.
|
|
2244
|
+
|
|
2245
|
+
Recovery is for an absent sidecar and nothing else. A file that is there is never
|
|
2246
|
+
replaced, whatever it holds: the shipped skill tells the agent to append executed chart
|
|
2247
|
+
cells to ``table.ipynb``, and those cells make the file differ from the deterministic
|
|
2248
|
+
render, which is exactly the state a regenerating watcher would overwrite -- silently, and
|
|
2249
|
+
within one poll. A present sidecar the viewer cannot admit is reported instead, and
|
|
2250
|
+
``mr-data notebook RUN_DIR --json`` is the explicit, human-initiated way to rebuild it.
|
|
2251
|
+
|
|
2252
|
+
Recovery stands down structurally while a build is active. ``build_candidate`` installs
|
|
2253
|
+
:data:`RUN_BUILD_ACTIVITY_LOCK` with the atomic run directory and holds an exclusive kernel
|
|
2254
|
+
lock on it until installed verification has closed its namespace monitor. A viewer acquires
|
|
2255
|
+
and retains an exclusive lock on that exact inode plus the descriptor-pinned run through its
|
|
2256
|
+
own verification and create-only write. The run-directory path therefore has no
|
|
2257
|
+
elapsed-time guess or check/use gap: an active Build keeps the writer out, while process
|
|
2258
|
+
death releases the lock and permits recovery. Workspace mode retains its execution-lock
|
|
2259
|
+
probe as an additional guard around the wider offline operation.
|
|
2260
|
+
"""
|
|
2261
|
+
|
|
2262
|
+
with self._lock:
|
|
2263
|
+
sidecar_line = next(
|
|
2264
|
+
(
|
|
2265
|
+
line
|
|
2266
|
+
for line in self._lifecycle.get("lines", [])
|
|
2267
|
+
if isinstance(line, Mapping) and line.get("key") == "table-notebook"
|
|
2268
|
+
),
|
|
2269
|
+
None,
|
|
2270
|
+
)
|
|
2271
|
+
if isinstance(sidecar_line, Mapping) and sidecar_line.get("status") in {
|
|
2272
|
+
"running",
|
|
2273
|
+
"failed",
|
|
2274
|
+
}:
|
|
2275
|
+
# The CLI owns the write while it is running. A narrated failure remains best-effort
|
|
2276
|
+
# and visible; silently retrying it here would contradict the real terminal outcome.
|
|
2277
|
+
return
|
|
2278
|
+
try:
|
|
2279
|
+
self._member_stat("table")
|
|
2280
|
+
except FileNotFoundError:
|
|
2281
|
+
pass
|
|
2282
|
+
except OSError:
|
|
2283
|
+
return
|
|
2284
|
+
else:
|
|
2285
|
+
return
|
|
2286
|
+
if self._workspace_build_is_active():
|
|
2287
|
+
return
|
|
2288
|
+
recovery_lease = self._acquire_run_recovery_lease()
|
|
2289
|
+
if recovery_lease is None:
|
|
2290
|
+
return
|
|
2291
|
+
try:
|
|
2292
|
+
recovery_lease.validate()
|
|
2293
|
+
try:
|
|
2294
|
+
os.stat(_SIDECAR_NAME, dir_fd=recovery_lease.run_fd, follow_symlinks=False)
|
|
2295
|
+
except FileNotFoundError:
|
|
2296
|
+
pass
|
|
2297
|
+
except OSError:
|
|
2298
|
+
return
|
|
2299
|
+
else:
|
|
2300
|
+
return
|
|
2301
|
+
observation = recovery_lease.manifest_observation()
|
|
2302
|
+
if observation is None or observation == self._sidecar_manifest_observation:
|
|
2303
|
+
return
|
|
2304
|
+
self._sidecar_manifest_observation = observation
|
|
2305
|
+
try:
|
|
2306
|
+
verified = verify_candidate(
|
|
2307
|
+
self.run_dir,
|
|
2308
|
+
_run_fd=recovery_lease.run_fd,
|
|
2309
|
+
_ancestor_validator=recovery_lease.validate,
|
|
2310
|
+
)
|
|
2311
|
+
except Exception:
|
|
2312
|
+
self._mark_sidecar_unavailable()
|
|
2313
|
+
return
|
|
2314
|
+
identity = verified.candidate_digest
|
|
2315
|
+
if identity == self._sidecar_candidate_identity:
|
|
2316
|
+
return
|
|
2317
|
+
# Record before rendering: failures are stable for this verified candidate and cannot
|
|
2318
|
+
# turn the 250ms watcher into an unbounded retry loop.
|
|
2319
|
+
self._sidecar_candidate_identity = identity
|
|
2320
|
+
try:
|
|
2321
|
+
# The shared activity lock and exact run descriptor remain owned through the
|
|
2322
|
+
# create-only placement. A sidecar that arrived inside the render still wins.
|
|
2323
|
+
render_table_notebook(
|
|
2324
|
+
self.run_dir,
|
|
2325
|
+
replace_existing=False,
|
|
2326
|
+
_run_fd=recovery_lease.run_fd,
|
|
2327
|
+
_ancestor_validator=recovery_lease.validate,
|
|
2328
|
+
)
|
|
2329
|
+
recovery_lease.validate()
|
|
2330
|
+
except FileExistsError:
|
|
2331
|
+
self._sidecar_manifest_observation = None
|
|
2332
|
+
self._sidecar_candidate_identity = None
|
|
2333
|
+
return
|
|
2334
|
+
except Exception:
|
|
2335
|
+
self._mark_sidecar_unavailable()
|
|
2336
|
+
return
|
|
2337
|
+
finally:
|
|
2338
|
+
recovery_lease.close()
|
|
2339
|
+
# A successful recovery is not rate-limited: if the sidecar is later deleted while this
|
|
2340
|
+
# same candidate remains current, the next poll regenerates it. That release is only safe
|
|
2341
|
+
# once the file just written is one the viewer will admit. A render this watcher would
|
|
2342
|
+
# reject again -- one over the size limit, say -- would otherwise re-verify the whole
|
|
2343
|
+
# candidate, re-render it and replace the file four times a second, forever.
|
|
2344
|
+
if not self._recovered_sidecar_is_admissible():
|
|
2345
|
+
self._mark_sidecar_unavailable()
|
|
2346
|
+
return
|
|
2347
|
+
self._sidecar_manifest_observation = None
|
|
2348
|
+
self._sidecar_candidate_identity = None
|
|
2349
|
+
self._rejected_table_signature = None
|
|
2350
|
+
with self._condition:
|
|
2351
|
+
if self._health_status == "unavailable":
|
|
2352
|
+
self._health_status = "running"
|
|
2353
|
+
self._health_code = None
|
|
2354
|
+
self._condition.notify_all()
|
|
2355
|
+
|
|
2356
|
+
def _workspace_build_is_active(self) -> bool:
|
|
2357
|
+
"""Whether a build currently holds this workspace's execution lock.
|
|
2358
|
+
|
|
2359
|
+
``offline._execute_workspace`` opens the workspace, taking that lock exclusively, and calls
|
|
2360
|
+
``build_candidate`` inside it -- so the lock is held across the post-rename verification a
|
|
2361
|
+
recovery write would break. Probing it shared and non-blocking answers the question without
|
|
2362
|
+
excluding anything for longer than the probe. The run-local recovery lease below provides
|
|
2363
|
+
the write-owning exclusion interval in both modes.
|
|
2364
|
+
"""
|
|
2365
|
+
|
|
2366
|
+
if fcntl is None:
|
|
2367
|
+
raise ViewerError(
|
|
2368
|
+
"VIEWER_PLATFORM_UNSUPPORTED",
|
|
2369
|
+
"secure viewer locking is unavailable on this platform",
|
|
2370
|
+
)
|
|
2371
|
+
if self.target.mode != "workspace":
|
|
2372
|
+
return False
|
|
2373
|
+
control_fd = -1
|
|
2374
|
+
lock_fd = -1
|
|
2375
|
+
try:
|
|
2376
|
+
control_fd = self._selected_path.open_directory(
|
|
2377
|
+
self.target.selected_path / CONTROL_PATH
|
|
2378
|
+
)
|
|
2379
|
+
lock_fd = os.open(
|
|
2380
|
+
WORKSPACE_LOCK_NAME,
|
|
2381
|
+
os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
|
|
2382
|
+
dir_fd=control_fd,
|
|
2383
|
+
)
|
|
2384
|
+
try:
|
|
2385
|
+
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2386
|
+
except OSError as exc:
|
|
2387
|
+
return exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES)
|
|
2388
|
+
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
2389
|
+
return False
|
|
2390
|
+
except OSError:
|
|
2391
|
+
# No control tree, or one this viewer may not read through: there is no build to defer
|
|
2392
|
+
# to, and the quiet-poll wait below still applies.
|
|
2393
|
+
return False
|
|
2394
|
+
finally:
|
|
2395
|
+
if lock_fd >= 0:
|
|
2396
|
+
os.close(lock_fd)
|
|
2397
|
+
if control_fd >= 0:
|
|
2398
|
+
os.close(control_fd)
|
|
2399
|
+
|
|
2400
|
+
def _acquire_run_recovery_lease(self) -> _RunRecoveryLease | None:
|
|
2401
|
+
"""Acquire the exact run and activity lock for the complete recovery transaction.
|
|
2402
|
+
|
|
2403
|
+
Absence, contention, malformed state, and every identity race fail closed. In particular,
|
|
2404
|
+
an absent future run cannot become a published Build after this method returns a negative
|
|
2405
|
+
answer and still be touched by the same poll: no lease means no subsequent recovery work.
|
|
2406
|
+
Legacy Builds without the structural activity file require explicit ``mr-data notebook``.
|
|
2407
|
+
"""
|
|
2408
|
+
|
|
2409
|
+
if fcntl is None:
|
|
2410
|
+
raise ViewerError(
|
|
2411
|
+
"VIEWER_PLATFORM_UNSUPPORTED",
|
|
2412
|
+
"secure viewer locking is unavailable on this platform",
|
|
2413
|
+
)
|
|
2414
|
+
run_handle: _CandidateRunHandle | None = None
|
|
2415
|
+
lock_fd = -1
|
|
2416
|
+
acquired = False
|
|
2417
|
+
transferred = False
|
|
2418
|
+
try:
|
|
2419
|
+
run_handle = _open_candidate_run_handle(self.run_dir)
|
|
2420
|
+
_validate_candidate_run_handle(run_handle)
|
|
2421
|
+
try:
|
|
2422
|
+
named_before = os.stat(
|
|
2423
|
+
RUN_BUILD_ACTIVITY_LOCK,
|
|
2424
|
+
dir_fd=run_handle.run_fd,
|
|
2425
|
+
follow_symlinks=False,
|
|
2426
|
+
)
|
|
2427
|
+
except FileNotFoundError:
|
|
2428
|
+
return None
|
|
2429
|
+
if not _is_regular_single_link(named_before):
|
|
2430
|
+
return None
|
|
2431
|
+
lock_fd = os.open(
|
|
2432
|
+
RUN_BUILD_ACTIVITY_LOCK,
|
|
2433
|
+
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0),
|
|
2434
|
+
dir_fd=run_handle.run_fd,
|
|
2435
|
+
)
|
|
2436
|
+
opened = os.fstat(lock_fd)
|
|
2437
|
+
if not _is_regular_single_link(opened) or _entry_identity(opened) != _entry_identity(
|
|
2438
|
+
named_before
|
|
2439
|
+
):
|
|
2440
|
+
return None
|
|
2441
|
+
try:
|
|
2442
|
+
fcntl.flock(lock_fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
2443
|
+
acquired = True
|
|
2444
|
+
except OSError as exc:
|
|
2445
|
+
if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES):
|
|
2446
|
+
return None
|
|
2447
|
+
return None
|
|
2448
|
+
named_after = os.stat(
|
|
2449
|
+
RUN_BUILD_ACTIVITY_LOCK,
|
|
2450
|
+
dir_fd=run_handle.run_fd,
|
|
2451
|
+
follow_symlinks=False,
|
|
2452
|
+
)
|
|
2453
|
+
lock_identity = _entry_identity(opened)
|
|
2454
|
+
if (
|
|
2455
|
+
not _is_regular_single_link(named_after)
|
|
2456
|
+
or _entry_identity(named_after) != lock_identity
|
|
2457
|
+
):
|
|
2458
|
+
return None
|
|
2459
|
+
lease = _RunRecoveryLease(run_handle, lock_fd, lock_identity)
|
|
2460
|
+
lease.validate()
|
|
2461
|
+
transferred = True
|
|
2462
|
+
return lease
|
|
2463
|
+
except Exception:
|
|
2464
|
+
return None
|
|
2465
|
+
finally:
|
|
2466
|
+
if not transferred:
|
|
2467
|
+
if acquired:
|
|
2468
|
+
try:
|
|
2469
|
+
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
2470
|
+
except OSError:
|
|
2471
|
+
pass
|
|
2472
|
+
if lock_fd >= 0:
|
|
2473
|
+
try:
|
|
2474
|
+
os.close(lock_fd)
|
|
2475
|
+
except OSError:
|
|
2476
|
+
pass
|
|
2477
|
+
if run_handle is not None:
|
|
2478
|
+
try:
|
|
2479
|
+
run_handle.close()
|
|
2480
|
+
except Exception:
|
|
2481
|
+
pass
|
|
2482
|
+
|
|
2483
|
+
def _recovered_sidecar_is_admissible(self) -> bool:
|
|
2484
|
+
"""Read back the sidecar this recovery wrote under the live admission rules."""
|
|
2485
|
+
|
|
2486
|
+
try:
|
|
2487
|
+
sidecar = self._member_stat("table")
|
|
2488
|
+
except OSError:
|
|
2489
|
+
return False
|
|
2490
|
+
if not _is_regular_single_link(sidecar) or sidecar.st_size > _MAX_NOTEBOOK_BYTES:
|
|
2491
|
+
return False
|
|
2492
|
+
return self._table_is_admissible(_stat_signature(sidecar))
|
|
2493
|
+
|
|
2494
|
+
def _run_directory_exists(self) -> bool:
|
|
2495
|
+
"""Ask whether the run directory itself is there, without following a link to it."""
|
|
2496
|
+
|
|
2497
|
+
descriptor = -1
|
|
2498
|
+
try:
|
|
2499
|
+
descriptor = self._selected_path.open_directory(self.run_dir)
|
|
2500
|
+
except OSError:
|
|
2501
|
+
return False
|
|
2502
|
+
finally:
|
|
2503
|
+
if descriptor >= 0:
|
|
2504
|
+
os.close(descriptor)
|
|
2505
|
+
return True
|
|
2506
|
+
|
|
2507
|
+
def _mark_sidecar_unavailable(self) -> None:
|
|
2508
|
+
build_present = self._manifest_observation() is not None
|
|
2509
|
+
path_exists = build_present or self._run_directory_exists()
|
|
2510
|
+
with self._condition:
|
|
2511
|
+
self._health_build_present = build_present
|
|
2512
|
+
self._health_path_exists = path_exists
|
|
2513
|
+
self._health_status = "unavailable"
|
|
2514
|
+
self._health_code = HEALTH_SIDECAR_UNAVAILABLE
|
|
2515
|
+
self._condition.notify_all()
|
|
2516
|
+
|
|
2517
|
+
@property
|
|
2518
|
+
def version(self) -> int:
|
|
2519
|
+
with self._lock:
|
|
2520
|
+
return self._version
|
|
2521
|
+
|
|
2522
|
+
def visual_cursor_is_admitted(self, cursor: int) -> bool:
|
|
2523
|
+
"""Return whether a cursor can safely be announced to a snapshot client."""
|
|
2524
|
+
|
|
2525
|
+
with self._lock:
|
|
2526
|
+
return cursor <= self._visual_cursor
|
|
2527
|
+
|
|
2528
|
+
def versions(self) -> list[int]:
|
|
2529
|
+
with self._lock:
|
|
2530
|
+
return [entry.version for entry in self._history]
|
|
2531
|
+
|
|
2532
|
+
def snapshot(self, version: int | None = None) -> _JournalEntry:
|
|
2533
|
+
with self._lock:
|
|
2534
|
+
return self._snapshot_unlocked(version)
|
|
2535
|
+
|
|
2536
|
+
def _snapshot_unlocked(self, version: int | None = None) -> _JournalEntry:
|
|
2537
|
+
if version is None:
|
|
2538
|
+
return self._history[-1]
|
|
2539
|
+
if version < 0 or version > self._version:
|
|
2540
|
+
raise KeyError(version)
|
|
2541
|
+
for entry in self._history:
|
|
2542
|
+
if entry.version == version:
|
|
2543
|
+
return entry
|
|
2544
|
+
raise LookupError(version)
|
|
2545
|
+
|
|
2546
|
+
def _snapshot_cursor_unlocked(self, cursor: int) -> _JournalEntry:
|
|
2547
|
+
if cursor < 0 or cursor > self._visual_cursor:
|
|
2548
|
+
raise KeyError(cursor)
|
|
2549
|
+
for entry in self._history:
|
|
2550
|
+
if entry.visual_cursor == cursor:
|
|
2551
|
+
return entry
|
|
2552
|
+
raise LookupError(cursor)
|
|
2553
|
+
|
|
2554
|
+
def presentation(
|
|
2555
|
+
self, version: int | None = None
|
|
2556
|
+
) -> tuple[_JournalEntry, dict[str, str | None], dict[str, object]]:
|
|
2557
|
+
"""Return notebook, health, and lifecycle from one atomic viewer observation."""
|
|
2558
|
+
|
|
2559
|
+
with self._lock:
|
|
2560
|
+
return (
|
|
2561
|
+
self._snapshot_unlocked(version),
|
|
2562
|
+
self._health_unlocked(),
|
|
2563
|
+
dict(self._lifecycle),
|
|
2564
|
+
)
|
|
2565
|
+
|
|
2566
|
+
def presentation_cursor(
|
|
2567
|
+
self, cursor: int
|
|
2568
|
+
) -> tuple[_JournalEntry, dict[str, str | None], dict[str, object]]:
|
|
2569
|
+
"""Return the exact retained notebook state produced by one durable visual cursor."""
|
|
2570
|
+
|
|
2571
|
+
with self._lock:
|
|
2572
|
+
return (
|
|
2573
|
+
self._snapshot_cursor_unlocked(cursor),
|
|
2574
|
+
self._health_unlocked(),
|
|
2575
|
+
dict(self._lifecycle),
|
|
2576
|
+
)
|
|
2577
|
+
|
|
2578
|
+
@property
|
|
2579
|
+
def health(self) -> dict[str, str | None]:
|
|
2580
|
+
with self._lock:
|
|
2581
|
+
return self._health_unlocked()
|
|
2582
|
+
|
|
2583
|
+
@property
|
|
2584
|
+
def lifecycle(self) -> dict[str, object]:
|
|
2585
|
+
with self._lock:
|
|
2586
|
+
return dict(self._lifecycle)
|
|
2587
|
+
|
|
2588
|
+
def handoff_candidate_digest(self) -> str | None:
|
|
2589
|
+
"""Return the exact admitted Build identity only while its table is handed off."""
|
|
2590
|
+
|
|
2591
|
+
with self._lock:
|
|
2592
|
+
if not self._lifecycle.get("handoff"):
|
|
2593
|
+
return None
|
|
2594
|
+
if "table" not in self._history[-1].documents:
|
|
2595
|
+
return None
|
|
2596
|
+
return self._admitted_candidate_digest
|
|
2597
|
+
|
|
2598
|
+
def _health_unlocked(self) -> dict[str, str | None]:
|
|
2599
|
+
"""Describe one health state from the registry every other surface answers through.
|
|
2600
|
+
|
|
2601
|
+
The page, ``/health``, ``/snapshot`` and the command line all read this, and the sentences
|
|
2602
|
+
come from :func:`remediation_for` so that none of them can drift from the answer a person
|
|
2603
|
+
gets for the same code anywhere else. The command is the one part the map cannot write:
|
|
2604
|
+
restarting this viewer needs the exact target and research folders this process was given,
|
|
2605
|
+
and a map keyed by one path has no way to name two.
|
|
2606
|
+
|
|
2607
|
+
The observations are the ones taken when this state was entered rather than a fresh look:
|
|
2608
|
+
every streaming client reads this four times a second, under the lock the watcher needs,
|
|
2609
|
+
and the map's question is what the caller found when it looked. Both are passed, because
|
|
2610
|
+
a viewer started before the build has a run directory with no Build in it, and every
|
|
2611
|
+
default sentence for the table code is written about a Build that is there.
|
|
2612
|
+
"""
|
|
2613
|
+
|
|
2614
|
+
health: dict[str, str | None] = {"status": self._health_status, "code": self._health_code}
|
|
2615
|
+
if self._health_code is None:
|
|
2616
|
+
return health
|
|
2617
|
+
lines = remediation_for(
|
|
2618
|
+
self._health_code,
|
|
2619
|
+
path=str(self.run_dir),
|
|
2620
|
+
path_exists=self._health_path_exists,
|
|
2621
|
+
build_present=self._health_build_present,
|
|
2622
|
+
)
|
|
2623
|
+
prefix = "Run: "
|
|
2624
|
+
detail = " ".join(line for line in lines if not line.startswith(prefix))
|
|
2625
|
+
command = next(
|
|
2626
|
+
(line[len(prefix) :] for line in lines if line.startswith(prefix)),
|
|
2627
|
+
None,
|
|
2628
|
+
)
|
|
2629
|
+
if command is None and self._health_code == HEALTH_WATCHER_FAILED:
|
|
2630
|
+
target_flag = "--workspace" if self.target.mode == "workspace" else "--run-dir"
|
|
2631
|
+
research = (
|
|
2632
|
+
["--research-dir", str(self.research_path.parent)]
|
|
2633
|
+
if self.target.explicit_research
|
|
2634
|
+
else []
|
|
2635
|
+
)
|
|
2636
|
+
command = shlex.join(
|
|
2637
|
+
[
|
|
2638
|
+
"mr-data",
|
|
2639
|
+
"view",
|
|
2640
|
+
target_flag,
|
|
2641
|
+
str(self.target.selected_path),
|
|
2642
|
+
*research,
|
|
2643
|
+
"--json",
|
|
2644
|
+
]
|
|
2645
|
+
)
|
|
2646
|
+
health.update({"detail": detail, "remediation": command})
|
|
2647
|
+
return health
|
|
2648
|
+
|
|
2649
|
+
def mark_watcher_failed(self) -> None:
|
|
2650
|
+
with self._condition:
|
|
2651
|
+
self._health_status = "stopped"
|
|
2652
|
+
self._health_code = HEALTH_WATCHER_FAILED
|
|
2653
|
+
self._condition.notify_all()
|
|
2654
|
+
|
|
2655
|
+
def _notice_unlocked(self) -> dict[str, object]:
|
|
2656
|
+
return {
|
|
2657
|
+
"epoch": self.epoch,
|
|
2658
|
+
"version": self._version,
|
|
2659
|
+
"cursor": self._visual_cursor,
|
|
2660
|
+
"docs": list(self._history[-1].documents),
|
|
2661
|
+
"health": self._health_unlocked(),
|
|
2662
|
+
"lifecycle": self._lifecycle,
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
def notice(self) -> dict[str, object]:
|
|
2666
|
+
with self._lock:
|
|
2667
|
+
return self._notice_unlocked()
|
|
2668
|
+
|
|
2669
|
+
def wait_for_notice(
|
|
2670
|
+
self, last: tuple[object, ...] | None, timeout: float
|
|
2671
|
+
) -> tuple[dict[str, object], tuple[object, ...]]:
|
|
2672
|
+
with self._condition:
|
|
2673
|
+
current = (
|
|
2674
|
+
self.epoch,
|
|
2675
|
+
self._version,
|
|
2676
|
+
self._health_status,
|
|
2677
|
+
self._health_code,
|
|
2678
|
+
self._lifecycle_serial,
|
|
2679
|
+
)
|
|
2680
|
+
if last == current:
|
|
2681
|
+
self._condition.wait(timeout)
|
|
2682
|
+
notice = self._notice_unlocked()
|
|
2683
|
+
key = (
|
|
2684
|
+
notice["epoch"],
|
|
2685
|
+
notice["version"],
|
|
2686
|
+
notice["health"]["status"], # type: ignore[index]
|
|
2687
|
+
notice["health"]["code"], # type: ignore[index]
|
|
2688
|
+
self._lifecycle_serial,
|
|
2689
|
+
)
|
|
2690
|
+
return notice, key
|
|
2691
|
+
|
|
2692
|
+
|
|
2693
|
+
def _notebook_shell(raw: bytes, filename: str) -> str:
|
|
2694
|
+
notebook = json.loads(raw.decode("utf-8"))
|
|
2695
|
+
return render_notebook(notebook, mode="embed", filename=filename)
|
|
2696
|
+
|
|
2697
|
+
|
|
2698
|
+
def _waiting() -> str:
|
|
2699
|
+
return (
|
|
2700
|
+
'<section class="mr-waiting" role="status" aria-live="polite">'
|
|
2701
|
+
f"{_MARK_ONLY}"
|
|
2702
|
+
"<strong>No notebook has been written yet</strong>"
|
|
2703
|
+
"<span>Create research.ipynb for research notes. table.ipynb appears only after a "
|
|
2704
|
+
"successful verified Build.</span></section>"
|
|
2705
|
+
)
|
|
2706
|
+
|
|
2707
|
+
|
|
2708
|
+
def _render_snapshot(
|
|
2709
|
+
entry: _JournalEntry, requested: str, *, allow_table: bool = True
|
|
2710
|
+
) -> tuple[str, str, list[str]]:
|
|
2711
|
+
"""Render admitted documents, withholding table UI until narrated handoff completes."""
|
|
2712
|
+
|
|
2713
|
+
docs = [
|
|
2714
|
+
name for name in _DOCUMENTS if name in entry.documents and (name != "table" or allow_table)
|
|
2715
|
+
]
|
|
2716
|
+
active = requested if requested in docs else (docs[0] if docs else "")
|
|
2717
|
+
if not active:
|
|
2718
|
+
return _waiting(), active, docs
|
|
2719
|
+
filename = _DOCUMENTS[active]
|
|
2720
|
+
return _notebook_shell(entry.documents[active], filename), active, docs
|
|
2721
|
+
|
|
2722
|
+
|
|
2723
|
+
def _tabs(docs: list[str], active: str) -> str:
|
|
2724
|
+
tabs = []
|
|
2725
|
+
for name in _DOCUMENTS:
|
|
2726
|
+
hidden = "" if name in docs else " hidden"
|
|
2727
|
+
selected = name == active
|
|
2728
|
+
classes = "mr-tab is-active" if selected else "mr-tab"
|
|
2729
|
+
tabs.append(
|
|
2730
|
+
f'<button class="{classes}" type="button" role="tab" '
|
|
2731
|
+
f'aria-selected="{str(selected).lower()}" data-doc="{name}"{hidden}>'
|
|
2732
|
+
'<svg class="mr-tab-icon" viewBox="0 0 16 16" aria-hidden="true">'
|
|
2733
|
+
'<rect x="3.25" y="1.75" width="10.5" height="12.5" rx="1.75"></rect>'
|
|
2734
|
+
'<path d="M6 1.75v12.5M8.5 5h3M8.5 8h3M8.5 11h2"></path></svg>'
|
|
2735
|
+
f'<span class="mr-tab-name">{name}<span class="mr-tab-ext">.ipynb</span></span></button>'
|
|
2736
|
+
)
|
|
2737
|
+
return "".join(tabs)
|
|
2738
|
+
|
|
2739
|
+
|
|
2740
|
+
def _lifecycle_block(lifecycle: dict[str, object]) -> str:
|
|
2741
|
+
status = str(lifecycle.get("status", "idle"))
|
|
2742
|
+
hidden = " hidden" if status == "idle" else ""
|
|
2743
|
+
state_label = (
|
|
2744
|
+
"Stopped"
|
|
2745
|
+
if status == "failed"
|
|
2746
|
+
else "Complete"
|
|
2747
|
+
if status == "completed"
|
|
2748
|
+
else "Ready"
|
|
2749
|
+
if status == "idle"
|
|
2750
|
+
else "Running"
|
|
2751
|
+
)
|
|
2752
|
+
items = []
|
|
2753
|
+
for line in lifecycle.get("lines", []):
|
|
2754
|
+
if not isinstance(line, dict):
|
|
2755
|
+
continue
|
|
2756
|
+
line_status = str(line.get("status", "running"))
|
|
2757
|
+
mark = "!" if line_status == "failed" else "ok" if line_status == "done" else "."
|
|
2758
|
+
items.append(
|
|
2759
|
+
f'<li class="mr-build-line is-{html.escape(line_status)}" '
|
|
2760
|
+
f'data-build-span="{html.escape(str(line.get("key", "")))}">'
|
|
2761
|
+
f'<span class="mr-build-mark" aria-hidden="true">{mark}</span>'
|
|
2762
|
+
'<span class="mr-build-copy">'
|
|
2763
|
+
f'<span class="mr-build-label">{html.escape(str(line.get("label", "")))}</span>'
|
|
2764
|
+
f'<span class="mr-build-detail">{html.escape(str(line.get("detail", "")))}</span>'
|
|
2765
|
+
"</span></li>"
|
|
2766
|
+
)
|
|
2767
|
+
classes = "mr-build-progress is-failed" if status == "failed" else "mr-build-progress"
|
|
2768
|
+
opened = " open" if status not in {"idle", "completed"} else ""
|
|
2769
|
+
return (
|
|
2770
|
+
f'<details class="{classes}" data-build-progress{hidden}{opened}>'
|
|
2771
|
+
'<summary class="mr-build-progress-head" aria-live="polite">'
|
|
2772
|
+
f'<span class="mr-build-progress-state" data-build-state>{state_label}</span>'
|
|
2773
|
+
f"<strong data-build-phase>{html.escape(str(lifecycle.get('phase', '')))}</strong>"
|
|
2774
|
+
"</summary><ol data-build-lines>" + "".join(items) + "</ol></details>"
|
|
2775
|
+
)
|
|
2776
|
+
|
|
2777
|
+
|
|
2778
|
+
def _research_terminal_block(lifecycle: dict[str, object], active: str) -> str:
|
|
2779
|
+
"""Render a failed attempt's real feed terminal as the research document's closing block."""
|
|
2780
|
+
|
|
2781
|
+
terminal = lifecycle.get("terminal")
|
|
2782
|
+
visible = isinstance(terminal, Mapping) and active != "table"
|
|
2783
|
+
if not isinstance(terminal, Mapping):
|
|
2784
|
+
terminal = {}
|
|
2785
|
+
metadata = [
|
|
2786
|
+
str(value)
|
|
2787
|
+
for value in (terminal.get("severity"), terminal.get("finding_id"))
|
|
2788
|
+
if isinstance(value, str) and value
|
|
2789
|
+
]
|
|
2790
|
+
metadata_text = " · ".join(metadata)
|
|
2791
|
+
hidden = "" if visible else " hidden"
|
|
2792
|
+
metadata_hidden = "" if metadata_text else " hidden"
|
|
2793
|
+
return (
|
|
2794
|
+
'<section class="mr-research-terminal" data-research-terminal role="status" aria-atomic="true"'
|
|
2795
|
+
f' data-terminal-event="{html.escape(str(terminal.get("event", "")))}"{hidden}>'
|
|
2796
|
+
f"<strong>{html.escape(str(terminal.get('title', '')))}</strong>"
|
|
2797
|
+
f"<p>{html.escape(str(terminal.get('detail', '')))}</p>"
|
|
2798
|
+
f"<small{metadata_hidden}>{html.escape(metadata_text)}</small></section>"
|
|
2799
|
+
)
|
|
2800
|
+
|
|
2801
|
+
|
|
2802
|
+
def _workbench_document(
|
|
2803
|
+
shell: str,
|
|
2804
|
+
*,
|
|
2805
|
+
run_name: str,
|
|
2806
|
+
active: str,
|
|
2807
|
+
docs: list[str],
|
|
2808
|
+
version: int,
|
|
2809
|
+
epoch: str,
|
|
2810
|
+
health: dict[str, str | None],
|
|
2811
|
+
visual_cursor: int = 0,
|
|
2812
|
+
lifecycle: dict[str, object] | None = None,
|
|
2813
|
+
) -> str:
|
|
2814
|
+
lifecycle = lifecycle or reduce_lifecycle([], table_ready=False)
|
|
2815
|
+
handoff = bool(lifecycle.get("handoff"))
|
|
2816
|
+
download_state = (
|
|
2817
|
+
' href="/download/table.parquet" aria-disabled="false"'
|
|
2818
|
+
if handoff
|
|
2819
|
+
else ' aria-disabled="true" tabindex="-1"'
|
|
2820
|
+
)
|
|
2821
|
+
lifecycle_phase = str(lifecycle.get("phase", "Research notebook"))
|
|
2822
|
+
phase = html.escape("Research" if lifecycle_phase == "Research notebook" else lifecycle_phase)
|
|
2823
|
+
progress = int(lifecycle.get("progress", 0))
|
|
2824
|
+
elapsed = format_elapsed(elapsed_seconds(lifecycle, now=time.time()))
|
|
2825
|
+
elapsed_hidden = "" if elapsed else " hidden"
|
|
2826
|
+
files_label = "Show table in Finder" if handoff else "Show run in Finder"
|
|
2827
|
+
initial_status = (
|
|
2828
|
+
"Stopped"
|
|
2829
|
+
if health["status"] == "stopped"
|
|
2830
|
+
else "Unavailable"
|
|
2831
|
+
if health["status"] == "unavailable"
|
|
2832
|
+
else "Connecting"
|
|
2833
|
+
)
|
|
2834
|
+
detail_hidden = "" if health["status"] in {"unavailable", "stopped"} else " hidden"
|
|
2835
|
+
detail_code = html.escape(health.get("code") or "")
|
|
2836
|
+
detail_text = html.escape(health.get("detail") or "")
|
|
2837
|
+
detail_command = html.escape(health.get("remediation") or "")
|
|
2838
|
+
tabs_class = "mr-tabs is-single" if len(docs) == 1 else "mr-tabs"
|
|
2839
|
+
return (
|
|
2840
|
+
'<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'
|
|
2841
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
2842
|
+
f"<title>{html.escape(run_name)} · Mostly Right</title><style>{_WORKBENCH_CSS}</style>"
|
|
2843
|
+
"</head>"
|
|
2844
|
+
f'<body data-doc="{html.escape(active)}" data-version="{version}" '
|
|
2845
|
+
f'data-cursor="{visual_cursor}" '
|
|
2846
|
+
f'data-epoch="{html.escape(epoch)}" data-health="{html.escape(str(health["status"]))}" '
|
|
2847
|
+
f'data-handoff="{str(handoff).lower()}">'
|
|
2848
|
+
'<header class="mr-topbar">'
|
|
2849
|
+
f'<a class="mr-brand" href="/" aria-label="Mostly Right">{_LOGO}</a>'
|
|
2850
|
+
'<div class="mr-session"><span class="mr-session-copy">'
|
|
2851
|
+
f"<small>{html.escape(run_name)}</small><strong data-lifecycle-phase>{phase}</strong></span></div>"
|
|
2852
|
+
f'<nav class="{tabs_class}" role="tablist" aria-label="Open notebooks">{_tabs(docs, active)}</nav>'
|
|
2853
|
+
'<div class="mr-actions"><span class="mr-live" data-live-status role="status" '
|
|
2854
|
+
f'aria-live="polite"><i></i><span>{initial_status}</span></span>'
|
|
2855
|
+
f'<time class="mr-run-clock" data-run-clock{elapsed_hidden}>{html.escape(elapsed)}</time>'
|
|
2856
|
+
'<button class="mr-button" type="button" data-follow-agent hidden>Follow agent</button>'
|
|
2857
|
+
f'<button class="mr-button" type="button" data-explore{"" if handoff else " disabled"}>Explore rows</button>'
|
|
2858
|
+
f'<button class="mr-button" type="button" data-files aria-label="{files_label}">'
|
|
2859
|
+
'<svg aria-hidden="true" width="15" height="15" viewBox="0 0 16 16" fill="none"><path d="M1.5 4.25h5l1.2 1.5h6.8v6.5H1.5z" stroke="currentColor" stroke-width="1.25"/></svg>'
|
|
2860
|
+
f'<span class="mr-files-label">{files_label}</span></button>'
|
|
2861
|
+
f'<a class="mr-download" data-download{download_state}>Download table</a>'
|
|
2862
|
+
f'</div><div class="mr-overall" role="progressbar" aria-label="Build progress" aria-valuemin="0" '
|
|
2863
|
+
f'aria-valuemax="100" aria-valuenow="{progress}" data-overall-progress><span style="width:{progress}%"></span></div></header>'
|
|
2864
|
+
'<main class="mr-workbench"><div class="mr-stage">'
|
|
2865
|
+
f"{_lifecycle_block(lifecycle)}"
|
|
2866
|
+
f'<section class="mr-health-detail" data-health-detail role="alert" aria-atomic="true"{detail_hidden}>'
|
|
2867
|
+
f"<strong>{detail_code}</strong><span>{detail_text}</span><code>{detail_command}</code>"
|
|
2868
|
+
'</section><div class="mr-sr-status" data-stream-announcement role="status" aria-live="polite" aria-atomic="true"></div>'
|
|
2869
|
+
'<div class="mr-notebook-root" '
|
|
2870
|
+
f'data-notebook-root data-doc="{html.escape(active)}">{shell}</div>'
|
|
2871
|
+
f"{_research_terminal_block(lifecycle, active)}</div></main>"
|
|
2872
|
+
'<dialog class="mr-explorer" data-explorer aria-labelledby="mr-explorer-title">'
|
|
2873
|
+
'<div class="mr-explorer-head"><strong id="mr-explorer-title">Verified table rows</strong>'
|
|
2874
|
+
'<button class="mr-button" type="button" data-explorer-close>Close</button></div>'
|
|
2875
|
+
'<div class="mr-explorer-body" data-explorer-body></div></dialog>'
|
|
2876
|
+
f"{_RELOAD_SCRIPT}</body></html>\n"
|
|
2877
|
+
)
|
|
2878
|
+
|
|
2879
|
+
|
|
2880
|
+
def _document(shell: str, title: str) -> str:
|
|
2881
|
+
"""Compatibility full document used by direct ``render_html`` callers."""
|
|
2882
|
+
|
|
2883
|
+
return (
|
|
2884
|
+
'<!doctype html>\n<html lang="en"><head><meta charset="utf-8">'
|
|
2885
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
2886
|
+
f"<title>{html.escape(title)}</title>"
|
|
2887
|
+
"<style>body{margin:0;background:#f1f3f6;padding:24px}.nb-shell{max-width:1180px;margin:0 auto}</style>"
|
|
2888
|
+
f"</head><body>{shell}{_RELOAD_SCRIPT}</body></html>\n"
|
|
2889
|
+
)
|
|
2890
|
+
|
|
2891
|
+
|
|
2892
|
+
def render_html(ipynb_path: Path) -> str:
|
|
2893
|
+
"""Render one notebook as a self-contained static document, without a kernel."""
|
|
2894
|
+
|
|
2895
|
+
ipynb_path = Path(ipynb_path)
|
|
2896
|
+
return render_html_bytes(ipynb_path.read_bytes(), filename=ipynb_path.name)
|
|
2897
|
+
|
|
2898
|
+
|
|
2899
|
+
def render_html_bytes(raw: bytes, *, filename: str = _SIDECAR_NAME) -> str:
|
|
2900
|
+
"""Render already-read notebook bytes without reopening the selected path."""
|
|
2901
|
+
|
|
2902
|
+
notebook = json.loads(raw)
|
|
2903
|
+
shell = render_notebook(notebook, mode="static", filename=filename)
|
|
2904
|
+
return _document(shell, title=filename)
|
|
2905
|
+
|
|
2906
|
+
|
|
2907
|
+
class _ViewerServer(ThreadingHTTPServer):
|
|
2908
|
+
daemon_threads = True
|
|
2909
|
+
|
|
2910
|
+
def __init__(self, address: tuple[str, int], handler: type, state: _ViewerState) -> None:
|
|
2911
|
+
self.viewer_state = state
|
|
2912
|
+
self.viewer_stopped = False
|
|
2913
|
+
self.viewer_watcher: threading.Thread | None = None
|
|
2914
|
+
self._stream_lock = threading.Lock()
|
|
2915
|
+
self._stream_connections: set[socket.socket] = set()
|
|
2916
|
+
super().__init__(address, handler)
|
|
2917
|
+
|
|
2918
|
+
def register_stream(self, connection: socket.socket) -> None:
|
|
2919
|
+
with self._stream_lock:
|
|
2920
|
+
self._stream_connections.add(connection)
|
|
2921
|
+
|
|
2922
|
+
def unregister_stream(self, connection: socket.socket) -> None:
|
|
2923
|
+
with self._stream_lock:
|
|
2924
|
+
self._stream_connections.discard(connection)
|
|
2925
|
+
|
|
2926
|
+
def close_streams(self) -> None:
|
|
2927
|
+
with self._stream_lock:
|
|
2928
|
+
connections = tuple(self._stream_connections)
|
|
2929
|
+
for connection in connections:
|
|
2930
|
+
try:
|
|
2931
|
+
connection.shutdown(socket.SHUT_RDWR)
|
|
2932
|
+
except OSError:
|
|
2933
|
+
pass
|
|
2934
|
+
|
|
2935
|
+
def shutdown(self) -> None:
|
|
2936
|
+
self.viewer_stopped = True
|
|
2937
|
+
self.close_streams()
|
|
2938
|
+
super().shutdown()
|
|
2939
|
+
|
|
2940
|
+
def server_close(self) -> None:
|
|
2941
|
+
try:
|
|
2942
|
+
super().server_close()
|
|
2943
|
+
finally:
|
|
2944
|
+
self.viewer_state.close()
|
|
2945
|
+
|
|
2946
|
+
def handle_error(self, _request: object, _client_address: object) -> None:
|
|
2947
|
+
"""A browser closing an SSE socket is normal and should not spill a traceback."""
|
|
2948
|
+
|
|
2949
|
+
return
|
|
2950
|
+
|
|
2951
|
+
|
|
2952
|
+
class _ViewerServerV6(_ViewerServer):
|
|
2953
|
+
address_family = socket.AF_INET6
|
|
2954
|
+
|
|
2955
|
+
|
|
2956
|
+
class _TableUnavailable(RuntimeError):
|
|
2957
|
+
pass
|
|
2958
|
+
|
|
2959
|
+
|
|
2960
|
+
def _verified_table_snapshot(
|
|
2961
|
+
state: _ViewerState,
|
|
2962
|
+
*,
|
|
2963
|
+
expected_candidate_digest: str,
|
|
2964
|
+
) -> tuple[tempfile.SpooledTemporaryFile[bytes], int]:
|
|
2965
|
+
"""Copy exact admitted-and-reverified Parquet bytes into an immutable response snapshot."""
|
|
2966
|
+
|
|
2967
|
+
retained: list[object] = []
|
|
2968
|
+
data_fd = -1
|
|
2969
|
+
source_fd = -1
|
|
2970
|
+
spool: tempfile.SpooledTemporaryFile[bytes] | None = None
|
|
2971
|
+
try:
|
|
2972
|
+
verified = verify_candidate(state.run_dir, _retained_leases=retained) # type: ignore[arg-type]
|
|
2973
|
+
if len(retained) != 1:
|
|
2974
|
+
raise _TableUnavailable("Build verification lease unavailable")
|
|
2975
|
+
if verified.candidate_digest != expected_candidate_digest:
|
|
2976
|
+
raise _TableUnavailable("verified Build differs from the admitted handoff")
|
|
2977
|
+
lease = retained[0]
|
|
2978
|
+
candidate_fd = lease.candidate_fd # type: ignore[attr-defined]
|
|
2979
|
+
directory_flags = (
|
|
2980
|
+
os.O_RDONLY
|
|
2981
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
2982
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
2983
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
2984
|
+
)
|
|
2985
|
+
data_fd = os.open("data", directory_flags, dir_fd=candidate_fd)
|
|
2986
|
+
if not stat.S_ISDIR(os.fstat(data_fd).st_mode):
|
|
2987
|
+
raise _TableUnavailable("Build data member is not a directory")
|
|
2988
|
+
source_fd = os.open(
|
|
2989
|
+
"table.parquet",
|
|
2990
|
+
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
2991
|
+
dir_fd=data_fd,
|
|
2992
|
+
)
|
|
2993
|
+
before = os.fstat(source_fd)
|
|
2994
|
+
named = os.stat("table.parquet", dir_fd=data_fd, follow_symlinks=False)
|
|
2995
|
+
if not _is_regular_single_link(before) or _stat_signature(before) != _stat_signature(named):
|
|
2996
|
+
raise _TableUnavailable("table member is not a stable regular file")
|
|
2997
|
+
expected_member = next(
|
|
2998
|
+
(
|
|
2999
|
+
item
|
|
3000
|
+
for item in verified.manifest.get("members", [])
|
|
3001
|
+
if isinstance(item, dict) and item.get("path") == "data/table.parquet"
|
|
3002
|
+
),
|
|
3003
|
+
None,
|
|
3004
|
+
)
|
|
3005
|
+
if not isinstance(expected_member, dict):
|
|
3006
|
+
raise _TableUnavailable("verified manifest has no table member")
|
|
3007
|
+
expected_size = expected_member.get("bytes")
|
|
3008
|
+
if (
|
|
3009
|
+
not isinstance(expected_size, int)
|
|
3010
|
+
or isinstance(expected_size, bool)
|
|
3011
|
+
or expected_size < 0
|
|
3012
|
+
):
|
|
3013
|
+
raise _TableUnavailable("verified table size is invalid")
|
|
3014
|
+
spool = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024, mode="w+b")
|
|
3015
|
+
digest = hashlib.sha256()
|
|
3016
|
+
copied = 0
|
|
3017
|
+
while True:
|
|
3018
|
+
chunk = os.read(source_fd, _READ_CHUNK_BYTES)
|
|
3019
|
+
if not chunk:
|
|
3020
|
+
break
|
|
3021
|
+
copied += len(chunk)
|
|
3022
|
+
if copied > expected_size:
|
|
3023
|
+
raise _TableUnavailable("table changed while preparing the download")
|
|
3024
|
+
digest.update(chunk)
|
|
3025
|
+
spool.write(chunk)
|
|
3026
|
+
after = os.fstat(source_fd)
|
|
3027
|
+
named_after = os.stat("table.parquet", dir_fd=data_fd, follow_symlinks=False)
|
|
3028
|
+
if (
|
|
3029
|
+
copied != expected_size
|
|
3030
|
+
or copied != before.st_size
|
|
3031
|
+
or _stat_signature(after) != _stat_signature(before)
|
|
3032
|
+
or _stat_signature(named_after) != _stat_signature(before)
|
|
3033
|
+
or digest.hexdigest() != verified.table_sha256
|
|
3034
|
+
):
|
|
3035
|
+
raise _TableUnavailable("table changed or failed its verified digest")
|
|
3036
|
+
lease.validate() # type: ignore[attr-defined]
|
|
3037
|
+
spool.seek(0)
|
|
3038
|
+
ready = spool
|
|
3039
|
+
spool = None
|
|
3040
|
+
return ready, copied
|
|
3041
|
+
except _TableUnavailable:
|
|
3042
|
+
raise
|
|
3043
|
+
except Exception as exc:
|
|
3044
|
+
raise _TableUnavailable("table is not a currently verified snapshot") from exc
|
|
3045
|
+
finally:
|
|
3046
|
+
if source_fd >= 0:
|
|
3047
|
+
os.close(source_fd)
|
|
3048
|
+
if data_fd >= 0:
|
|
3049
|
+
os.close(data_fd)
|
|
3050
|
+
for lease in retained:
|
|
3051
|
+
lease.close() # type: ignore[attr-defined]
|
|
3052
|
+
if spool is not None:
|
|
3053
|
+
spool.close()
|
|
3054
|
+
|
|
3055
|
+
|
|
3056
|
+
def _verified_query_snapshot(
|
|
3057
|
+
state: _ViewerState, *, expected_candidate_digest: str
|
|
3058
|
+
) -> tuple[bytes, QueryEvidence]:
|
|
3059
|
+
"""Pin Parquet and all explorer evidence to one replay-verified snapshot."""
|
|
3060
|
+
|
|
3061
|
+
try:
|
|
3062
|
+
with open_verified_snapshot(state.run_dir) as snapshot:
|
|
3063
|
+
verified = snapshot.verified
|
|
3064
|
+
if verified.candidate_digest != expected_candidate_digest:
|
|
3065
|
+
raise _TableUnavailable("verified Build differs from the admitted handoff")
|
|
3066
|
+
parquet_raw = snapshot.member_bytes("data/table.parquet")
|
|
3067
|
+
profile_raw = snapshot.member_bytes("evidence/profile.json")
|
|
3068
|
+
quality_raw = snapshot.member_bytes("evidence/quality.json")
|
|
3069
|
+
lineage_raw = snapshot.member_bytes("evidence/lineage.json")
|
|
3070
|
+
sources_raw = snapshot.member_bytes("evidence/sources.json")
|
|
3071
|
+
profile = json.loads(profile_raw)
|
|
3072
|
+
quality = json.loads(quality_raw)
|
|
3073
|
+
lineage_value = json.loads(lineage_raw)
|
|
3074
|
+
sources = json.loads(sources_raw)
|
|
3075
|
+
if not all(
|
|
3076
|
+
isinstance(value, dict) for value in (profile, quality, lineage_value)
|
|
3077
|
+
) or not isinstance(sources, list):
|
|
3078
|
+
raise _TableUnavailable("verified explorer evidence has an invalid shape")
|
|
3079
|
+
|
|
3080
|
+
observations_raw: bytes | None = None
|
|
3081
|
+
observations: list[dict[str, object]] = []
|
|
3082
|
+
if "evidence/observations.json" in snapshot.members:
|
|
3083
|
+
observations_raw = snapshot.member_bytes("evidence/observations.json")
|
|
3084
|
+
observation_set = json.loads(observations_raw)
|
|
3085
|
+
if (
|
|
3086
|
+
not isinstance(observation_set, dict)
|
|
3087
|
+
or not isinstance(observation_set.get("observations"), list)
|
|
3088
|
+
or any(not isinstance(item, dict) for item in observation_set["observations"])
|
|
3089
|
+
):
|
|
3090
|
+
raise _TableUnavailable("verified observation evidence is invalid")
|
|
3091
|
+
observations = observation_set["observations"]
|
|
3092
|
+
|
|
3093
|
+
columns = profile.get("columns")
|
|
3094
|
+
if not isinstance(columns, list) or any(not isinstance(item, str) for item in columns):
|
|
3095
|
+
raise _TableUnavailable("verified profile does not name the table columns")
|
|
3096
|
+
terminal = observations[-1] if observations else None
|
|
3097
|
+
terminal_profiles = terminal.get("profiles") if terminal is not None else None
|
|
3098
|
+
if isinstance(terminal_profiles, list):
|
|
3099
|
+
profiles = {
|
|
3100
|
+
str(item["column"]): dict(item)
|
|
3101
|
+
for item in terminal_profiles
|
|
3102
|
+
if isinstance(item, dict) and isinstance(item.get("column"), str)
|
|
3103
|
+
}
|
|
3104
|
+
else:
|
|
3105
|
+
types = profile.get("types")
|
|
3106
|
+
null_counts = profile.get("null_counts")
|
|
3107
|
+
row_count = profile.get("row_count")
|
|
3108
|
+
if (
|
|
3109
|
+
not isinstance(types, dict)
|
|
3110
|
+
or not isinstance(null_counts, dict)
|
|
3111
|
+
or isinstance(row_count, bool)
|
|
3112
|
+
or not isinstance(row_count, int)
|
|
3113
|
+
):
|
|
3114
|
+
raise _TableUnavailable("verified profiles are incomplete")
|
|
3115
|
+
profiles = {
|
|
3116
|
+
column: {
|
|
3117
|
+
"column": column,
|
|
3118
|
+
"rows_examined": row_count,
|
|
3119
|
+
"logical_type": types.get(column),
|
|
3120
|
+
"null_count": null_counts.get(column),
|
|
3121
|
+
}
|
|
3122
|
+
for column in columns
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
lineage_columns = lineage_value.get("columns", lineage_value)
|
|
3126
|
+
if not isinstance(lineage_columns, dict):
|
|
3127
|
+
raise _TableUnavailable("verified lineage is incomplete")
|
|
3128
|
+
normalized_lineage = {
|
|
3129
|
+
column: {"column": column, **dict(lineage_columns[column])}
|
|
3130
|
+
for column in columns
|
|
3131
|
+
if isinstance(lineage_columns.get(column), dict)
|
|
3132
|
+
}
|
|
3133
|
+
findings = terminal.get("findings", []) if terminal is not None else []
|
|
3134
|
+
if not isinstance(findings, list) or any(
|
|
3135
|
+
not isinstance(item, dict) for item in findings
|
|
3136
|
+
):
|
|
3137
|
+
raise _TableUnavailable("verified findings are invalid")
|
|
3138
|
+
rights_statuses: list[str] = []
|
|
3139
|
+
classifications: list[str] = []
|
|
3140
|
+
for source in sources:
|
|
3141
|
+
if not isinstance(source, dict) or not isinstance(source.get("rights"), dict):
|
|
3142
|
+
raise _TableUnavailable("verified source governance is invalid")
|
|
3143
|
+
rights_statuses.append(str(source["rights"].get("status", "unknown")))
|
|
3144
|
+
classifications.append(str(source.get("classification", "unknown")))
|
|
3145
|
+
display = (
|
|
3146
|
+
str(terminal.get("display"))
|
|
3147
|
+
if terminal is not None
|
|
3148
|
+
else (
|
|
3149
|
+
"full"
|
|
3150
|
+
if rights_statuses
|
|
3151
|
+
and all(
|
|
3152
|
+
status in {"public_domain", "permissive_license"}
|
|
3153
|
+
for status in rights_statuses
|
|
3154
|
+
)
|
|
3155
|
+
and classifications
|
|
3156
|
+
and all(item == "public" for item in classifications)
|
|
3157
|
+
else "schema_only"
|
|
3158
|
+
)
|
|
3159
|
+
)
|
|
3160
|
+
member_bytes = {
|
|
3161
|
+
"evidence/profile.json": profile_raw,
|
|
3162
|
+
"evidence/quality.json": quality_raw,
|
|
3163
|
+
"evidence/lineage.json": lineage_raw,
|
|
3164
|
+
"evidence/sources.json": sources_raw,
|
|
3165
|
+
}
|
|
3166
|
+
if observations_raw is not None:
|
|
3167
|
+
member_bytes["evidence/observations.json"] = observations_raw
|
|
3168
|
+
evidence = QueryEvidence(
|
|
3169
|
+
candidate_digest=verified.candidate_digest,
|
|
3170
|
+
table_sha256=verified.table_sha256,
|
|
3171
|
+
display=display,
|
|
3172
|
+
rights_statuses=tuple(rights_statuses),
|
|
3173
|
+
classifications=tuple(classifications),
|
|
3174
|
+
profiles=profiles,
|
|
3175
|
+
findings=tuple(findings),
|
|
3176
|
+
lineage=normalized_lineage,
|
|
3177
|
+
evidence_sha256={
|
|
3178
|
+
path: hashlib.sha256(raw).hexdigest() for path, raw in member_bytes.items()
|
|
3179
|
+
},
|
|
3180
|
+
)
|
|
3181
|
+
return parquet_raw, evidence
|
|
3182
|
+
except _TableUnavailable:
|
|
3183
|
+
raise
|
|
3184
|
+
except Exception as exc:
|
|
3185
|
+
raise _TableUnavailable("table is not a currently verified query snapshot") from exc
|
|
3186
|
+
|
|
3187
|
+
|
|
3188
|
+
def _reveal_verified_table(state: _ViewerState, *, expected_candidate_digest: str) -> None:
|
|
3189
|
+
"""Launch the platform file action only for the exact admitted, still-verified Build."""
|
|
3190
|
+
|
|
3191
|
+
retained: list[object] = []
|
|
3192
|
+
data_fd = -1
|
|
3193
|
+
table_fd = -1
|
|
3194
|
+
try:
|
|
3195
|
+
try:
|
|
3196
|
+
verified = verify_candidate( # type: ignore[arg-type]
|
|
3197
|
+
state.run_dir,
|
|
3198
|
+
_retained_leases=retained,
|
|
3199
|
+
)
|
|
3200
|
+
if len(retained) != 1 or verified.candidate_digest != expected_candidate_digest:
|
|
3201
|
+
raise _TableUnavailable("verified Build differs from the admitted handoff")
|
|
3202
|
+
lease = retained[0]
|
|
3203
|
+
lease.validate() # type: ignore[attr-defined]
|
|
3204
|
+
except _TableUnavailable:
|
|
3205
|
+
raise
|
|
3206
|
+
except Exception as exc:
|
|
3207
|
+
raise _TableUnavailable("table is not the admitted verified snapshot") from exc
|
|
3208
|
+
directory_flags = (
|
|
3209
|
+
os.O_RDONLY
|
|
3210
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
3211
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
3212
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
3213
|
+
)
|
|
3214
|
+
data_fd = os.open("data", directory_flags, dir_fd=lease.candidate_fd) # type: ignore[attr-defined]
|
|
3215
|
+
table_fd = os.open(
|
|
3216
|
+
"table.parquet",
|
|
3217
|
+
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
3218
|
+
dir_fd=data_fd,
|
|
3219
|
+
)
|
|
3220
|
+
opened = os.fstat(table_fd)
|
|
3221
|
+
named = os.stat("table.parquet", dir_fd=data_fd, follow_symlinks=False)
|
|
3222
|
+
if not _is_regular_single_link(opened) or _stat_signature(opened) != _stat_signature(named):
|
|
3223
|
+
raise _TableUnavailable("table member is not a stable regular file")
|
|
3224
|
+
descriptor_path = (
|
|
3225
|
+
f"/dev/fd/{table_fd}" if sys.platform == "darwin" else f"/proc/self/fd/{table_fd}"
|
|
3226
|
+
)
|
|
3227
|
+
command = (
|
|
3228
|
+
["open", "-R", descriptor_path]
|
|
3229
|
+
if sys.platform == "darwin"
|
|
3230
|
+
else ["xdg-open", descriptor_path]
|
|
3231
|
+
)
|
|
3232
|
+
# The child inherits the already-open table descriptor. A same-UID actor may replace the
|
|
3233
|
+
# public pathname before launch, but cannot redirect this descriptor-backed target.
|
|
3234
|
+
subprocess.Popen(
|
|
3235
|
+
command,
|
|
3236
|
+
stdout=subprocess.DEVNULL,
|
|
3237
|
+
stderr=subprocess.DEVNULL,
|
|
3238
|
+
pass_fds=(table_fd,),
|
|
3239
|
+
)
|
|
3240
|
+
try:
|
|
3241
|
+
lease.validate() # type: ignore[attr-defined]
|
|
3242
|
+
except Exception as exc:
|
|
3243
|
+
raise _TableUnavailable("table changed while opening its location") from exc
|
|
3244
|
+
finally:
|
|
3245
|
+
if table_fd >= 0:
|
|
3246
|
+
os.close(table_fd)
|
|
3247
|
+
if data_fd >= 0:
|
|
3248
|
+
os.close(data_fd)
|
|
3249
|
+
for lease in retained:
|
|
3250
|
+
lease.close() # type: ignore[attr-defined]
|
|
3251
|
+
|
|
3252
|
+
|
|
3253
|
+
class _ViewerHandler(BaseHTTPRequestHandler):
|
|
3254
|
+
def log_message(self, *_args: object) -> None:
|
|
3255
|
+
return
|
|
3256
|
+
|
|
3257
|
+
@property
|
|
3258
|
+
def _state(self) -> _ViewerState:
|
|
3259
|
+
return self.server.viewer_state # type: ignore[attr-defined]
|
|
3260
|
+
|
|
3261
|
+
def do_GET(self) -> None:
|
|
3262
|
+
if not self._request_targets_this_loopback_server():
|
|
3263
|
+
self.send_error(421)
|
|
3264
|
+
return
|
|
3265
|
+
parsed = urlsplit(self.path)
|
|
3266
|
+
if parsed.path == "/":
|
|
3267
|
+
self._serve_index(parse_qs(parsed.query))
|
|
3268
|
+
elif parsed.path == "/snapshot":
|
|
3269
|
+
self._serve_snapshot(parse_qs(parsed.query))
|
|
3270
|
+
elif parsed.path == "/health":
|
|
3271
|
+
self._write_json(200, self._state.notice())
|
|
3272
|
+
elif parsed.path == "/events":
|
|
3273
|
+
self._serve_events()
|
|
3274
|
+
elif parsed.path == "/download/table.parquet":
|
|
3275
|
+
self._serve_table()
|
|
3276
|
+
elif parsed.path == "/rows":
|
|
3277
|
+
self._serve_rows(parse_qs(parsed.query))
|
|
3278
|
+
else:
|
|
3279
|
+
self.send_error(404)
|
|
3280
|
+
|
|
3281
|
+
def do_POST(self) -> None:
|
|
3282
|
+
if not self._request_targets_this_loopback_server():
|
|
3283
|
+
self.send_error(421)
|
|
3284
|
+
return
|
|
3285
|
+
if urlsplit(self.path).path != "/open-files":
|
|
3286
|
+
self.send_error(404)
|
|
3287
|
+
return
|
|
3288
|
+
candidate_digest = self._state.handoff_candidate_digest()
|
|
3289
|
+
if candidate_digest is None:
|
|
3290
|
+
location = (
|
|
3291
|
+
self._state.run_dir
|
|
3292
|
+
if self._state._run_directory_exists()
|
|
3293
|
+
else self._state.target.selected_path.parent
|
|
3294
|
+
)
|
|
3295
|
+
command = (
|
|
3296
|
+
["open", str(location)] if sys.platform == "darwin" else ["xdg-open", str(location)]
|
|
3297
|
+
)
|
|
3298
|
+
try:
|
|
3299
|
+
subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
3300
|
+
except OSError:
|
|
3301
|
+
self._write_json(500, {"ok": False})
|
|
3302
|
+
return
|
|
3303
|
+
else:
|
|
3304
|
+
try:
|
|
3305
|
+
_reveal_verified_table(
|
|
3306
|
+
self._state,
|
|
3307
|
+
expected_candidate_digest=candidate_digest,
|
|
3308
|
+
)
|
|
3309
|
+
except _TableUnavailable:
|
|
3310
|
+
self._write_json(409, {"ok": False})
|
|
3311
|
+
return
|
|
3312
|
+
except OSError:
|
|
3313
|
+
self._write_json(500, {"ok": False})
|
|
3314
|
+
return
|
|
3315
|
+
self._write_json(200, {"ok": True})
|
|
3316
|
+
|
|
3317
|
+
def _request_targets_this_loopback_server(self) -> bool:
|
|
3318
|
+
"""Reject DNS-rebound and cross-origin access to local run contents."""
|
|
3319
|
+
|
|
3320
|
+
bound_port = int(self.server.server_address[1])
|
|
3321
|
+
try:
|
|
3322
|
+
bound_address = ipaddress.ip_address(str(self.server.server_address[0]))
|
|
3323
|
+
except ValueError:
|
|
3324
|
+
return False
|
|
3325
|
+
|
|
3326
|
+
def local_authority(value: str, *, origin: bool) -> bool:
|
|
3327
|
+
try:
|
|
3328
|
+
parsed = urlsplit(value if origin else f"//{value}")
|
|
3329
|
+
hostname = parsed.hostname
|
|
3330
|
+
port = parsed.port
|
|
3331
|
+
except ValueError:
|
|
3332
|
+
return False
|
|
3333
|
+
if parsed.username is not None or parsed.password is not None:
|
|
3334
|
+
return False
|
|
3335
|
+
if hostname is None or port != bound_port:
|
|
3336
|
+
return False
|
|
3337
|
+
if hostname == "localhost":
|
|
3338
|
+
return bound_address.is_loopback
|
|
3339
|
+
try:
|
|
3340
|
+
requested = ipaddress.ip_address(hostname)
|
|
3341
|
+
except ValueError:
|
|
3342
|
+
return False
|
|
3343
|
+
return requested.is_loopback and requested == bound_address
|
|
3344
|
+
|
|
3345
|
+
host = self.headers.get("Host")
|
|
3346
|
+
if host is None or not local_authority(host, origin=False):
|
|
3347
|
+
return False
|
|
3348
|
+
origin = self.headers.get("Origin")
|
|
3349
|
+
return origin is None or local_authority(origin, origin=True)
|
|
3350
|
+
|
|
3351
|
+
def _write_html(self, status: int, markup: str) -> None:
|
|
3352
|
+
body = markup.encode("utf-8")
|
|
3353
|
+
self.send_response(status)
|
|
3354
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
3355
|
+
self.send_header("Content-Length", str(len(body)))
|
|
3356
|
+
self.send_header("Cache-Control", "no-store")
|
|
3357
|
+
self.end_headers()
|
|
3358
|
+
self.wfile.write(body)
|
|
3359
|
+
|
|
3360
|
+
def _write_json(self, status: int, value: object) -> None:
|
|
3361
|
+
body = json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
3362
|
+
self.send_response(status)
|
|
3363
|
+
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
3364
|
+
self.send_header("Content-Length", str(len(body)))
|
|
3365
|
+
self.send_header("Cache-Control", "no-store")
|
|
3366
|
+
self.end_headers()
|
|
3367
|
+
self.wfile.write(body)
|
|
3368
|
+
|
|
3369
|
+
@staticmethod
|
|
3370
|
+
def _requested(query: dict[str, list[str]]) -> str:
|
|
3371
|
+
value = query.get("doc", ["table"])[0]
|
|
3372
|
+
return value if value in _DOCUMENTS else "table"
|
|
3373
|
+
|
|
3374
|
+
@staticmethod
|
|
3375
|
+
def _version(query: dict[str, list[str]]) -> int | None:
|
|
3376
|
+
if "version" not in query:
|
|
3377
|
+
return None
|
|
3378
|
+
try:
|
|
3379
|
+
value = int(query["version"][0])
|
|
3380
|
+
except (IndexError, TypeError, ValueError) as exc:
|
|
3381
|
+
raise KeyError("malformed snapshot version") from exc
|
|
3382
|
+
if value < 0:
|
|
3383
|
+
raise KeyError(value)
|
|
3384
|
+
return value
|
|
3385
|
+
|
|
3386
|
+
def _serve_index(self, query: dict[str, list[str]]) -> None:
|
|
3387
|
+
entry, health, lifecycle = self._state.presentation()
|
|
3388
|
+
shell, active, docs = _render_snapshot(
|
|
3389
|
+
entry,
|
|
3390
|
+
self._requested(query),
|
|
3391
|
+
allow_table=bool(lifecycle.get("handoff")),
|
|
3392
|
+
)
|
|
3393
|
+
markup = _workbench_document(
|
|
3394
|
+
shell,
|
|
3395
|
+
run_name=self._state.run_dir.name,
|
|
3396
|
+
active=active,
|
|
3397
|
+
docs=docs,
|
|
3398
|
+
version=entry.version,
|
|
3399
|
+
epoch=self._state.epoch,
|
|
3400
|
+
health=health,
|
|
3401
|
+
visual_cursor=entry.visual_cursor,
|
|
3402
|
+
lifecycle=lifecycle,
|
|
3403
|
+
)
|
|
3404
|
+
self._write_html(200, markup)
|
|
3405
|
+
|
|
3406
|
+
def _serve_snapshot(self, query: dict[str, list[str]]) -> None:
|
|
3407
|
+
try:
|
|
3408
|
+
version = self._version(query)
|
|
3409
|
+
cursor_values = query.get("cursor")
|
|
3410
|
+
if cursor_values is not None:
|
|
3411
|
+
if version is not None or len(cursor_values) != 1:
|
|
3412
|
+
raise KeyError("snapshot accepts one coordinate")
|
|
3413
|
+
cursor = int(cursor_values[0])
|
|
3414
|
+
entry, health, lifecycle = self._state.presentation_cursor(cursor)
|
|
3415
|
+
else:
|
|
3416
|
+
entry, health, lifecycle = self._state.presentation(version)
|
|
3417
|
+
except (KeyError, ValueError):
|
|
3418
|
+
self.send_error(404, "snapshot version does not exist")
|
|
3419
|
+
return
|
|
3420
|
+
except LookupError:
|
|
3421
|
+
self.send_error(410, "snapshot version is no longer retained")
|
|
3422
|
+
return
|
|
3423
|
+
shell, active, docs = _render_snapshot(
|
|
3424
|
+
entry,
|
|
3425
|
+
self._requested(query),
|
|
3426
|
+
allow_table=bool(lifecycle.get("handoff")),
|
|
3427
|
+
)
|
|
3428
|
+
self._write_json(
|
|
3429
|
+
200,
|
|
3430
|
+
{
|
|
3431
|
+
"epoch": self._state.epoch,
|
|
3432
|
+
"version": entry.version,
|
|
3433
|
+
"cursor": entry.visual_cursor,
|
|
3434
|
+
"doc": active,
|
|
3435
|
+
"docs": docs,
|
|
3436
|
+
"html": shell,
|
|
3437
|
+
"mutation": (
|
|
3438
|
+
{key: value for key, value in entry.mutation.items() if key != "phase_start"}
|
|
3439
|
+
if entry.mutation is not None
|
|
3440
|
+
else None
|
|
3441
|
+
),
|
|
3442
|
+
"health": health,
|
|
3443
|
+
"lifecycle": lifecycle,
|
|
3444
|
+
},
|
|
3445
|
+
)
|
|
3446
|
+
|
|
3447
|
+
def _serve_table(self) -> None:
|
|
3448
|
+
candidate_digest = self._state.handoff_candidate_digest()
|
|
3449
|
+
if candidate_digest is None:
|
|
3450
|
+
self.send_error(409, "table handoff is not complete")
|
|
3451
|
+
return
|
|
3452
|
+
try:
|
|
3453
|
+
source, size = _verified_table_snapshot(
|
|
3454
|
+
self._state,
|
|
3455
|
+
expected_candidate_digest=candidate_digest,
|
|
3456
|
+
)
|
|
3457
|
+
except _TableUnavailable:
|
|
3458
|
+
status = 404 if not (self._state.run_dir / "candidate").exists() else 409
|
|
3459
|
+
self.send_error(status, "table is not ready as a verified snapshot")
|
|
3460
|
+
return
|
|
3461
|
+
with source:
|
|
3462
|
+
self.send_response(200)
|
|
3463
|
+
self.send_header("Content-Type", "application/vnd.apache.parquet")
|
|
3464
|
+
self.send_header("Content-Disposition", 'attachment; filename="table.parquet"')
|
|
3465
|
+
self.send_header("Content-Length", str(size))
|
|
3466
|
+
self.send_header("Cache-Control", "no-store")
|
|
3467
|
+
self.end_headers()
|
|
3468
|
+
while True:
|
|
3469
|
+
chunk = source.read(_READ_CHUNK_BYTES)
|
|
3470
|
+
if not chunk:
|
|
3471
|
+
break
|
|
3472
|
+
self.wfile.write(chunk)
|
|
3473
|
+
|
|
3474
|
+
def _serve_rows(self, query: dict[str, list[str]]) -> None:
|
|
3475
|
+
candidate_digest = self._state.handoff_candidate_digest()
|
|
3476
|
+
if candidate_digest is None:
|
|
3477
|
+
self.send_error(409, "table handoff is not complete")
|
|
3478
|
+
return
|
|
3479
|
+
try:
|
|
3480
|
+
parquet_raw, evidence = _verified_query_snapshot(
|
|
3481
|
+
self._state,
|
|
3482
|
+
expected_candidate_digest=candidate_digest,
|
|
3483
|
+
)
|
|
3484
|
+
columns = tuple(
|
|
3485
|
+
item for value in query.get("columns", []) for item in value.split(",") if item
|
|
3486
|
+
)
|
|
3487
|
+
if not columns:
|
|
3488
|
+
columns = sealed_parquet_columns(parquet_raw, evidence=evidence)
|
|
3489
|
+
filters_value = json.loads(query.get("filters", ["[]"])[0])
|
|
3490
|
+
if not isinstance(filters_value, list):
|
|
3491
|
+
raise QueryError("QUERY_FILTER", "filters must be an array")
|
|
3492
|
+
payload = query_sealed_parquet(
|
|
3493
|
+
parquet_raw,
|
|
3494
|
+
columns=columns,
|
|
3495
|
+
predicates=filters_value,
|
|
3496
|
+
offset=int(query.get("offset", ["0"])[0]),
|
|
3497
|
+
limit=int(query.get("limit", ["50"])[0]),
|
|
3498
|
+
evidence=evidence,
|
|
3499
|
+
)
|
|
3500
|
+
except _TableUnavailable:
|
|
3501
|
+
self.send_error(409, "table changed while opening its verified snapshot")
|
|
3502
|
+
return
|
|
3503
|
+
except (QueryError, ValueError, json.JSONDecodeError) as exc:
|
|
3504
|
+
self.send_error(400, str(exc))
|
|
3505
|
+
return
|
|
3506
|
+
self._write_json(200, payload)
|
|
3507
|
+
|
|
3508
|
+
def _serve_events(self) -> None:
|
|
3509
|
+
if self._state._visual_transport is not None:
|
|
3510
|
+
self._serve_visual_events()
|
|
3511
|
+
return
|
|
3512
|
+
self.send_response(200)
|
|
3513
|
+
self.send_header("Content-Type", "text/event-stream")
|
|
3514
|
+
self.send_header("Cache-Control", "no-cache")
|
|
3515
|
+
self.send_header("Connection", "keep-alive")
|
|
3516
|
+
self.end_headers()
|
|
3517
|
+
server = self.server
|
|
3518
|
+
server.register_stream(self.connection) # type: ignore[attr-defined]
|
|
3519
|
+
last: tuple[object, ...] | None = None
|
|
3520
|
+
sent_version: int | None = None
|
|
3521
|
+
sent_documents: dict[str, bytes] | None = None
|
|
3522
|
+
try:
|
|
3523
|
+
while not getattr(self.server, "viewer_stopped", False):
|
|
3524
|
+
# The viewer is intentionally allowed to open before an agent creates the
|
|
3525
|
+
# durable visual-run log. Discover it from this live request as well as from the
|
|
3526
|
+
# watcher, then continue on the same EventSource connection. Closing the legacy
|
|
3527
|
+
# response and relying on browser reconnection left already-open pages stale until
|
|
3528
|
+
# refresh when the first visual event appeared during unrelated watcher work.
|
|
3529
|
+
self._state._refresh_visual_notebook()
|
|
3530
|
+
transport = self._state._visual_transport
|
|
3531
|
+
if transport is not None:
|
|
3532
|
+
self._stream_visual_events(transport, cursor=0)
|
|
3533
|
+
return
|
|
3534
|
+
notice, current = self._state.wait_for_notice(last, _POLL_SECONDS)
|
|
3535
|
+
transport = self._state._visual_transport
|
|
3536
|
+
if transport is not None:
|
|
3537
|
+
self._stream_visual_events(transport, cursor=0)
|
|
3538
|
+
return
|
|
3539
|
+
if current != last:
|
|
3540
|
+
last = current
|
|
3541
|
+
current_version = int(notice["version"])
|
|
3542
|
+
versions = [current_version]
|
|
3543
|
+
if sent_version is not None and current_version > sent_version:
|
|
3544
|
+
retained = self._state.versions()
|
|
3545
|
+
versions = [
|
|
3546
|
+
version
|
|
3547
|
+
for version in retained
|
|
3548
|
+
if sent_version < version <= current_version
|
|
3549
|
+
]
|
|
3550
|
+
for version in versions:
|
|
3551
|
+
versioned = dict(notice)
|
|
3552
|
+
versioned["version"] = version
|
|
3553
|
+
try:
|
|
3554
|
+
entry = self._state.snapshot(version)
|
|
3555
|
+
except (KeyError, LookupError):
|
|
3556
|
+
continue
|
|
3557
|
+
versioned["docs"] = list(entry.documents)
|
|
3558
|
+
versioned["changed_docs"] = [
|
|
3559
|
+
name
|
|
3560
|
+
for name in _DOCUMENTS
|
|
3561
|
+
if sent_documents is None
|
|
3562
|
+
or sent_documents.get(name) != entry.documents.get(name)
|
|
3563
|
+
]
|
|
3564
|
+
payload = json.dumps(versioned, separators=(",", ":"))
|
|
3565
|
+
self.wfile.write(f"data: {payload}\n\n".encode())
|
|
3566
|
+
sent_documents = entry.documents
|
|
3567
|
+
sent_version = current_version
|
|
3568
|
+
else:
|
|
3569
|
+
self.wfile.write(b": ping\n\n")
|
|
3570
|
+
self.wfile.flush()
|
|
3571
|
+
time.sleep(_POLL_SECONDS)
|
|
3572
|
+
except (BrokenPipeError, ConnectionResetError, OSError, VisualRunError):
|
|
3573
|
+
return
|
|
3574
|
+
finally:
|
|
3575
|
+
server.unregister_stream(self.connection) # type: ignore[attr-defined]
|
|
3576
|
+
|
|
3577
|
+
def _serve_visual_events(self) -> None:
|
|
3578
|
+
transport = self._state._visual_transport
|
|
3579
|
+
assert transport is not None
|
|
3580
|
+
try:
|
|
3581
|
+
cursor = parse_last_event_id(self.headers.get("Last-Event-ID"))
|
|
3582
|
+
transport.events_after(cursor, limit=1)
|
|
3583
|
+
except VisualRunError:
|
|
3584
|
+
self.send_error(409, "visual event cursor is invalid")
|
|
3585
|
+
return
|
|
3586
|
+
self.send_response(200)
|
|
3587
|
+
self.send_header("Content-Type", "text/event-stream")
|
|
3588
|
+
self.send_header("Cache-Control", "no-cache")
|
|
3589
|
+
self.send_header("Connection", "keep-alive")
|
|
3590
|
+
self.end_headers()
|
|
3591
|
+
server = self.server
|
|
3592
|
+
server.register_stream(self.connection) # type: ignore[attr-defined]
|
|
3593
|
+
try:
|
|
3594
|
+
self._stream_visual_events(transport, cursor=cursor)
|
|
3595
|
+
except (BrokenPipeError, ConnectionResetError, OSError, VisualRunError):
|
|
3596
|
+
return
|
|
3597
|
+
finally:
|
|
3598
|
+
server.unregister_stream(self.connection) # type: ignore[attr-defined]
|
|
3599
|
+
|
|
3600
|
+
def _stream_visual_events(self, transport: VisualRunTransport, *, cursor: int) -> None:
|
|
3601
|
+
"""Multiplex durable visual cursors and ordinary viewer state on one live SSE."""
|
|
3602
|
+
|
|
3603
|
+
server = self.server
|
|
3604
|
+
_, last_notice = self._state.wait_for_notice(None, 0)
|
|
3605
|
+
sent_documents = self._state.snapshot().documents
|
|
3606
|
+
while not getattr(server, "viewer_stopped", False):
|
|
3607
|
+
try:
|
|
3608
|
+
page = transport.events_after(cursor, limit=1)
|
|
3609
|
+
except VisualRunError:
|
|
3610
|
+
self._state.poll(refresh_operational=False)
|
|
3611
|
+
self._write_visual_health_notice()
|
|
3612
|
+
return
|
|
3613
|
+
if page.events:
|
|
3614
|
+
# Admit the corresponding reduced state before publishing its cursor. The
|
|
3615
|
+
# browser can therefore request /snapshot?cursor=N without racing the watcher.
|
|
3616
|
+
self._state.poll(refresh_operational=False)
|
|
3617
|
+
for event in page.events:
|
|
3618
|
+
if not self._state.visual_cursor_is_admitted(event.sequence):
|
|
3619
|
+
# Another request may currently own the non-blocking projection lock.
|
|
3620
|
+
# Keep the durable cursor unannounced until that projection finishes.
|
|
3621
|
+
if self._state.health["status"] == "stopped":
|
|
3622
|
+
self._write_visual_health_notice()
|
|
3623
|
+
return
|
|
3624
|
+
self.wfile.write(b": waiting for visual projection\n\n")
|
|
3625
|
+
break
|
|
3626
|
+
cursor = event.sequence
|
|
3627
|
+
payload = json.dumps(
|
|
3628
|
+
{"cursor": cursor, "visual_event": event.to_mapping()},
|
|
3629
|
+
separators=(",", ":"),
|
|
3630
|
+
)
|
|
3631
|
+
self.wfile.write(f"id: {cursor}\ndata: {payload}\n\n".encode())
|
|
3632
|
+
notice, current_notice = self._state.wait_for_notice(last_notice, 0)
|
|
3633
|
+
notice_cursor = int(notice.get("cursor", 0))
|
|
3634
|
+
if current_notice != last_notice and notice_cursor <= cursor:
|
|
3635
|
+
documents = self._state.snapshot().documents
|
|
3636
|
+
notice["changed_docs"] = [
|
|
3637
|
+
name for name in _DOCUMENTS if sent_documents.get(name) != documents.get(name)
|
|
3638
|
+
]
|
|
3639
|
+
payload = json.dumps(notice, separators=(",", ":"))
|
|
3640
|
+
self.wfile.write(f"data: {payload}\n\n".encode())
|
|
3641
|
+
sent_documents = documents
|
|
3642
|
+
last_notice = current_notice
|
|
3643
|
+
elif not page.events:
|
|
3644
|
+
self.wfile.write(b": ping\n\n")
|
|
3645
|
+
self.wfile.flush()
|
|
3646
|
+
time.sleep(_POLL_SECONDS)
|
|
3647
|
+
|
|
3648
|
+
def _write_visual_health_notice(self) -> None:
|
|
3649
|
+
"""Tell an already-open visual client why projection stopped before closing SSE."""
|
|
3650
|
+
|
|
3651
|
+
notice = self._state.notice()
|
|
3652
|
+
payload = json.dumps(notice, separators=(",", ":"))
|
|
3653
|
+
self.wfile.write(f"data: {payload}\n\n".encode())
|
|
3654
|
+
self.wfile.flush()
|
|
3655
|
+
|
|
3656
|
+
|
|
3657
|
+
def _watch_loop(server: _ViewerServer) -> None:
|
|
3658
|
+
try:
|
|
3659
|
+
while not server.viewer_stopped:
|
|
3660
|
+
server.viewer_state.poll()
|
|
3661
|
+
time.sleep(_WATCH_POLL_SECONDS)
|
|
3662
|
+
except Exception:
|
|
3663
|
+
server.viewer_state.mark_watcher_failed()
|
|
3664
|
+
|
|
3665
|
+
|
|
3666
|
+
def _build_server(target: ViewerTarget | Path, host: str, port: int) -> tuple[_ViewerServer, str]:
|
|
3667
|
+
address = ipaddress.ip_address(host)
|
|
3668
|
+
if not address.is_loopback:
|
|
3669
|
+
raise ValueError("mr-data view must bind to a loopback address")
|
|
3670
|
+
state = _ViewerState(target)
|
|
3671
|
+
server_type = _ViewerServerV6 if address.version == 6 else _ViewerServer
|
|
3672
|
+
# ``TCPServer.__init__`` calls ``server_close`` if binding fails. The server receives ownership
|
|
3673
|
+
# of ``state`` before its base initializer starts, so that partial-construction cleanup closes
|
|
3674
|
+
# the retained descriptors without replacing the original socket error.
|
|
3675
|
+
server = server_type((address.compressed, port), _ViewerHandler, state)
|
|
3676
|
+
bound_host = str(server.server_address[0])
|
|
3677
|
+
url_host = f"[{bound_host}]" if address.version == 6 else bound_host
|
|
3678
|
+
url = f"http://{url_host}:{server.server_address[1]}/"
|
|
3679
|
+
server.viewer_watcher = threading.Thread(target=_watch_loop, args=(server,), daemon=True)
|
|
3680
|
+
server.viewer_watcher.start()
|
|
3681
|
+
return server, url
|
|
3682
|
+
|
|
3683
|
+
|
|
3684
|
+
def serve_notebook(
|
|
3685
|
+
target: ViewerTarget | Path,
|
|
3686
|
+
*,
|
|
3687
|
+
host: str = "127.0.0.1",
|
|
3688
|
+
port: int = 0,
|
|
3689
|
+
open_browser: bool = False,
|
|
3690
|
+
announce: Callable[[str], None] | None = None,
|
|
3691
|
+
) -> None:
|
|
3692
|
+
"""Serve a local run's notebooks as a streaming, read-only visual workbench."""
|
|
3693
|
+
|
|
3694
|
+
server, url = _build_server(target, host, port)
|
|
3695
|
+
if announce is None:
|
|
3696
|
+
print(f"mr-data view: serving {url} (Ctrl-C to stop)", flush=True)
|
|
3697
|
+
else:
|
|
3698
|
+
announce(url)
|
|
3699
|
+
if open_browser:
|
|
3700
|
+
webbrowser.open(url)
|
|
3701
|
+
try:
|
|
3702
|
+
server.serve_forever()
|
|
3703
|
+
except KeyboardInterrupt:
|
|
3704
|
+
pass
|
|
3705
|
+
finally:
|
|
3706
|
+
server.viewer_stopped = True
|
|
3707
|
+
server.close_streams()
|
|
3708
|
+
if server.viewer_watcher is not None:
|
|
3709
|
+
# The watcher reads notebooks through descriptors ``server_close`` releases with the
|
|
3710
|
+
# state, so it is given a moment to see the stop flag first. A watcher still inside a
|
|
3711
|
+
# long verification is left to the daemon flag rather than waited on.
|
|
3712
|
+
server.viewer_watcher.join(timeout=_WATCHER_STOP_SECONDS)
|
|
3713
|
+
server.server_close()
|