mostlyright-data 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mostlyright/data_harness/__init__.py +158 -0
- mostlyright/data_harness/acquisition/__init__.py +55 -0
- mostlyright/data_harness/acquisition/http.py +2773 -0
- mostlyright/data_harness/acquisition/parsing.py +809 -0
- mostlyright/data_harness/acquisition/ranges.py +495 -0
- mostlyright/data_harness/acquisition/result_download.py +360 -0
- mostlyright/data_harness/acquisition/retention_admission.py +248 -0
- mostlyright/data_harness/acquisition/sandbox.py +4888 -0
- mostlyright/data_harness/acquisition/url_policy.py +530 -0
- mostlyright/data_harness/agent_runtime.py +2743 -0
- mostlyright/data_harness/assets/logo-ink.svg +31 -0
- mostlyright/data_harness/backends/__init__.py +28 -0
- mostlyright/data_harness/backends/pandas_backend.py +350 -0
- mostlyright/data_harness/backends/polars_backend.py +366 -0
- mostlyright/data_harness/backends/protocol.py +124 -0
- mostlyright/data_harness/backends/reference.py +83 -0
- mostlyright/data_harness/backends/registry.py +55 -0
- mostlyright/data_harness/backends/restrictions.py +126 -0
- mostlyright/data_harness/canonical.py +333 -0
- mostlyright/data_harness/catalog_job.py +625 -0
- mostlyright/data_harness/cli.py +5398 -0
- mostlyright/data_harness/contracts.py +53 -0
- mostlyright/data_harness/coordinator.py +1307 -0
- mostlyright/data_harness/deploy.py +924 -0
- mostlyright/data_harness/deploy_target.py +312 -0
- mostlyright/data_harness/deployment_evidence.py +1067 -0
- mostlyright/data_harness/event_presentation.py +576 -0
- mostlyright/data_harness/events.py +2152 -0
- mostlyright/data_harness/fast_delimited.py +239 -0
- mostlyright/data_harness/fleet.py +237 -0
- mostlyright/data_harness/formats.py +236 -0
- mostlyright/data_harness/governors.py +1163 -0
- mostlyright/data_harness/hosted_bootstrap.py +972 -0
- mostlyright/data_harness/hosted_crawler.py +1115 -0
- mostlyright/data_harness/hosted_crawler_container_smoke.py +351 -0
- mostlyright/data_harness/hosted_crawler_fetch.py +423 -0
- mostlyright/data_harness/hosted_crawler_job.py +1277 -0
- mostlyright/data_harness/hosted_crawler_protocol.py +676 -0
- mostlyright/data_harness/hosted_dataset.py +1500 -0
- mostlyright/data_harness/hosted_deploy.py +3037 -0
- mostlyright/data_harness/hosted_handoff.py +62 -0
- mostlyright/data_harness/hosted_ingestion_contract.py +504 -0
- mostlyright/data_harness/hosted_ingestion_job.py +356 -0
- mostlyright/data_harness/hosted_ingestion_job_smoke.py +40 -0
- mostlyright/data_harness/hosted_session_container_smoke.py +194 -0
- mostlyright/data_harness/hosted_session_worker.py +3554 -0
- mostlyright/data_harness/hosted_session_worker_job_smoke.py +46 -0
- mostlyright/data_harness/hosted_worker.py +6784 -0
- mostlyright/data_harness/ingestion/__init__.py +56 -0
- mostlyright/data_harness/ingestion/contracts.py +461 -0
- mostlyright/data_harness/ingestion/faults.py +42 -0
- mostlyright/data_harness/ingestion/gcs_store.py +1162 -0
- mostlyright/data_harness/ingestion/spool.py +130 -0
- mostlyright/data_harness/ingestion/store.py +885 -0
- mostlyright/data_harness/key_seam.py +434 -0
- mostlyright/data_harness/linux_process_boundary.py +262 -0
- mostlyright/data_harness/local_contracts.py +2880 -0
- mostlyright/data_harness/local_search/__init__.py +5 -0
- mostlyright/data_harness/local_search/build_index.py +1087 -0
- mostlyright/data_harness/local_search/contracts.py +920 -0
- mostlyright/data_harness/local_search/query_trace.py +266 -0
- mostlyright/data_harness/local_search/retrieval.py +700 -0
- mostlyright/data_harness/local_search/sealed.py +474 -0
- mostlyright/data_harness/local_search/service.py +784 -0
- mostlyright/data_harness/nbrender/CONTRACT.md +212 -0
- mostlyright/data_harness/nbrender/__init__.py +12 -0
- mostlyright/data_harness/nbrender/chrome.py +359 -0
- mostlyright/data_harness/nbrender/code_body.py +266 -0
- mostlyright/data_harness/nbrender/document.py +407 -0
- mostlyright/data_harness/nbrender/frame.py +275 -0
- mostlyright/data_harness/nbrender/interactive.py +337 -0
- mostlyright/data_harness/nbrender/markdown_body.py +477 -0
- mostlyright/data_harness/nbrender/mr_components.py +134 -0
- mostlyright/data_harness/nbrender/outputs_data.py +595 -0
- mostlyright/data_harness/nbrender/outputs_rich.py +906 -0
- mostlyright/data_harness/nbrender/outputs_source.py +260 -0
- mostlyright/data_harness/nbrender/outputs_stage.py +176 -0
- mostlyright/data_harness/nbrender/outputs_text.py +400 -0
- mostlyright/data_harness/nbrender/parse.py +394 -0
- mostlyright/data_harness/nbrender/status.py +40 -0
- mostlyright/data_harness/nbrender/tokens.py +1295 -0
- mostlyright/data_harness/notebook.py +1710 -0
- mostlyright/data_harness/offline.py +2049 -0
- mostlyright/data_harness/operation_registry.py +1007 -0
- mostlyright/data_harness/operator_setup.py +239 -0
- mostlyright/data_harness/pipeline.py +6428 -0
- mostlyright/data_harness/plan_graph.py +2026 -0
- mostlyright/data_harness/preparation/__init__.py +104 -0
- mostlyright/data_harness/preparation/contracts.py +1017 -0
- mostlyright/data_harness/preparation/engine.py +221 -0
- mostlyright/data_harness/preparation/errors.py +14 -0
- mostlyright/data_harness/preparation/gates.py +751 -0
- mostlyright/data_harness/preparation/joins.py +574 -0
- mostlyright/data_harness/preparation/profile.py +384 -0
- mostlyright/data_harness/preparation/table.py +217 -0
- mostlyright/data_harness/preparation/transforms.py +568 -0
- mostlyright/data_harness/progress_events.py +534 -0
- mostlyright/data_harness/readers/__init__.py +46 -0
- mostlyright/data_harness/readers/containers.py +963 -0
- mostlyright/data_harness/readers/contracts.py +542 -0
- mostlyright/data_harness/readers/delimited.py +257 -0
- mostlyright/data_harness/readers/grib2/__init__.py +33 -0
- mostlyright/data_harness/readers/grib2/admission.py +722 -0
- mostlyright/data_harness/readers/grib2/decode.py +1009 -0
- mostlyright/data_harness/readers/grib2/geometry.py +1133 -0
- mostlyright/data_harness/readers/grib2/portable_math.py +501 -0
- mostlyright/data_harness/readers/json_tabular.py +485 -0
- mostlyright/data_harness/readers/registry.py +514 -0
- mostlyright/data_harness/readers/samples/README.md +110 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.0.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/cities.csv.gz +0 -0
- mostlyright/data_harness/readers/samples/archive.gzip/1.1.0/cities_one_stream/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.0.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/cities.tar +0 -0
- mostlyright/data_harness/readers/samples/archive.tar/1.1.0/cities_beside_a_directory_entry/expected.json +24 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/cities.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.0.0/cities_beside_a_second_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.1.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/dwd-station.zip +0 -0
- mostlyright/data_harness/readers/samples/archive.zip/1.2.0/dwd_semicolon_station_member/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/cities.csv +3 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/an_ordinary_comma_separated_table/expected.json +23 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/cities.tsv +5 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.0.0/quoted_fields_holding_the_delimiter/expected.json +25 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/expected.json +30 -0
- mostlyright/data_harness/readers/samples/delimited_text/1.1.0/an_hourly_observation_table_served_as_plain_text/observations.csv +5 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/expected.json +44 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.0.0/nested_hourly_observations/stations.json +1 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/expected.json +48 -0
- mostlyright/data_harness/readers/samples/json.tabular/1.1.0/an_observation_stream_served_as_plain_text/observations.ndjson +4 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/cities.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/an_ordinary_table_beside_a_second_sheet/expected.json +24 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.0.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/expected.json +27 -0
- mostlyright/data_harness/readers/samples/spreadsheet.xlsx/1.1.0/shares_the_workbook_had_already_computed/shares.xlsx +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/README.md +20 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/expected.json +55 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/gfs_2m_temperature/gfs-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/1.0.0/hrrr_categorical_rain/hrrr-categorical-rain.grib2 +0 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/expected.json +54 -0
- mostlyright/data_harness/readers/samples/weather.grib2/2.0.0/hrrr_2m_temperature/hrrr-2m-temperature.grib2 +0 -0
- mostlyright/data_harness/readers/samples.py +582 -0
- mostlyright/data_harness/readers/spreadsheet.py +803 -0
- mostlyright/data_harness/readers/tabular.py +510 -0
- mostlyright/data_harness/recipe.py +5321 -0
- mostlyright/data_harness/repair/__init__.py +78 -0
- mostlyright/data_harness/repair/adapters.py +274 -0
- mostlyright/data_harness/repair/contracts.py +872 -0
- mostlyright/data_harness/repair/coordinator.py +1099 -0
- mostlyright/data_harness/repair/errors.py +16 -0
- mostlyright/data_harness/review.py +2533 -0
- mostlyright/data_harness/rowset.py +283 -0
- mostlyright/data_harness/serving.py +1975 -0
- mostlyright/data_harness/serving_edge.py +590 -0
- mostlyright/data_harness/serving_http.py +1031 -0
- mostlyright/data_harness/session_probes.py +759 -0
- mostlyright/data_harness/signing.py +101 -0
- mostlyright/data_harness/source_discovery.py +898 -0
- mostlyright/data_harness/sources/__init__.py +209 -0
- mostlyright/data_harness/sources/_adapter_steps.py +213 -0
- mostlyright/data_harness/sources/adapters.py +1214 -0
- mostlyright/data_harness/sources/cadence.py +1428 -0
- mostlyright/data_harness/sources/cadence_emission.py +453 -0
- mostlyright/data_harness/sources/cadence_history.py +546 -0
- mostlyright/data_harness/sources/catalog/__init__.py +17 -0
- mostlyright/data_harness/sources/catalog/admission.py +477 -0
- mostlyright/data_harness/sources/catalog/authoring.py +1701 -0
- mostlyright/data_harness/sources/catalog/authoring_policy.py +701 -0
- mostlyright/data_harness/sources/catalog/authoring_shards.py +1217 -0
- mostlyright/data_harness/sources/catalog/bounded_io.py +231 -0
- mostlyright/data_harness/sources/catalog/channel.py +523 -0
- mostlyright/data_harness/sources/catalog/channel_client.py +296 -0
- mostlyright/data_harness/sources/catalog/contracts.py +825 -0
- mostlyright/data_harness/sources/catalog/coverage.py +137 -0
- mostlyright/data_harness/sources/catalog/delta.py +1340 -0
- mostlyright/data_harness/sources/catalog/embedding.py +532 -0
- mostlyright/data_harness/sources/catalog/entry_v2.py +1182 -0
- mostlyright/data_harness/sources/catalog/fill.py +3889 -0
- mostlyright/data_harness/sources/catalog/fill_partitions.py +459 -0
- mostlyright/data_harness/sources/catalog/fill_staging.py +1105 -0
- mostlyright/data_harness/sources/catalog/gating.py +374 -0
- mostlyright/data_harness/sources/catalog/generation_receipt.py +1607 -0
- mostlyright/data_harness/sources/catalog/harvest/__init__.py +7 -0
- mostlyright/data_harness/sources/catalog/harvest/ckan.py +384 -0
- mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py +798 -0
- mostlyright/data_harness/sources/catalog/harvest/protocol.py +964 -0
- mostlyright/data_harness/sources/catalog/harvest/sdmx.py +445 -0
- mostlyright/data_harness/sources/catalog/harvest/stac.py +384 -0
- mostlyright/data_harness/sources/catalog/health.py +447 -0
- mostlyright/data_harness/sources/catalog/hosted_catalog.py +105 -0
- mostlyright/data_harness/sources/catalog/identity_history.py +1549 -0
- mostlyright/data_harness/sources/catalog/neural.py +1618 -0
- mostlyright/data_harness/sources/catalog/packed_catalog.py +2345 -0
- mostlyright/data_harness/sources/catalog/packed_retrieval.py +1517 -0
- mostlyright/data_harness/sources/catalog/packed_writer.py +2802 -0
- mostlyright/data_harness/sources/catalog/query_trace.py +1037 -0
- mostlyright/data_harness/sources/catalog/recommend.py +171 -0
- mostlyright/data_harness/sources/catalog/retrieval.py +230 -0
- mostlyright/data_harness/sources/catalog/retrieval_manifest.py +995 -0
- mostlyright/data_harness/sources/catalog/rights_decisions.py +254 -0
- mostlyright/data_harness/sources/catalog/sealed.py +560 -0
- mostlyright/data_harness/sources/catalog/search.py +230 -0
- mostlyright/data_harness/sources/catalog/streaming_delta.py +1097 -0
- mostlyright/data_harness/sources/catalog/update.py +891 -0
- mostlyright/data_harness/sources/collections.py +815 -0
- mostlyright/data_harness/sources/contracts.py +2223 -0
- mostlyright/data_harness/sources/deletion.py +761 -0
- mostlyright/data_harness/sources/fitness.py +162 -0
- mostlyright/data_harness/sources/governance.py +163 -0
- mostlyright/data_harness/sources/hosted.py +173 -0
- mostlyright/data_harness/sources/integration.py +218 -0
- mostlyright/data_harness/sources/range_reader.py +418 -0
- mostlyright/data_harness/sources/registry.py +514 -0
- mostlyright/data_harness/sources/rights_rule.py +59 -0
- mostlyright/data_harness/sources/source_cadence_vectors.v1.json +1 -0
- mostlyright/data_harness/sources/sports.py +521 -0
- mostlyright/data_harness/sources/stream.py +524 -0
- mostlyright/data_harness/sources/stream_connector.py +418 -0
- mostlyright/data_harness/sources/stream_recorder.py +1404 -0
- mostlyright/data_harness/studio_boundary.py +2019 -0
- mostlyright/data_harness/thin/__init__.py +37 -0
- mostlyright/data_harness/thin/acquire.py +1137 -0
- mostlyright/data_harness/thin/acquire_cancel.py +579 -0
- mostlyright/data_harness/thin/approvals.py +617 -0
- mostlyright/data_harness/thin/commands.py +406 -0
- mostlyright/data_harness/thin/download.py +194 -0
- mostlyright/data_harness/thin/narrative.py +589 -0
- mostlyright/data_harness/thin/parity.py +1070 -0
- mostlyright/data_harness/thin/propose.py +2759 -0
- mostlyright/data_harness/thin/research.py +1663 -0
- mostlyright/data_harness/thin/router.py +924 -0
- mostlyright/data_harness/thin/runs.py +519 -0
- mostlyright/data_harness/thin/session.py +281 -0
- mostlyright/data_harness/thin/stream.py +501 -0
- mostlyright/data_harness/thin/transport.py +187 -0
- mostlyright/data_harness/thin/vocabulary.py +368 -0
- mostlyright/data_harness/thin/workers.py +164 -0
- mostlyright/data_harness/ucum/TABLE-PIN.json +40 -0
- mostlyright/data_harness/ucum/ucum-subset.v1.json +632 -0
- mostlyright/data_harness/unit_flow.py +927 -0
- mostlyright/data_harness/units.py +572 -0
- mostlyright/data_harness/ux/__init__.py +9 -0
- mostlyright/data_harness/ux/approve.py +485 -0
- mostlyright/data_harness/ux/author_yaml.py +597 -0
- mostlyright/data_harness/ux/cloud_auth.py +447 -0
- mostlyright/data_harness/ux/commands/__init__.py +260 -0
- mostlyright/data_harness/ux/commands/approve.py +136 -0
- mostlyright/data_harness/ux/commands/auth.py +744 -0
- mostlyright/data_harness/ux/commands/author.py +79 -0
- mostlyright/data_harness/ux/commands/catalog_author.py +403 -0
- mostlyright/data_harness/ux/commands/catalog_fill.py +523 -0
- mostlyright/data_harness/ux/commands/catalog_harvest.py +545 -0
- mostlyright/data_harness/ux/commands/catalog_publish.py +1838 -0
- mostlyright/data_harness/ux/commands/catalog_search.py +71 -0
- mostlyright/data_harness/ux/commands/catalog_update.py +437 -0
- mostlyright/data_harness/ux/commands/deploy.py +134 -0
- mostlyright/data_harness/ux/commands/deploy_dataset.py +98 -0
- mostlyright/data_harness/ux/commands/deploy_plan.py +105 -0
- mostlyright/data_harness/ux/commands/deploy_status.py +104 -0
- mostlyright/data_harness/ux/commands/diff.py +74 -0
- mostlyright/data_harness/ux/commands/index.py +84 -0
- mostlyright/data_harness/ux/commands/inventory.py +47 -0
- mostlyright/data_harness/ux/commands/list_builds.py +143 -0
- mostlyright/data_harness/ux/commands/login.py +63 -0
- mostlyright/data_harness/ux/commands/peek.py +236 -0
- mostlyright/data_harness/ux/commands/plan_check.py +90 -0
- mostlyright/data_harness/ux/commands/preflight.py +97 -0
- mostlyright/data_harness/ux/commands/record.py +107 -0
- mostlyright/data_harness/ux/commands/review_setup.py +47 -0
- mostlyright/data_harness/ux/commands/search.py +440 -0
- mostlyright/data_harness/ux/commands/show.py +61 -0
- mostlyright/data_harness/ux/commands/whoami.py +37 -0
- mostlyright/data_harness/ux/credential_native.py +551 -0
- mostlyright/data_harness/ux/credential_store.py +1055 -0
- mostlyright/data_harness/ux/credentials.py +631 -0
- mostlyright/data_harness/ux/diffing.py +444 -0
- mostlyright/data_harness/ux/headline.py +671 -0
- mostlyright/data_harness/ux/hosted_acquisition.py +974 -0
- mostlyright/data_harness/ux/hosted_run_status.py +619 -0
- mostlyright/data_harness/ux/inventory.py +427 -0
- mostlyright/data_harness/ux/local_review.py +375 -0
- mostlyright/data_harness/ux/login.py +691 -0
- mostlyright/data_harness/ux/path_kind.py +147 -0
- mostlyright/data_harness/ux/peek.py +1000 -0
- mostlyright/data_harness/ux/plain_file.py +178 -0
- mostlyright/data_harness/ux/plan_check.py +311 -0
- mostlyright/data_harness/ux/preflight.py +918 -0
- mostlyright/data_harness/ux/readers.py +1124 -0
- mostlyright/data_harness/ux/remediation.py +2195 -0
- mostlyright/data_harness/ux/render.py +657 -0
- mostlyright/data_harness/ux/workload.py +1077 -0
- mostlyright/data_harness/viewer.py +3713 -0
- mostlyright/data_harness/visual_run/__init__.py +83 -0
- mostlyright/data_harness/visual_run/authoring.py +235 -0
- mostlyright/data_harness/visual_run/contracts.py +673 -0
- mostlyright/data_harness/visual_run/materialize.py +486 -0
- mostlyright/data_harness/visual_run/observations.py +874 -0
- mostlyright/data_harness/visual_run/query.py +259 -0
- mostlyright/data_harness/visual_run/reducer.py +280 -0
- mostlyright/data_harness/visual_run/sdk.py +892 -0
- mostlyright/data_harness/visual_run/store.py +584 -0
- mostlyright/data_harness/visual_run/transport.py +239 -0
- mostlyright/data_harness/watch.py +2999 -0
- mostlyright_data-0.9.0.dist-info/METADATA +607 -0
- mostlyright_data-0.9.0.dist-info/RECORD +314 -0
- mostlyright_data-0.9.0.dist-info/WHEEL +4 -0
- mostlyright_data-0.9.0.dist-info/entry_points.txt +12 -0
|
@@ -0,0 +1,1404 @@
|
|
|
1
|
+
"""Deterministic stream capture: seal recorded messages, then read them back as batches.
|
|
2
|
+
|
|
3
|
+
Capture and build stay separate. A live socket can never satisfy the determinism contract --
|
|
4
|
+
a Checker cannot replay a subscription -- so a connection is a *capture* role, not an
|
|
5
|
+
acquisition transport. This module is the half of that split which is deterministic, and it is
|
|
6
|
+
the whole of what this build ships:
|
|
7
|
+
|
|
8
|
+
* :func:`seal_recorded_capture` takes an already-recorded list of provider messages and turns
|
|
9
|
+
it into bounded, sealed, ordered batch files plus a capture manifest. Epochs, rotation,
|
|
10
|
+
message classification, event identity, and gap boundaries are all decided here, from the
|
|
11
|
+
connector configuration, with no clock and no connection.
|
|
12
|
+
* :class:`RecordedStreamGateway` implements the one-method ``StreamGateway`` the existing
|
|
13
|
+
``StreamBatchAdapter`` already consumes, reading those sealed batches back off disk.
|
|
14
|
+
|
|
15
|
+
What is deliberately absent is the socket. Nothing in this module opens, resolves, or reads a
|
|
16
|
+
network address; network access belongs to the Courier, and a Recorder that holds a long-lived
|
|
17
|
+
connection is a separate trusted role that does not exist yet. The recording this module reads
|
|
18
|
+
is therefore an input the operator supplies, not something this build goes and gets. Everything
|
|
19
|
+
downstream of the recording -- rotation, sealing, identity, ordering, the bookmark -- is the
|
|
20
|
+
real behaviour, exercised on real bytes.
|
|
21
|
+
|
|
22
|
+
The recording format is one JSON object per line, each with exactly three keys::
|
|
23
|
+
|
|
24
|
+
{"epoch": 1, "received_at": "2026-07-17T12:00:00Z", "message": {...}}
|
|
25
|
+
|
|
26
|
+
``epoch`` is the connection the message arrived on. A disconnect closes an epoch and the next
|
|
27
|
+
message carries the next epoch number; that boundary is a gap and is recorded as one. Gaps stay
|
|
28
|
+
loud: nothing here silently splices two connections together.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import os
|
|
34
|
+
import re
|
|
35
|
+
import stat
|
|
36
|
+
from dataclasses import dataclass
|
|
37
|
+
from datetime import datetime
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from mostlyright.data_harness.acquisition.http import seal_content_addressed_snapshot
|
|
42
|
+
from mostlyright.data_harness.canonical import (
|
|
43
|
+
canonical_json_bytes,
|
|
44
|
+
canonical_sha256,
|
|
45
|
+
parse_json,
|
|
46
|
+
sha256_bytes,
|
|
47
|
+
)
|
|
48
|
+
from mostlyright.data_harness.sources.contracts import (
|
|
49
|
+
AcquisitionRequest,
|
|
50
|
+
SourceContractError,
|
|
51
|
+
)
|
|
52
|
+
from mostlyright.data_harness.sources.stream import (
|
|
53
|
+
EVENT_ID_GRAMMAR,
|
|
54
|
+
MAX_BATCH_EVENTS,
|
|
55
|
+
StreamBatch,
|
|
56
|
+
)
|
|
57
|
+
from mostlyright.data_harness.sources.stream_connector import (
|
|
58
|
+
EVENT_TIME_KINDS,
|
|
59
|
+
StreamConnectorConfig,
|
|
60
|
+
stream_connector,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
STREAM_CAPTURE_VERSION = "harness-stream-capture.v1"
|
|
64
|
+
|
|
65
|
+
CAPTURE_MANIFEST_NAME = "capture.json"
|
|
66
|
+
BATCH_DIRECTORY_NAME = "batches"
|
|
67
|
+
# The durable per-source bookmark ledger lives beside the batches it is about, so one capture
|
|
68
|
+
# directory is the whole replayable unit. ``stream_connector.RESERVED_STREAM_IDS`` keeps a
|
|
69
|
+
# stream from claiming this name.
|
|
70
|
+
LEDGER_DIRECTORY_NAME = "bookmarks"
|
|
71
|
+
|
|
72
|
+
MAX_RECORDING_BYTES = 64 * 1024 * 1024
|
|
73
|
+
MAX_MANIFEST_BYTES = 64 * 1024 * 1024
|
|
74
|
+
MAX_RECORDED_FRAMES = 200_000
|
|
75
|
+
MAX_EPOCHS = 4_096
|
|
76
|
+
MAX_BATCHES = 4_096
|
|
77
|
+
|
|
78
|
+
_TIMESTAMP = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z")
|
|
79
|
+
_STORAGE_NAME = re.compile(r"[0-9a-f]{64}\.(?:json|ndjson)")
|
|
80
|
+
# The bookmark grammar: the Recorder's own ``{epoch}:{counter}`` position, zero-padded so that
|
|
81
|
+
# text order and number order are the same thing. It is deliberately not the event name -- a
|
|
82
|
+
# venue that assigns its own identifiers assigns ``8``, ``9``, ``10``, which do not sort as
|
|
83
|
+
# text, and a bookmark that cannot be compared is not a bookmark.
|
|
84
|
+
_BOOKMARK = re.compile(r"e([0-9]{8}):([0-9]{12})")
|
|
85
|
+
_RECORDING_KEYS = frozenset({"epoch", "received_at", "message"})
|
|
86
|
+
|
|
87
|
+
_MEDIA_TYPES = {"ndjson": "application/x-ndjson", "json": "application/json"}
|
|
88
|
+
|
|
89
|
+
# A gap boundary is one of exactly these. ``opening`` is the first connection of a capture and
|
|
90
|
+
# is not a gap at all; the other two say whether the venue's own repair rule closed the hole.
|
|
91
|
+
GAP_BOUNDARIES = ("opening", "repaired_by_snapshot", "recorded_gap")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass(frozen=True)
|
|
95
|
+
class CapturedEpoch:
|
|
96
|
+
"""One connection: what it covered and whether the boundary before it left a hole."""
|
|
97
|
+
|
|
98
|
+
epoch: int
|
|
99
|
+
boundary: str
|
|
100
|
+
event_count: int
|
|
101
|
+
# Null when the connection admitted no event at all. A connection that carried only
|
|
102
|
+
# heartbeats has no first or last event, and inventing one would be a claim about data that
|
|
103
|
+
# was never captured.
|
|
104
|
+
first_event_id: str | None
|
|
105
|
+
last_event_id: str | None
|
|
106
|
+
opened_at: str
|
|
107
|
+
closed_at: str
|
|
108
|
+
|
|
109
|
+
def to_dict(self) -> dict[str, Any]:
|
|
110
|
+
return {
|
|
111
|
+
"epoch": self.epoch,
|
|
112
|
+
"boundary": self.boundary,
|
|
113
|
+
"event_count": self.event_count,
|
|
114
|
+
"first_event_id": self.first_event_id,
|
|
115
|
+
"last_event_id": self.last_event_id,
|
|
116
|
+
"opened_at": self.opened_at,
|
|
117
|
+
"closed_at": self.closed_at,
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass(frozen=True)
|
|
122
|
+
class CapturedBatch:
|
|
123
|
+
"""One sealed batch file and the bookmark it advances to."""
|
|
124
|
+
|
|
125
|
+
batch_index: int
|
|
126
|
+
epoch: int
|
|
127
|
+
storage_name: str
|
|
128
|
+
content_sha256: str
|
|
129
|
+
size_bytes: int
|
|
130
|
+
previous_bookmark: str | None
|
|
131
|
+
next_bookmark: str
|
|
132
|
+
event_ids: tuple[str, ...]
|
|
133
|
+
first_event_time: str
|
|
134
|
+
last_event_time: str
|
|
135
|
+
first_received_at: str
|
|
136
|
+
last_received_at: str
|
|
137
|
+
|
|
138
|
+
def to_dict(self) -> dict[str, Any]:
|
|
139
|
+
return {
|
|
140
|
+
"batch_index": self.batch_index,
|
|
141
|
+
"epoch": self.epoch,
|
|
142
|
+
"storage_name": self.storage_name,
|
|
143
|
+
"content_sha256": self.content_sha256,
|
|
144
|
+
"size_bytes": self.size_bytes,
|
|
145
|
+
"previous_bookmark": self.previous_bookmark,
|
|
146
|
+
"next_bookmark": self.next_bookmark,
|
|
147
|
+
"event_ids": list(self.event_ids),
|
|
148
|
+
"first_event_time": self.first_event_time,
|
|
149
|
+
"last_event_time": self.last_event_time,
|
|
150
|
+
"first_received_at": self.first_received_at,
|
|
151
|
+
"last_received_at": self.last_received_at,
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True)
|
|
156
|
+
class CaptureManifest:
|
|
157
|
+
"""The capture's own evidence: what was configured, what arrived, and what was sealed.
|
|
158
|
+
|
|
159
|
+
This is not the chained capture-evidence contract that stream work still ahead describes. It
|
|
160
|
+
is the flat, digest-bound record needed to read a capture back deterministically, and it is
|
|
161
|
+
versioned separately so a chained contract can supersede it without rewriting captures.
|
|
162
|
+
|
|
163
|
+
``recorded_through`` is the arrival time of the last message the capture admitted, read out
|
|
164
|
+
of the recording. It is deliberately not "when the recorder ran": a capture sealed from one
|
|
165
|
+
recording has one record whatever day it is sealed on, which is what lets a build be replayed
|
|
166
|
+
from the recording alone.
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
connector_id: str
|
|
170
|
+
connector_config_digest: str
|
|
171
|
+
stream_id: str
|
|
172
|
+
data_format: str
|
|
173
|
+
batch_filename: str
|
|
174
|
+
recorded_through: str
|
|
175
|
+
admitted_event_count: int
|
|
176
|
+
ignored_message_count: int
|
|
177
|
+
epochs: tuple[CapturedEpoch, ...]
|
|
178
|
+
batches: tuple[CapturedBatch, ...]
|
|
179
|
+
schema_version: str = STREAM_CAPTURE_VERSION
|
|
180
|
+
|
|
181
|
+
def to_dict(self) -> dict[str, Any]:
|
|
182
|
+
return {
|
|
183
|
+
"schema_version": self.schema_version,
|
|
184
|
+
"connector_id": self.connector_id,
|
|
185
|
+
"connector_config_digest": self.connector_config_digest,
|
|
186
|
+
"stream_id": self.stream_id,
|
|
187
|
+
"data_format": self.data_format,
|
|
188
|
+
"batch_filename": self.batch_filename,
|
|
189
|
+
"recorded_through": self.recorded_through,
|
|
190
|
+
"admitted_event_count": self.admitted_event_count,
|
|
191
|
+
"ignored_message_count": self.ignored_message_count,
|
|
192
|
+
"epochs": [epoch.to_dict() for epoch in self.epochs],
|
|
193
|
+
"batches": [batch.to_dict() for batch in self.batches],
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
@property
|
|
197
|
+
def digest(self) -> str:
|
|
198
|
+
return canonical_sha256(self.to_dict())
|
|
199
|
+
|
|
200
|
+
@property
|
|
201
|
+
def gap_count(self) -> int:
|
|
202
|
+
return sum(1 for epoch in self.epochs if epoch.boundary == "recorded_gap")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass(frozen=True)
|
|
206
|
+
class _Connection:
|
|
207
|
+
"""One connection the recording saw, whether or not it contributed an event.
|
|
208
|
+
|
|
209
|
+
A connection that carried only heartbeats is still a connection, and dropping it from the
|
|
210
|
+
record would splice its neighbours together silently -- which is the one thing this module
|
|
211
|
+
promises not to do.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
epoch: int
|
|
215
|
+
opened_at: str
|
|
216
|
+
closed_at: str
|
|
217
|
+
opened_with_snapshot: bool
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@dataclass(frozen=True)
|
|
221
|
+
class _Frame:
|
|
222
|
+
"""One recorded message after classification, with the identity the Recorder assigned."""
|
|
223
|
+
|
|
224
|
+
epoch: int
|
|
225
|
+
sequence: int
|
|
226
|
+
event_id: str
|
|
227
|
+
position: str
|
|
228
|
+
message_class: str
|
|
229
|
+
message_type: str
|
|
230
|
+
event_time: str
|
|
231
|
+
received_at: str
|
|
232
|
+
line: bytes
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def capture_root(captures_root: Path, stream_id: str) -> Path:
|
|
236
|
+
"""Return the one directory a stream's sealed capture lives in.
|
|
237
|
+
|
|
238
|
+
The whole layout is derived from the capture root, never assembled by a caller, so a build
|
|
239
|
+
and a recording cannot disagree about where a capture is::
|
|
240
|
+
|
|
241
|
+
<captures_root>/bookmarks/ the durable per-source ledger, shared
|
|
242
|
+
<captures_root>/<stream_id>/capture.json one stream's sealed record
|
|
243
|
+
<captures_root>/<stream_id>/batches/ its digest-addressed sealed batches
|
|
244
|
+
|
|
245
|
+
The ledger is shared across streams because one build holds one bookmark store; the reserved
|
|
246
|
+
names in ``stream_connector`` are what keep a stream from claiming that directory.
|
|
247
|
+
"""
|
|
248
|
+
|
|
249
|
+
return captures_root / _capture_component(stream_id)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def ledger_root(captures_root: Path) -> Path:
|
|
253
|
+
"""Return the durable bookmark ledger directory for a whole capture root."""
|
|
254
|
+
|
|
255
|
+
return captures_root / LEDGER_DIRECTORY_NAME
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def seal_recorded_capture(
|
|
259
|
+
*,
|
|
260
|
+
config: StreamConnectorConfig,
|
|
261
|
+
recording: bytes,
|
|
262
|
+
captures_root: Path,
|
|
263
|
+
) -> CaptureManifest:
|
|
264
|
+
"""Turn one recorded message log into sealed batches beneath ``captures_root``.
|
|
265
|
+
|
|
266
|
+
Every directory the layout needs is created if missing, so one call leaves a complete
|
|
267
|
+
capture a build can read.
|
|
268
|
+
|
|
269
|
+
Re-running the same recording is a no-op that produces the identical manifest: batch files
|
|
270
|
+
are content-addressed, sealing bytes that are already there returns the existing file, and
|
|
271
|
+
every time in the record is read out of the recording rather than off a clock. There is no
|
|
272
|
+
clock input at all, which is what makes a capture reproducible from its recording alone.
|
|
273
|
+
"""
|
|
274
|
+
|
|
275
|
+
if not captures_root.is_absolute():
|
|
276
|
+
raise SourceContractError(
|
|
277
|
+
"STREAM_CAPTURE_ROOT",
|
|
278
|
+
"captures_root",
|
|
279
|
+
"must be an absolute coordinator-owned path",
|
|
280
|
+
)
|
|
281
|
+
frames, ignored, connections = _classify_recording(config, recording)
|
|
282
|
+
epochs, batches_of_frames = _partition(config, frames, connections)
|
|
283
|
+
stream_root = capture_root(captures_root, config.stream_id)
|
|
284
|
+
batch_root = stream_root / BATCH_DIRECTORY_NAME
|
|
285
|
+
# Every directory is made and then checked to be a real directory rather than a link to
|
|
286
|
+
# one. ``mkdir(exist_ok=True)`` accepts a pre-existing symlink to a directory, so the
|
|
287
|
+
# check is what stops a planted link from deciding where a capture is written. It happens
|
|
288
|
+
# before any byte is written, so a refusal leaves nothing behind.
|
|
289
|
+
for directory in (captures_root, stream_root, batch_root, ledger_root(captures_root)):
|
|
290
|
+
_require_plain_directory(directory)
|
|
291
|
+
batches: list[CapturedBatch] = []
|
|
292
|
+
previous: str | None = None
|
|
293
|
+
for index, group in enumerate(batches_of_frames, start=1):
|
|
294
|
+
content = b"".join(frame.line for frame in group)
|
|
295
|
+
# The one sealing implementation, reused rather than written a second time: it is
|
|
296
|
+
# content-addressed, refuses to overwrite, follows no link, and returns the existing
|
|
297
|
+
# file when the bytes are already there, which is what makes re-sealing a no-op. It
|
|
298
|
+
# raises ``AcquisitionSecurityError`` rather than this module's own error type; both
|
|
299
|
+
# are typed refusals the command boundary already renders.
|
|
300
|
+
#
|
|
301
|
+
# The path is handed over unresolved on purpose. ``seal_content_addressed_snapshot``
|
|
302
|
+
# opens its root with ``O_NOFOLLOW``, and resolving the path here would hand it the
|
|
303
|
+
# link's target instead, so the guard would open a real directory and never fire.
|
|
304
|
+
path = seal_content_addressed_snapshot(
|
|
305
|
+
batch_root,
|
|
306
|
+
content=content,
|
|
307
|
+
suffix=config.data_format,
|
|
308
|
+
)
|
|
309
|
+
bookmark = group[-1].position
|
|
310
|
+
batches.append(
|
|
311
|
+
CapturedBatch(
|
|
312
|
+
batch_index=index,
|
|
313
|
+
epoch=group[0].epoch,
|
|
314
|
+
storage_name=path.name,
|
|
315
|
+
content_sha256=sha256_bytes(content),
|
|
316
|
+
size_bytes=len(content),
|
|
317
|
+
previous_bookmark=previous,
|
|
318
|
+
next_bookmark=bookmark,
|
|
319
|
+
event_ids=tuple(frame.event_id for frame in group),
|
|
320
|
+
first_event_time=group[0].event_time,
|
|
321
|
+
last_event_time=group[-1].event_time,
|
|
322
|
+
first_received_at=group[0].received_at,
|
|
323
|
+
last_received_at=group[-1].received_at,
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
previous = bookmark
|
|
327
|
+
manifest = CaptureManifest(
|
|
328
|
+
connector_id=config.connector_id,
|
|
329
|
+
connector_config_digest=config.digest,
|
|
330
|
+
stream_id=config.stream_id,
|
|
331
|
+
data_format=config.data_format,
|
|
332
|
+
batch_filename=f"{config.stream_id}.{config.data_format}",
|
|
333
|
+
recorded_through=frames[-1].received_at,
|
|
334
|
+
admitted_event_count=len(frames),
|
|
335
|
+
ignored_message_count=ignored,
|
|
336
|
+
epochs=epochs,
|
|
337
|
+
batches=tuple(batches),
|
|
338
|
+
)
|
|
339
|
+
_write_manifest(stream_root, manifest)
|
|
340
|
+
return manifest
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@dataclass(frozen=True)
|
|
344
|
+
class RecordedStreamGateway:
|
|
345
|
+
"""A ``StreamGateway`` over sealed recorded batches; it reaches no network at all.
|
|
346
|
+
|
|
347
|
+
``observed_at`` and ``available_at`` are coordinator inputs, not capture facts. The adapter
|
|
348
|
+
checks the batch it gets back against the observation evidence the coordinator recorded, and
|
|
349
|
+
a gateway that answered with times it read out of a file would make that check a tautology.
|
|
350
|
+
When the capture happened is separate evidence and lives in the manifest.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
captures_root: Path
|
|
354
|
+
observed_at: str
|
|
355
|
+
available_at: str
|
|
356
|
+
|
|
357
|
+
def __post_init__(self) -> None:
|
|
358
|
+
if (
|
|
359
|
+
not self.captures_root.is_absolute()
|
|
360
|
+
or self.captures_root.is_symlink()
|
|
361
|
+
or not self.captures_root.is_dir()
|
|
362
|
+
):
|
|
363
|
+
raise SourceContractError(
|
|
364
|
+
"STREAM_CAPTURE_ROOT",
|
|
365
|
+
"gateway.captures_root",
|
|
366
|
+
"must be an existing absolute non-symlink directory",
|
|
367
|
+
)
|
|
368
|
+
for name, value in (("observed_at", self.observed_at), ("available_at", self.available_at)):
|
|
369
|
+
_moment(value, f"gateway.{name}")
|
|
370
|
+
|
|
371
|
+
def read_batch(self, request: AcquisitionRequest) -> StreamBatch:
|
|
372
|
+
"""Return the one sealed batch that follows the bookmark the request carries."""
|
|
373
|
+
|
|
374
|
+
query = request.query
|
|
375
|
+
stream_id = query["stream_id"]
|
|
376
|
+
manifest = self.manifest(stream_id)
|
|
377
|
+
if query["data_format"] != manifest.data_format:
|
|
378
|
+
raise SourceContractError(
|
|
379
|
+
"STREAM_CAPTURE_BINDING",
|
|
380
|
+
"request.query.data_format",
|
|
381
|
+
"differs from the format this capture was sealed in",
|
|
382
|
+
)
|
|
383
|
+
if query["filename"] != manifest.batch_filename:
|
|
384
|
+
raise SourceContractError(
|
|
385
|
+
"STREAM_CAPTURE_BINDING",
|
|
386
|
+
"request.query.filename",
|
|
387
|
+
"differs from the name this capture seals its batches under",
|
|
388
|
+
)
|
|
389
|
+
expected = query["expected_watermark"]
|
|
390
|
+
following = [batch for batch in manifest.batches if batch.previous_bookmark == expected]
|
|
391
|
+
if not following:
|
|
392
|
+
raise SourceContractError(
|
|
393
|
+
"STREAM_CAPTURE_EXHAUSTED",
|
|
394
|
+
"request.query.expected_watermark",
|
|
395
|
+
"this capture holds no sealed batch after the bookmark the request carries",
|
|
396
|
+
)
|
|
397
|
+
if len(following) > 1:
|
|
398
|
+
# The loader already refuses a record whose batches do not form one unbroken chain,
|
|
399
|
+
# so this cannot fire against a record that got this far. It is kept as the second
|
|
400
|
+
# gate on the same invariant, at the point the choice is actually made: a bug that
|
|
401
|
+
# served two batches for one bookmark would have to defeat both.
|
|
402
|
+
raise SourceContractError(
|
|
403
|
+
"STREAM_CAPTURE_AMBIGUOUS",
|
|
404
|
+
"capture.batches",
|
|
405
|
+
"more than one sealed batch claims the same predecessor bookmark",
|
|
406
|
+
)
|
|
407
|
+
selected = following[0]
|
|
408
|
+
content = _read_sealed_batch(
|
|
409
|
+
capture_root(self.captures_root, stream_id) / BATCH_DIRECTORY_NAME,
|
|
410
|
+
storage_name=selected.storage_name,
|
|
411
|
+
expected_size=selected.size_bytes,
|
|
412
|
+
)
|
|
413
|
+
if sha256_bytes(content) != selected.content_sha256:
|
|
414
|
+
raise SourceContractError(
|
|
415
|
+
"STREAM_CAPTURE_DIGEST",
|
|
416
|
+
"capture.batches",
|
|
417
|
+
"sealed batch bytes do not match the digest the capture recorded",
|
|
418
|
+
)
|
|
419
|
+
return StreamBatch(
|
|
420
|
+
query_digest=request.query_digest,
|
|
421
|
+
previous_watermark=selected.previous_bookmark,
|
|
422
|
+
next_watermark=selected.next_bookmark,
|
|
423
|
+
provider_event_ids=selected.event_ids,
|
|
424
|
+
content=content,
|
|
425
|
+
content_sha256=selected.content_sha256,
|
|
426
|
+
media_type=_MEDIA_TYPES[manifest.data_format],
|
|
427
|
+
data_format=manifest.data_format,
|
|
428
|
+
filename=manifest.batch_filename,
|
|
429
|
+
observed_at=self.observed_at,
|
|
430
|
+
available_at=self.available_at,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
def manifest(self, stream_id: str) -> CaptureManifest:
|
|
434
|
+
"""Return the sealed capture manifest for one stream beneath the capture root."""
|
|
435
|
+
|
|
436
|
+
root = capture_root(self.captures_root, stream_id)
|
|
437
|
+
raw = _read_capture_file(root, CAPTURE_MANIFEST_NAME, MAX_MANIFEST_BYTES)
|
|
438
|
+
manifest = read_capture_manifest(raw)
|
|
439
|
+
if manifest.stream_id != stream_id:
|
|
440
|
+
raise SourceContractError(
|
|
441
|
+
"STREAM_CAPTURE_BINDING",
|
|
442
|
+
"capture.stream_id",
|
|
443
|
+
"the capture found under this name records a different stream",
|
|
444
|
+
)
|
|
445
|
+
return manifest
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def read_capture_manifest(raw: bytes) -> CaptureManifest:
|
|
449
|
+
"""Parse and re-validate one sealed capture manifest.
|
|
450
|
+
|
|
451
|
+
Every structural rule the writer applied is checked again here, because a manifest is read
|
|
452
|
+
back from disk long after it was written and a build that trusted it unchecked would be
|
|
453
|
+
trusting a file, not evidence.
|
|
454
|
+
"""
|
|
455
|
+
|
|
456
|
+
value = parse_json(raw)
|
|
457
|
+
_exact_keys(
|
|
458
|
+
value,
|
|
459
|
+
{
|
|
460
|
+
"schema_version",
|
|
461
|
+
"connector_id",
|
|
462
|
+
"connector_config_digest",
|
|
463
|
+
"stream_id",
|
|
464
|
+
"data_format",
|
|
465
|
+
"batch_filename",
|
|
466
|
+
"recorded_through",
|
|
467
|
+
"admitted_event_count",
|
|
468
|
+
"ignored_message_count",
|
|
469
|
+
"epochs",
|
|
470
|
+
"batches",
|
|
471
|
+
},
|
|
472
|
+
"capture",
|
|
473
|
+
)
|
|
474
|
+
if value["schema_version"] != STREAM_CAPTURE_VERSION:
|
|
475
|
+
raise SourceContractError(
|
|
476
|
+
"STREAM_CAPTURE_VERSION",
|
|
477
|
+
"capture.schema_version",
|
|
478
|
+
"is unsupported",
|
|
479
|
+
)
|
|
480
|
+
config = stream_connector(value["connector_id"])
|
|
481
|
+
if value["connector_config_digest"] != config.digest:
|
|
482
|
+
raise SourceContractError(
|
|
483
|
+
"STREAM_CAPTURE_CONNECTOR",
|
|
484
|
+
"capture.connector_config_digest",
|
|
485
|
+
"this capture was sealed under a different connector configuration",
|
|
486
|
+
)
|
|
487
|
+
if value["stream_id"] != config.stream_id or value["data_format"] != config.data_format:
|
|
488
|
+
raise SourceContractError(
|
|
489
|
+
"STREAM_CAPTURE_CONNECTOR",
|
|
490
|
+
"capture.stream_id",
|
|
491
|
+
"capture identity differs from its own connector configuration",
|
|
492
|
+
)
|
|
493
|
+
if value["batch_filename"] != f"{config.stream_id}.{config.data_format}":
|
|
494
|
+
raise SourceContractError(
|
|
495
|
+
"STREAM_CAPTURE_CONNECTOR",
|
|
496
|
+
"capture.batch_filename",
|
|
497
|
+
"capture batch name differs from its own connector configuration",
|
|
498
|
+
)
|
|
499
|
+
_moment(value["recorded_through"], "capture.recorded_through")
|
|
500
|
+
for name in ("admitted_event_count", "ignored_message_count"):
|
|
501
|
+
if type(value[name]) is not int or not 0 <= value[name] <= MAX_RECORDED_FRAMES:
|
|
502
|
+
raise SourceContractError("STREAM_CAPTURE", f"capture.{name}", "is out of bounds")
|
|
503
|
+
epochs = _read_epochs(value["epochs"])
|
|
504
|
+
batches = _read_batches(value["batches"], config=config)
|
|
505
|
+
admitted = sum(len(batch.event_ids) for batch in batches)
|
|
506
|
+
if admitted != value["admitted_event_count"]:
|
|
507
|
+
raise SourceContractError(
|
|
508
|
+
"STREAM_CAPTURE",
|
|
509
|
+
"capture.admitted_event_count",
|
|
510
|
+
"does not match the events the sealed batches hold",
|
|
511
|
+
)
|
|
512
|
+
_require_consistent_capture(epochs, batches, recorded_through=value["recorded_through"])
|
|
513
|
+
return CaptureManifest(
|
|
514
|
+
connector_id=value["connector_id"],
|
|
515
|
+
connector_config_digest=value["connector_config_digest"],
|
|
516
|
+
stream_id=value["stream_id"],
|
|
517
|
+
data_format=value["data_format"],
|
|
518
|
+
batch_filename=value["batch_filename"],
|
|
519
|
+
recorded_through=value["recorded_through"],
|
|
520
|
+
admitted_event_count=value["admitted_event_count"],
|
|
521
|
+
ignored_message_count=value["ignored_message_count"],
|
|
522
|
+
epochs=epochs,
|
|
523
|
+
batches=batches,
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _classify_recording(
|
|
528
|
+
config: StreamConnectorConfig,
|
|
529
|
+
recording: bytes,
|
|
530
|
+
) -> tuple[tuple[_Frame, ...], int, tuple[_Connection, ...]]:
|
|
531
|
+
"""Read the recording, classify every message, and assign identity and event time.
|
|
532
|
+
|
|
533
|
+
Returns the admitted frames, how many messages the connector classified as ignored, and
|
|
534
|
+
every connection the recording saw. The ignored count is carried rather than recomputed
|
|
535
|
+
because it is capture evidence: a capture that dropped a thousand heartbeats and one that
|
|
536
|
+
dropped none are different captures. The connections are carried for the same reason and a
|
|
537
|
+
stronger one -- a connection that admitted nothing at all still has to appear in the record,
|
|
538
|
+
or its two neighbours are spliced together with nothing saying so.
|
|
539
|
+
"""
|
|
540
|
+
|
|
541
|
+
if not isinstance(recording, bytes | bytearray):
|
|
542
|
+
raise SourceContractError("STREAM_RECORDING", "recording", "must be exact bytes")
|
|
543
|
+
if not recording or len(recording) > MAX_RECORDING_BYTES:
|
|
544
|
+
raise SourceContractError(
|
|
545
|
+
"STREAM_RECORDING",
|
|
546
|
+
"recording",
|
|
547
|
+
f"must be non-empty and no larger than {MAX_RECORDING_BYTES} bytes",
|
|
548
|
+
)
|
|
549
|
+
lines = bytes(recording).split(b"\n")
|
|
550
|
+
if lines[-1] != b"":
|
|
551
|
+
raise SourceContractError(
|
|
552
|
+
"STREAM_RECORDING",
|
|
553
|
+
"recording",
|
|
554
|
+
"every recorded line must end with a newline",
|
|
555
|
+
)
|
|
556
|
+
lines = lines[:-1]
|
|
557
|
+
if not lines or len(lines) > MAX_RECORDED_FRAMES:
|
|
558
|
+
raise SourceContractError(
|
|
559
|
+
"STREAM_RECORDING",
|
|
560
|
+
"recording",
|
|
561
|
+
f"must hold between one and {MAX_RECORDED_FRAMES} recorded messages",
|
|
562
|
+
)
|
|
563
|
+
frames: list[_Frame] = []
|
|
564
|
+
connections: list[dict[str, Any]] = []
|
|
565
|
+
ignored = 0
|
|
566
|
+
epoch = 0
|
|
567
|
+
sequence = 0
|
|
568
|
+
epoch_opened = False
|
|
569
|
+
previous_received: datetime | None = None
|
|
570
|
+
for number, line in enumerate(lines, start=1):
|
|
571
|
+
where = f"recording.line[{number}]"
|
|
572
|
+
value = parse_json(line)
|
|
573
|
+
_exact_keys(value, _RECORDING_KEYS, where)
|
|
574
|
+
frame_epoch = value["epoch"]
|
|
575
|
+
if type(frame_epoch) is not int or not 1 <= frame_epoch <= MAX_EPOCHS:
|
|
576
|
+
raise SourceContractError(
|
|
577
|
+
"STREAM_RECORDING",
|
|
578
|
+
f"{where}.epoch",
|
|
579
|
+
f"must be an integer between 1 and {MAX_EPOCHS}",
|
|
580
|
+
)
|
|
581
|
+
if frame_epoch != epoch:
|
|
582
|
+
if frame_epoch != epoch + 1:
|
|
583
|
+
raise SourceContractError(
|
|
584
|
+
"STREAM_RECORDING",
|
|
585
|
+
f"{where}.epoch",
|
|
586
|
+
"connections must be numbered from one with no skipped connection",
|
|
587
|
+
)
|
|
588
|
+
epoch = frame_epoch
|
|
589
|
+
sequence = 0
|
|
590
|
+
epoch_opened = False
|
|
591
|
+
connections.append(
|
|
592
|
+
{
|
|
593
|
+
"epoch": epoch,
|
|
594
|
+
"opened_at": value["received_at"],
|
|
595
|
+
"closed_at": value["received_at"],
|
|
596
|
+
"opened_with_snapshot": False,
|
|
597
|
+
}
|
|
598
|
+
)
|
|
599
|
+
received = _moment(value["received_at"], f"{where}.received_at")
|
|
600
|
+
if previous_received is not None and received < previous_received:
|
|
601
|
+
raise SourceContractError(
|
|
602
|
+
"STREAM_RECORDING",
|
|
603
|
+
f"{where}.received_at",
|
|
604
|
+
"recorded messages must not move backwards in time",
|
|
605
|
+
)
|
|
606
|
+
previous_received = received
|
|
607
|
+
message = value["message"]
|
|
608
|
+
if not isinstance(message, dict) or not message:
|
|
609
|
+
raise SourceContractError(
|
|
610
|
+
"STREAM_RECORDING",
|
|
611
|
+
f"{where}.message",
|
|
612
|
+
"must be a non-empty JSON object",
|
|
613
|
+
)
|
|
614
|
+
message_type = message.get(config.message_type_field)
|
|
615
|
+
if not isinstance(message_type, str):
|
|
616
|
+
raise SourceContractError(
|
|
617
|
+
"STREAM_MESSAGE_TYPE",
|
|
618
|
+
f"{where}.message.{config.message_type_field}",
|
|
619
|
+
"must be the text message type the connector classifies on",
|
|
620
|
+
)
|
|
621
|
+
message_class = config.message_class(message_type)
|
|
622
|
+
connections[-1]["closed_at"] = value["received_at"]
|
|
623
|
+
if message_class == "ignored":
|
|
624
|
+
ignored += 1
|
|
625
|
+
continue
|
|
626
|
+
if not epoch_opened and config.gap_repair == "snapshot_on_subscribe":
|
|
627
|
+
if message_class != "snapshot":
|
|
628
|
+
raise SourceContractError(
|
|
629
|
+
"STREAM_CAPTURE_GAP_UNREPAIRED",
|
|
630
|
+
f"{where}.message",
|
|
631
|
+
"this connector repairs a connection with a full snapshot and this "
|
|
632
|
+
"connection did not open with one",
|
|
633
|
+
)
|
|
634
|
+
if not epoch_opened:
|
|
635
|
+
connections[-1]["opened_with_snapshot"] = message_class == "snapshot"
|
|
636
|
+
epoch_opened = True
|
|
637
|
+
sequence += 1
|
|
638
|
+
event_id = _event_id(config, message, epoch=epoch, sequence=sequence, where=where)
|
|
639
|
+
event_time = _event_time(config, message, where=where)
|
|
640
|
+
frames.append(
|
|
641
|
+
_Frame(
|
|
642
|
+
epoch=epoch,
|
|
643
|
+
sequence=sequence,
|
|
644
|
+
event_id=event_id,
|
|
645
|
+
position=_position(epoch, sequence),
|
|
646
|
+
message_class=message_class,
|
|
647
|
+
message_type=message_type,
|
|
648
|
+
event_time=event_time,
|
|
649
|
+
received_at=value["received_at"],
|
|
650
|
+
line=canonical_json_bytes(
|
|
651
|
+
{
|
|
652
|
+
"epoch": epoch,
|
|
653
|
+
"event_id": event_id,
|
|
654
|
+
"event_time": event_time,
|
|
655
|
+
"message": message,
|
|
656
|
+
"message_class": message_class,
|
|
657
|
+
"message_type": message_type,
|
|
658
|
+
"received_at": value["received_at"],
|
|
659
|
+
"sequence": sequence,
|
|
660
|
+
}
|
|
661
|
+
)
|
|
662
|
+
+ b"\n",
|
|
663
|
+
)
|
|
664
|
+
)
|
|
665
|
+
if not frames:
|
|
666
|
+
raise SourceContractError(
|
|
667
|
+
"STREAM_RECORDING",
|
|
668
|
+
"recording",
|
|
669
|
+
"every recorded message was ignored, so there is nothing to seal",
|
|
670
|
+
)
|
|
671
|
+
identities = [frame.event_id for frame in frames]
|
|
672
|
+
if len(set(identities)) != len(identities):
|
|
673
|
+
raise SourceContractError(
|
|
674
|
+
"STREAM_EVENT_ID",
|
|
675
|
+
"recording",
|
|
676
|
+
"the connector's identity rule did not give every message a distinct name",
|
|
677
|
+
)
|
|
678
|
+
return (
|
|
679
|
+
tuple(frames),
|
|
680
|
+
ignored,
|
|
681
|
+
tuple(
|
|
682
|
+
_Connection(
|
|
683
|
+
epoch=item["epoch"],
|
|
684
|
+
opened_at=item["opened_at"],
|
|
685
|
+
closed_at=item["closed_at"],
|
|
686
|
+
opened_with_snapshot=item["opened_with_snapshot"],
|
|
687
|
+
)
|
|
688
|
+
for item in connections
|
|
689
|
+
),
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _partition(
|
|
694
|
+
config: StreamConnectorConfig,
|
|
695
|
+
frames: tuple[_Frame, ...],
|
|
696
|
+
connections: tuple[_Connection, ...],
|
|
697
|
+
) -> tuple[tuple[CapturedEpoch, ...], tuple[tuple[_Frame, ...], ...]]:
|
|
698
|
+
"""Split the classified frames into epochs and into rotation-bounded batches.
|
|
699
|
+
|
|
700
|
+
Every connection the recording saw becomes an epoch, including one that admitted nothing, so
|
|
701
|
+
the numbering stays unbroken and a dropped connection is visible as itself rather than as an
|
|
702
|
+
absence. A batch never spans a connection: an order book reconstructs only within a
|
|
703
|
+
contiguous epoch, so a batch that straddled a gap would carry a hole no downstream check
|
|
704
|
+
could see.
|
|
705
|
+
"""
|
|
706
|
+
|
|
707
|
+
by_epoch: dict[int, list[_Frame]] = {}
|
|
708
|
+
for frame in frames:
|
|
709
|
+
by_epoch.setdefault(frame.epoch, []).append(frame)
|
|
710
|
+
epochs: list[CapturedEpoch] = []
|
|
711
|
+
batches: list[tuple[_Frame, ...]] = []
|
|
712
|
+
for position, connection in enumerate(connections):
|
|
713
|
+
members = tuple(by_epoch.get(connection.epoch, ()))
|
|
714
|
+
epochs.append(
|
|
715
|
+
CapturedEpoch(
|
|
716
|
+
epoch=connection.epoch,
|
|
717
|
+
boundary=_boundary(config, connection, first=position == 0),
|
|
718
|
+
event_count=len(members),
|
|
719
|
+
first_event_id=members[0].event_id if members else None,
|
|
720
|
+
last_event_id=members[-1].event_id if members else None,
|
|
721
|
+
opened_at=connection.opened_at,
|
|
722
|
+
closed_at=connection.closed_at,
|
|
723
|
+
)
|
|
724
|
+
)
|
|
725
|
+
current: list[_Frame] = []
|
|
726
|
+
size = 0
|
|
727
|
+
opened = None if not members else _moment(members[0].received_at, "recording.received_at")
|
|
728
|
+
for frame in members:
|
|
729
|
+
arrived = _moment(frame.received_at, "recording.received_at")
|
|
730
|
+
rotate = bool(current) and (
|
|
731
|
+
len(current) >= config.max_batch_events
|
|
732
|
+
or size + len(frame.line) > config.max_batch_bytes
|
|
733
|
+
or (arrived - opened).total_seconds() > config.max_batch_seconds
|
|
734
|
+
)
|
|
735
|
+
if rotate:
|
|
736
|
+
batches.append(tuple(current))
|
|
737
|
+
current = []
|
|
738
|
+
size = 0
|
|
739
|
+
opened = arrived
|
|
740
|
+
current.append(frame)
|
|
741
|
+
size += len(frame.line)
|
|
742
|
+
if current:
|
|
743
|
+
batches.append(tuple(current))
|
|
744
|
+
if len(batches) > MAX_BATCHES:
|
|
745
|
+
raise SourceContractError(
|
|
746
|
+
"STREAM_CAPTURE",
|
|
747
|
+
"capture.batches",
|
|
748
|
+
f"one capture holds at most {MAX_BATCHES} sealed batches",
|
|
749
|
+
)
|
|
750
|
+
for group in batches:
|
|
751
|
+
if len(group) > MAX_BATCH_EVENTS:
|
|
752
|
+
raise SourceContractError(
|
|
753
|
+
"STREAM_EVENT_LIMIT",
|
|
754
|
+
"capture.batches",
|
|
755
|
+
"a sealed batch exceeds the adapter's own event bound",
|
|
756
|
+
)
|
|
757
|
+
if sum(len(frame.line) for frame in group) > config.max_batch_bytes:
|
|
758
|
+
raise SourceContractError(
|
|
759
|
+
"STREAM_CAPTURE_LIMIT",
|
|
760
|
+
"capture.batches",
|
|
761
|
+
"one recorded message is larger than this connector's whole batch bound",
|
|
762
|
+
)
|
|
763
|
+
return tuple(epochs), tuple(batches)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _boundary(config: StreamConnectorConfig, connection: _Connection, *, first: bool) -> str:
|
|
767
|
+
"""Say what the boundary before one connection left behind.
|
|
768
|
+
|
|
769
|
+
The first connection of a capture opens it and has no boundary before it. After that, a
|
|
770
|
+
hole is closed only when the venue's own repair rule actually ran: for a venue that re-sends
|
|
771
|
+
full state on subscribe, that means this connection really did open with a snapshot. A
|
|
772
|
+
connection that admitted nothing never re-sent one, so its boundary is a recorded gap --
|
|
773
|
+
which is the honest answer and the reason the gap count is not decorative.
|
|
774
|
+
"""
|
|
775
|
+
|
|
776
|
+
if first:
|
|
777
|
+
return "opening"
|
|
778
|
+
if config.gap_repair == "snapshot_on_subscribe" and connection.opened_with_snapshot:
|
|
779
|
+
return "repaired_by_snapshot"
|
|
780
|
+
return "recorded_gap"
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def _event_id(
|
|
784
|
+
config: StreamConnectorConfig,
|
|
785
|
+
message: dict[str, Any],
|
|
786
|
+
*,
|
|
787
|
+
epoch: int,
|
|
788
|
+
sequence: int,
|
|
789
|
+
where: str,
|
|
790
|
+
) -> str:
|
|
791
|
+
"""Assign one event its identity under the connector's declared rule."""
|
|
792
|
+
|
|
793
|
+
if config.event_id_rule == "derived":
|
|
794
|
+
return _position(epoch, sequence)
|
|
795
|
+
field = config.provider_event_id_field
|
|
796
|
+
assert field is not None # the configuration contract pairs the rule with the field
|
|
797
|
+
value = message.get(field)
|
|
798
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
799
|
+
value = str(value)
|
|
800
|
+
if not isinstance(value, str) or EVENT_ID_GRAMMAR.fullmatch(value) is None:
|
|
801
|
+
raise SourceContractError(
|
|
802
|
+
"STREAM_EVENT_ID",
|
|
803
|
+
f"{where}.message.{field}",
|
|
804
|
+
"the venue's own event identifier is missing or is not a usable name",
|
|
805
|
+
)
|
|
806
|
+
return value
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _position(epoch: int, sequence: int) -> str:
|
|
810
|
+
"""The Recorder-assigned ``{epoch}:{counter}`` position, ordered as text.
|
|
811
|
+
|
|
812
|
+
Zero padding is what makes text order and number order the same thing, which is what lets a
|
|
813
|
+
bookmark be compared without knowing how to take it apart.
|
|
814
|
+
"""
|
|
815
|
+
|
|
816
|
+
return f"e{epoch:08d}:{sequence:012d}"
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _event_time(config: StreamConnectorConfig, message: dict[str, Any], *, where: str) -> str:
|
|
820
|
+
"""Read the event-time field and check it against its declared interpretation."""
|
|
821
|
+
|
|
822
|
+
value = message.get(config.event_time_field)
|
|
823
|
+
if isinstance(value, int) and not isinstance(value, bool):
|
|
824
|
+
value = str(value)
|
|
825
|
+
if not isinstance(value, str) or not 1 <= len(value) <= 64:
|
|
826
|
+
raise SourceContractError(
|
|
827
|
+
"STREAM_EVENT_TIME",
|
|
828
|
+
f"{where}.message.{config.event_time_field}",
|
|
829
|
+
"must carry the event time the connector declares",
|
|
830
|
+
)
|
|
831
|
+
if EVENT_TIME_KINDS[config.event_time_kind].fullmatch(value) is None:
|
|
832
|
+
raise SourceContractError(
|
|
833
|
+
"STREAM_EVENT_TIME",
|
|
834
|
+
f"{where}.message.{config.event_time_field}",
|
|
835
|
+
f"does not read as {config.event_time_kind}",
|
|
836
|
+
)
|
|
837
|
+
return value
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _read_epochs(value: Any) -> tuple[CapturedEpoch, ...]:
|
|
841
|
+
if not isinstance(value, list) or not 1 <= len(value) <= MAX_EPOCHS:
|
|
842
|
+
raise SourceContractError("STREAM_CAPTURE", "capture.epochs", "must be a bounded list")
|
|
843
|
+
epochs: list[CapturedEpoch] = []
|
|
844
|
+
previous_closed: datetime | None = None
|
|
845
|
+
for index, item in enumerate(value, start=1):
|
|
846
|
+
_exact_keys(
|
|
847
|
+
item,
|
|
848
|
+
{
|
|
849
|
+
"epoch",
|
|
850
|
+
"boundary",
|
|
851
|
+
"event_count",
|
|
852
|
+
"first_event_id",
|
|
853
|
+
"last_event_id",
|
|
854
|
+
"opened_at",
|
|
855
|
+
"closed_at",
|
|
856
|
+
},
|
|
857
|
+
"capture.epochs",
|
|
858
|
+
)
|
|
859
|
+
if item["epoch"] != index or item["boundary"] not in GAP_BOUNDARIES:
|
|
860
|
+
raise SourceContractError(
|
|
861
|
+
"STREAM_CAPTURE",
|
|
862
|
+
"capture.epochs",
|
|
863
|
+
"connections must be numbered from one and carry a known boundary",
|
|
864
|
+
)
|
|
865
|
+
if (index == 1) != (item["boundary"] == "opening"):
|
|
866
|
+
raise SourceContractError(
|
|
867
|
+
"STREAM_CAPTURE",
|
|
868
|
+
"capture.epochs",
|
|
869
|
+
"only the first connection of a capture opens without a boundary before it",
|
|
870
|
+
)
|
|
871
|
+
if (
|
|
872
|
+
type(item["event_count"]) is not int
|
|
873
|
+
or not 0 <= item["event_count"] <= MAX_RECORDED_FRAMES
|
|
874
|
+
):
|
|
875
|
+
raise SourceContractError(
|
|
876
|
+
"STREAM_CAPTURE",
|
|
877
|
+
"capture.epochs.event_count",
|
|
878
|
+
"each connection must record how many events it carried",
|
|
879
|
+
)
|
|
880
|
+
# A connection that admitted nothing has no first or last event, and one that admitted
|
|
881
|
+
# something has both. Anything else is a record describing a connection that cannot
|
|
882
|
+
# exist.
|
|
883
|
+
named = [item["first_event_id"], item["last_event_id"]]
|
|
884
|
+
if item["event_count"] == 0:
|
|
885
|
+
if named != [None, None]:
|
|
886
|
+
raise SourceContractError(
|
|
887
|
+
"STREAM_CAPTURE",
|
|
888
|
+
"capture.epochs.first_event_id",
|
|
889
|
+
"a connection that carried no event cannot name one",
|
|
890
|
+
)
|
|
891
|
+
else:
|
|
892
|
+
for name in ("first_event_id", "last_event_id"):
|
|
893
|
+
_event_name(item[name], f"capture.epochs.{name}")
|
|
894
|
+
if item["first_event_id"] > item["last_event_id"]:
|
|
895
|
+
raise SourceContractError(
|
|
896
|
+
"STREAM_CAPTURE",
|
|
897
|
+
"capture.epochs.last_event_id",
|
|
898
|
+
"a connection cannot end at an event earlier than the one it began at",
|
|
899
|
+
)
|
|
900
|
+
opened = _moment(item["opened_at"], "capture.epochs.opened_at")
|
|
901
|
+
closed = _moment(item["closed_at"], "capture.epochs.closed_at")
|
|
902
|
+
if closed < opened:
|
|
903
|
+
raise SourceContractError(
|
|
904
|
+
"STREAM_CAPTURE",
|
|
905
|
+
"capture.epochs.closed_at",
|
|
906
|
+
"a connection cannot close before it opened",
|
|
907
|
+
)
|
|
908
|
+
if previous_closed is not None and opened < previous_closed:
|
|
909
|
+
raise SourceContractError(
|
|
910
|
+
"STREAM_CAPTURE",
|
|
911
|
+
"capture.epochs.opened_at",
|
|
912
|
+
"connections must not overlap in time",
|
|
913
|
+
)
|
|
914
|
+
previous_closed = closed
|
|
915
|
+
epochs.append(
|
|
916
|
+
CapturedEpoch(
|
|
917
|
+
epoch=item["epoch"],
|
|
918
|
+
boundary=item["boundary"],
|
|
919
|
+
event_count=item["event_count"],
|
|
920
|
+
first_event_id=item["first_event_id"],
|
|
921
|
+
last_event_id=item["last_event_id"],
|
|
922
|
+
opened_at=item["opened_at"],
|
|
923
|
+
closed_at=item["closed_at"],
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
return tuple(epochs)
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def _read_batches(value: Any, *, config: StreamConnectorConfig) -> tuple[CapturedBatch, ...]:
|
|
930
|
+
if not isinstance(value, list) or not 1 <= len(value) <= MAX_BATCHES:
|
|
931
|
+
raise SourceContractError("STREAM_CAPTURE", "capture.batches", "must be a bounded list")
|
|
932
|
+
batches: list[CapturedBatch] = []
|
|
933
|
+
previous: str | None = None
|
|
934
|
+
previous_epoch = 0
|
|
935
|
+
seen: set[str] = set()
|
|
936
|
+
for index, item in enumerate(value, start=1):
|
|
937
|
+
_exact_keys(
|
|
938
|
+
item,
|
|
939
|
+
{
|
|
940
|
+
"batch_index",
|
|
941
|
+
"epoch",
|
|
942
|
+
"storage_name",
|
|
943
|
+
"content_sha256",
|
|
944
|
+
"size_bytes",
|
|
945
|
+
"previous_bookmark",
|
|
946
|
+
"next_bookmark",
|
|
947
|
+
"event_ids",
|
|
948
|
+
"first_event_time",
|
|
949
|
+
"last_event_time",
|
|
950
|
+
"first_received_at",
|
|
951
|
+
"last_received_at",
|
|
952
|
+
},
|
|
953
|
+
"capture.batches",
|
|
954
|
+
)
|
|
955
|
+
if item["batch_index"] != index:
|
|
956
|
+
raise SourceContractError(
|
|
957
|
+
"STREAM_CAPTURE",
|
|
958
|
+
"capture.batches",
|
|
959
|
+
"sealed batches must be numbered from one in order",
|
|
960
|
+
)
|
|
961
|
+
if (
|
|
962
|
+
type(item["epoch"]) is not int
|
|
963
|
+
or not 1 <= item["epoch"] <= MAX_EPOCHS
|
|
964
|
+
or item["epoch"] < previous_epoch
|
|
965
|
+
):
|
|
966
|
+
raise SourceContractError(
|
|
967
|
+
"STREAM_CAPTURE",
|
|
968
|
+
"capture.batches.epoch",
|
|
969
|
+
"each sealed batch belongs to one connection, in order",
|
|
970
|
+
)
|
|
971
|
+
previous_epoch = item["epoch"]
|
|
972
|
+
if (
|
|
973
|
+
not isinstance(item["storage_name"], str)
|
|
974
|
+
or _STORAGE_NAME.fullmatch(item["storage_name"]) is None
|
|
975
|
+
):
|
|
976
|
+
raise SourceContractError(
|
|
977
|
+
"STREAM_CAPTURE",
|
|
978
|
+
"capture.batches.storage_name",
|
|
979
|
+
"must be the digest-addressed sealed batch name",
|
|
980
|
+
)
|
|
981
|
+
if item["storage_name"] != f"{item['content_sha256']}.{config.data_format}":
|
|
982
|
+
raise SourceContractError(
|
|
983
|
+
"STREAM_CAPTURE",
|
|
984
|
+
"capture.batches.storage_name",
|
|
985
|
+
"must be the exact digest-addressed name of its own bytes",
|
|
986
|
+
)
|
|
987
|
+
if (
|
|
988
|
+
type(item["size_bytes"]) is not int
|
|
989
|
+
or not 1 <= item["size_bytes"] <= config.max_batch_bytes
|
|
990
|
+
):
|
|
991
|
+
raise SourceContractError(
|
|
992
|
+
"STREAM_CAPTURE",
|
|
993
|
+
"capture.batches.size_bytes",
|
|
994
|
+
"is outside the batch bound this connector declares",
|
|
995
|
+
)
|
|
996
|
+
if item["previous_bookmark"] != previous:
|
|
997
|
+
raise SourceContractError(
|
|
998
|
+
"STREAM_CAPTURE",
|
|
999
|
+
"capture.batches.previous_bookmark",
|
|
1000
|
+
"sealed batches must form one unbroken chain of bookmarks",
|
|
1001
|
+
)
|
|
1002
|
+
if index == 1 and item["previous_bookmark"] is not None:
|
|
1003
|
+
raise SourceContractError(
|
|
1004
|
+
"STREAM_CAPTURE",
|
|
1005
|
+
"capture.batches.previous_bookmark",
|
|
1006
|
+
"the first sealed batch of a capture follows no bookmark",
|
|
1007
|
+
)
|
|
1008
|
+
identities = _read_event_ids(item["event_ids"], config=config)
|
|
1009
|
+
if seen & set(identities):
|
|
1010
|
+
raise SourceContractError(
|
|
1011
|
+
"STREAM_CAPTURE",
|
|
1012
|
+
"capture.batches.event_ids",
|
|
1013
|
+
"one event name appears in more than one sealed batch",
|
|
1014
|
+
)
|
|
1015
|
+
seen |= set(identities)
|
|
1016
|
+
marker = _BOOKMARK.fullmatch(item["next_bookmark"] or "")
|
|
1017
|
+
if marker is None or int(marker.group(1)) != item["epoch"]:
|
|
1018
|
+
raise SourceContractError(
|
|
1019
|
+
"STREAM_CAPTURE",
|
|
1020
|
+
"capture.batches.next_bookmark",
|
|
1021
|
+
"a batch's bookmark is its own position in its own connection",
|
|
1022
|
+
)
|
|
1023
|
+
# The bookmark is what a build commits against and then asks to continue from, so it has
|
|
1024
|
+
# to move one way. A record whose batches walked backwards would rewind a source that
|
|
1025
|
+
# has already been built, without anything downstream being able to tell.
|
|
1026
|
+
if previous is not None and item["next_bookmark"] <= previous:
|
|
1027
|
+
raise SourceContractError(
|
|
1028
|
+
"STREAM_CAPTURE",
|
|
1029
|
+
"capture.batches.next_bookmark",
|
|
1030
|
+
"each sealed batch must advance the bookmark, never move it back",
|
|
1031
|
+
)
|
|
1032
|
+
for name in ("first_event_time", "last_event_time"):
|
|
1033
|
+
_declared_event_time(item[name], config=config, path=f"capture.batches.{name}")
|
|
1034
|
+
first = _moment(item["first_received_at"], "capture.batches.first_received_at")
|
|
1035
|
+
if _moment(item["last_received_at"], "capture.batches.last_received_at") < first:
|
|
1036
|
+
raise SourceContractError(
|
|
1037
|
+
"STREAM_CAPTURE",
|
|
1038
|
+
"capture.batches.last_received_at",
|
|
1039
|
+
"a sealed batch cannot end before it began",
|
|
1040
|
+
)
|
|
1041
|
+
batches.append(
|
|
1042
|
+
CapturedBatch(
|
|
1043
|
+
batch_index=item["batch_index"],
|
|
1044
|
+
epoch=item["epoch"],
|
|
1045
|
+
storage_name=item["storage_name"],
|
|
1046
|
+
content_sha256=item["content_sha256"],
|
|
1047
|
+
size_bytes=item["size_bytes"],
|
|
1048
|
+
previous_bookmark=item["previous_bookmark"],
|
|
1049
|
+
next_bookmark=item["next_bookmark"],
|
|
1050
|
+
event_ids=identities,
|
|
1051
|
+
first_event_time=item["first_event_time"],
|
|
1052
|
+
last_event_time=item["last_event_time"],
|
|
1053
|
+
first_received_at=item["first_received_at"],
|
|
1054
|
+
last_received_at=item["last_received_at"],
|
|
1055
|
+
)
|
|
1056
|
+
)
|
|
1057
|
+
previous = item["next_bookmark"]
|
|
1058
|
+
return tuple(batches)
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _read_event_ids(value: Any, *, config: StreamConnectorConfig) -> tuple[str, ...]:
|
|
1062
|
+
"""Check one batch's event names are bounded, distinct, well formed, and in order."""
|
|
1063
|
+
|
|
1064
|
+
if not isinstance(value, list) or not 1 <= len(value) <= config.max_batch_events:
|
|
1065
|
+
raise SourceContractError(
|
|
1066
|
+
"STREAM_CAPTURE",
|
|
1067
|
+
"capture.batches.event_ids",
|
|
1068
|
+
"must be a bounded non-empty list of event names",
|
|
1069
|
+
)
|
|
1070
|
+
for item in value:
|
|
1071
|
+
_event_name(item, "capture.batches.event_ids")
|
|
1072
|
+
if len(set(value)) != len(value):
|
|
1073
|
+
raise SourceContractError(
|
|
1074
|
+
"STREAM_CAPTURE",
|
|
1075
|
+
"capture.batches.event_ids",
|
|
1076
|
+
"must name each event once",
|
|
1077
|
+
)
|
|
1078
|
+
return tuple(value)
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _event_name(value: Any, path: str) -> str:
|
|
1082
|
+
if not isinstance(value, str) or EVENT_ID_GRAMMAR.fullmatch(value) is None:
|
|
1083
|
+
raise SourceContractError(
|
|
1084
|
+
"STREAM_EVENT_ID",
|
|
1085
|
+
path,
|
|
1086
|
+
"must be one event name in the grammar the ledger admits",
|
|
1087
|
+
)
|
|
1088
|
+
return value
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
def _declared_event_time(value: Any, *, config: StreamConnectorConfig, path: str) -> str:
|
|
1092
|
+
if (
|
|
1093
|
+
not isinstance(value, str)
|
|
1094
|
+
or not 1 <= len(value) <= 64
|
|
1095
|
+
or EVENT_TIME_KINDS[config.event_time_kind].fullmatch(value) is None
|
|
1096
|
+
):
|
|
1097
|
+
raise SourceContractError(
|
|
1098
|
+
"STREAM_EVENT_TIME",
|
|
1099
|
+
path,
|
|
1100
|
+
f"does not read as {config.event_time_kind}",
|
|
1101
|
+
)
|
|
1102
|
+
return value
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
def _require_consistent_capture(
|
|
1106
|
+
epochs: tuple[CapturedEpoch, ...],
|
|
1107
|
+
batches: tuple[CapturedBatch, ...],
|
|
1108
|
+
*,
|
|
1109
|
+
recorded_through: str,
|
|
1110
|
+
) -> None:
|
|
1111
|
+
"""Check the two halves of a capture record describe the same capture.
|
|
1112
|
+
|
|
1113
|
+
Each half is well formed on its own by the time this runs. What is left is whether they
|
|
1114
|
+
agree: whether every batch belongs to a connection the record lists, whether each
|
|
1115
|
+
connection's stated event count is the number of events its batches actually hold, whether
|
|
1116
|
+
the connection's own first and last event names are the ones its batches begin and end with,
|
|
1117
|
+
and whether the capture's stated end is where its last batch actually stopped.
|
|
1118
|
+
"""
|
|
1119
|
+
|
|
1120
|
+
numbers = {epoch.epoch for epoch in epochs}
|
|
1121
|
+
unknown = sorted({batch.epoch for batch in batches} - numbers)
|
|
1122
|
+
if unknown:
|
|
1123
|
+
raise SourceContractError(
|
|
1124
|
+
"STREAM_CAPTURE",
|
|
1125
|
+
"capture.batches.epoch",
|
|
1126
|
+
f"sealed batches name connections this capture does not record: {unknown[:10]}",
|
|
1127
|
+
)
|
|
1128
|
+
for epoch in epochs:
|
|
1129
|
+
members = [batch for batch in batches if batch.epoch == epoch.epoch]
|
|
1130
|
+
held = [identity for batch in members for identity in batch.event_ids]
|
|
1131
|
+
if len(held) != epoch.event_count:
|
|
1132
|
+
raise SourceContractError(
|
|
1133
|
+
"STREAM_CAPTURE",
|
|
1134
|
+
"capture.epochs.event_count",
|
|
1135
|
+
"a connection's event count is not the number its sealed batches hold",
|
|
1136
|
+
)
|
|
1137
|
+
if held and (held[0] != epoch.first_event_id or held[-1] != epoch.last_event_id):
|
|
1138
|
+
raise SourceContractError(
|
|
1139
|
+
"STREAM_CAPTURE",
|
|
1140
|
+
"capture.epochs.first_event_id",
|
|
1141
|
+
"a connection does not begin and end at the events its batches do",
|
|
1142
|
+
)
|
|
1143
|
+
if batches[-1].last_received_at != recorded_through:
|
|
1144
|
+
raise SourceContractError(
|
|
1145
|
+
"STREAM_CAPTURE",
|
|
1146
|
+
"capture.recorded_through",
|
|
1147
|
+
"a capture ends where its last sealed batch ends",
|
|
1148
|
+
)
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def _write_manifest(stream_root: Path, manifest: CaptureManifest) -> None:
|
|
1152
|
+
"""Replace the capture manifest atomically beneath a descriptor-bound root."""
|
|
1153
|
+
|
|
1154
|
+
payload = canonical_json_bytes(manifest.to_dict())
|
|
1155
|
+
if len(payload) > MAX_MANIFEST_BYTES:
|
|
1156
|
+
raise SourceContractError(
|
|
1157
|
+
"STREAM_CAPTURE_LIMIT",
|
|
1158
|
+
"capture",
|
|
1159
|
+
f"the capture record exceeds its {MAX_MANIFEST_BYTES}-byte budget",
|
|
1160
|
+
)
|
|
1161
|
+
try:
|
|
1162
|
+
root_fd = os.open(
|
|
1163
|
+
stream_root,
|
|
1164
|
+
os.O_RDONLY
|
|
1165
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
1166
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
1167
|
+
| getattr(os, "O_CLOEXEC", 0),
|
|
1168
|
+
)
|
|
1169
|
+
except OSError:
|
|
1170
|
+
raise SourceContractError(
|
|
1171
|
+
"STREAM_CAPTURE_ROOT",
|
|
1172
|
+
"capture",
|
|
1173
|
+
f"{stream_root.name} is not a plain directory this capture may be written into",
|
|
1174
|
+
) from None
|
|
1175
|
+
temporary = f".{CAPTURE_MANIFEST_NAME}.{os.getpid()}.tmp"
|
|
1176
|
+
try:
|
|
1177
|
+
try:
|
|
1178
|
+
os.unlink(temporary, dir_fd=root_fd)
|
|
1179
|
+
except FileNotFoundError:
|
|
1180
|
+
pass
|
|
1181
|
+
descriptor = os.open(
|
|
1182
|
+
temporary,
|
|
1183
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
|
1184
|
+
0o600,
|
|
1185
|
+
dir_fd=root_fd,
|
|
1186
|
+
)
|
|
1187
|
+
try:
|
|
1188
|
+
view = memoryview(payload)
|
|
1189
|
+
written = 0
|
|
1190
|
+
while written < len(view):
|
|
1191
|
+
count = os.write(descriptor, view[written:])
|
|
1192
|
+
if count <= 0:
|
|
1193
|
+
raise SourceContractError(
|
|
1194
|
+
"STREAM_CAPTURE_WRITE",
|
|
1195
|
+
"capture",
|
|
1196
|
+
"write made no progress",
|
|
1197
|
+
)
|
|
1198
|
+
written += count
|
|
1199
|
+
os.fsync(descriptor)
|
|
1200
|
+
finally:
|
|
1201
|
+
os.close(descriptor)
|
|
1202
|
+
try:
|
|
1203
|
+
os.replace(temporary, CAPTURE_MANIFEST_NAME, src_dir_fd=root_fd, dst_dir_fd=root_fd)
|
|
1204
|
+
os.fsync(root_fd)
|
|
1205
|
+
finally:
|
|
1206
|
+
try:
|
|
1207
|
+
os.unlink(temporary, dir_fd=root_fd)
|
|
1208
|
+
except FileNotFoundError:
|
|
1209
|
+
pass
|
|
1210
|
+
except OSError:
|
|
1211
|
+
raise SourceContractError(
|
|
1212
|
+
"STREAM_CAPTURE_WRITE",
|
|
1213
|
+
"capture",
|
|
1214
|
+
"the capture record could not be written into its own directory",
|
|
1215
|
+
) from None
|
|
1216
|
+
finally:
|
|
1217
|
+
os.close(root_fd)
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def _capture_component(stream_id: Any) -> str:
|
|
1221
|
+
"""Check a stream name is one strict path component before it addresses a directory."""
|
|
1222
|
+
|
|
1223
|
+
if not isinstance(stream_id, str) or re.fullmatch(r"[a-z][a-z0-9-]{0,63}", stream_id) is None:
|
|
1224
|
+
raise SourceContractError(
|
|
1225
|
+
"STREAM_CAPTURE_ROOT",
|
|
1226
|
+
"request.query.stream_id",
|
|
1227
|
+
"must be one plain lowercase name, so it can address exactly one directory",
|
|
1228
|
+
)
|
|
1229
|
+
return stream_id
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
def _require_plain_directory(path: Path) -> None:
|
|
1233
|
+
"""Make one directory if it is missing, then check it is a directory and not a link to one.
|
|
1234
|
+
|
|
1235
|
+
``Path.mkdir(exist_ok=True)`` is satisfied by a pre-existing symlink to a directory, because
|
|
1236
|
+
the existence test follows the link. So making the directory proves nothing on its own, and
|
|
1237
|
+
the ``lstat`` below is the check: a planted link must not get to decide where a capture is
|
|
1238
|
+
written. Everything this module opens afterwards uses ``O_NOFOLLOW``, so this runs once, up
|
|
1239
|
+
front, and a refusal leaves nothing written.
|
|
1240
|
+
"""
|
|
1241
|
+
|
|
1242
|
+
try:
|
|
1243
|
+
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
1244
|
+
except OSError:
|
|
1245
|
+
raise SourceContractError(
|
|
1246
|
+
"STREAM_CAPTURE_ROOT",
|
|
1247
|
+
"capture",
|
|
1248
|
+
f"{path.name} could not be created as a plain directory",
|
|
1249
|
+
) from None
|
|
1250
|
+
try:
|
|
1251
|
+
info = os.lstat(path)
|
|
1252
|
+
except OSError:
|
|
1253
|
+
raise SourceContractError(
|
|
1254
|
+
"STREAM_CAPTURE_ROOT",
|
|
1255
|
+
"capture",
|
|
1256
|
+
f"{path.name} could not be inspected",
|
|
1257
|
+
) from None
|
|
1258
|
+
if not stat.S_ISDIR(info.st_mode):
|
|
1259
|
+
raise SourceContractError(
|
|
1260
|
+
"STREAM_CAPTURE_ROOT",
|
|
1261
|
+
"capture",
|
|
1262
|
+
f"{path.name} must be a plain directory, not a link to one",
|
|
1263
|
+
)
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
def _read_capture_file(root: Path, name: str, maximum: int) -> bytes:
|
|
1267
|
+
"""Read one bounded regular file beneath a root without following any link.
|
|
1268
|
+
|
|
1269
|
+
The controls are the ones ``adapters._read_confined_file`` already applies to a source file,
|
|
1270
|
+
for the same reasons and in the same order: open non-blocking so a planted FIFO cannot hold
|
|
1271
|
+
the command open forever, require one regular file with one link so a device or a hardlink
|
|
1272
|
+
farm is refused, and compare the full descriptor and path identity before and after the read
|
|
1273
|
+
so a file swapped underneath the read is caught rather than returned.
|
|
1274
|
+
"""
|
|
1275
|
+
|
|
1276
|
+
directory_flags = (
|
|
1277
|
+
os.O_RDONLY
|
|
1278
|
+
| getattr(os, "O_DIRECTORY", 0)
|
|
1279
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
1280
|
+
| getattr(os, "O_CLOEXEC", 0)
|
|
1281
|
+
)
|
|
1282
|
+
try:
|
|
1283
|
+
root_fd = os.open(root, directory_flags)
|
|
1284
|
+
except OSError:
|
|
1285
|
+
raise SourceContractError(
|
|
1286
|
+
"STREAM_CAPTURE_ROOT",
|
|
1287
|
+
"capture",
|
|
1288
|
+
f"no readable capture directory at {root.name}",
|
|
1289
|
+
) from None
|
|
1290
|
+
try:
|
|
1291
|
+
try:
|
|
1292
|
+
descriptor = os.open(
|
|
1293
|
+
name,
|
|
1294
|
+
os.O_RDONLY
|
|
1295
|
+
| getattr(os, "O_NOFOLLOW", 0)
|
|
1296
|
+
| getattr(os, "O_NONBLOCK", 0)
|
|
1297
|
+
| getattr(os, "O_CLOEXEC", 0),
|
|
1298
|
+
dir_fd=root_fd,
|
|
1299
|
+
)
|
|
1300
|
+
except OSError:
|
|
1301
|
+
raise SourceContractError(
|
|
1302
|
+
"STREAM_CAPTURE_ROOT",
|
|
1303
|
+
"capture",
|
|
1304
|
+
f"{name} is missing or is not a plain readable file",
|
|
1305
|
+
) from None
|
|
1306
|
+
try:
|
|
1307
|
+
before = os.fstat(descriptor)
|
|
1308
|
+
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
|
|
1309
|
+
raise SourceContractError(
|
|
1310
|
+
"STREAM_CAPTURE_ROOT",
|
|
1311
|
+
"capture",
|
|
1312
|
+
f"{name} must be one plain file with one name, not a link, a pipe, or a device",
|
|
1313
|
+
)
|
|
1314
|
+
if not 1 <= before.st_size <= maximum:
|
|
1315
|
+
raise SourceContractError(
|
|
1316
|
+
"STREAM_CAPTURE_LIMIT",
|
|
1317
|
+
"capture",
|
|
1318
|
+
f"{name} is empty or over its {maximum}-byte budget",
|
|
1319
|
+
)
|
|
1320
|
+
chunks: list[bytes] = []
|
|
1321
|
+
total = 0
|
|
1322
|
+
while total <= maximum:
|
|
1323
|
+
chunk = os.read(descriptor, min(64 * 1024, maximum + 1 - total))
|
|
1324
|
+
if not chunk:
|
|
1325
|
+
break
|
|
1326
|
+
chunks.append(chunk)
|
|
1327
|
+
total += len(chunk)
|
|
1328
|
+
after = os.fstat(descriptor)
|
|
1329
|
+
named = os.stat(name, dir_fd=root_fd, follow_symlinks=False)
|
|
1330
|
+
if (
|
|
1331
|
+
total != before.st_size
|
|
1332
|
+
or _capture_stat_signature(before) != _capture_stat_signature(after)
|
|
1333
|
+
or _capture_stat_signature(before) != _capture_stat_signature(named)
|
|
1334
|
+
):
|
|
1335
|
+
raise SourceContractError(
|
|
1336
|
+
"STREAM_CAPTURE_MUTATED",
|
|
1337
|
+
"capture",
|
|
1338
|
+
f"{name} changed while it was being read",
|
|
1339
|
+
)
|
|
1340
|
+
return b"".join(chunks)
|
|
1341
|
+
finally:
|
|
1342
|
+
os.close(descriptor)
|
|
1343
|
+
except OSError:
|
|
1344
|
+
raise SourceContractError(
|
|
1345
|
+
"STREAM_CAPTURE_ROOT",
|
|
1346
|
+
"capture",
|
|
1347
|
+
f"{name} could not be read as one plain bounded file",
|
|
1348
|
+
) from None
|
|
1349
|
+
finally:
|
|
1350
|
+
os.close(root_fd)
|
|
1351
|
+
|
|
1352
|
+
|
|
1353
|
+
def _capture_stat_signature(info: os.stat_result) -> tuple[int, ...]:
|
|
1354
|
+
"""Bind the complete regular-file state one capture read is allowed to see."""
|
|
1355
|
+
|
|
1356
|
+
return (
|
|
1357
|
+
info.st_dev,
|
|
1358
|
+
info.st_ino,
|
|
1359
|
+
info.st_uid,
|
|
1360
|
+
info.st_gid,
|
|
1361
|
+
info.st_mode,
|
|
1362
|
+
info.st_nlink,
|
|
1363
|
+
info.st_size,
|
|
1364
|
+
info.st_mtime_ns,
|
|
1365
|
+
info.st_ctime_ns,
|
|
1366
|
+
)
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
def _read_sealed_batch(root: Path, *, storage_name: str, expected_size: int) -> bytes:
|
|
1370
|
+
if _STORAGE_NAME.fullmatch(storage_name) is None:
|
|
1371
|
+
raise SourceContractError(
|
|
1372
|
+
"STREAM_CAPTURE",
|
|
1373
|
+
"capture.batches.storage_name",
|
|
1374
|
+
"must be the digest-addressed sealed batch name",
|
|
1375
|
+
)
|
|
1376
|
+
content = _read_capture_file(root, storage_name, MAX_MANIFEST_BYTES)
|
|
1377
|
+
if len(content) != expected_size:
|
|
1378
|
+
raise SourceContractError(
|
|
1379
|
+
"STREAM_CAPTURE_DIGEST",
|
|
1380
|
+
"capture.batches.size_bytes",
|
|
1381
|
+
"sealed batch is not the size the capture recorded",
|
|
1382
|
+
)
|
|
1383
|
+
return content
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
def _exact_keys(value: Any, keys: frozenset[str] | set[str], path: str) -> None:
|
|
1387
|
+
if not isinstance(value, dict) or set(value) != set(keys):
|
|
1388
|
+
raise SourceContractError(
|
|
1389
|
+
"STREAM_CAPTURE",
|
|
1390
|
+
path,
|
|
1391
|
+
f"must be an object holding exactly {sorted(keys)}",
|
|
1392
|
+
)
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _moment(value: Any, path: str) -> datetime:
|
|
1396
|
+
"""Check one canonical UTC time and return it, so spans can be measured from it."""
|
|
1397
|
+
|
|
1398
|
+
if not isinstance(value, str) or _TIMESTAMP.fullmatch(value) is None:
|
|
1399
|
+
raise SourceContractError(
|
|
1400
|
+
"STREAM_CAPTURE_TIME",
|
|
1401
|
+
path,
|
|
1402
|
+
"must be a canonical RFC 3339 UTC time",
|
|
1403
|
+
)
|
|
1404
|
+
return datetime.fromisoformat(value[:-1] + "+00:00")
|