local-data-platform 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/PKG-INFO +499 -29
  2. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/README.md +498 -28
  3. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/pyproject.toml +1 -1
  4. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/__init__.py +1 -1
  5. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/cli.py +24 -8
  6. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/config.py +13 -1
  7. local_data_platform-0.1.4/src/local_data_platform/contracts.py +479 -0
  8. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/datasets.py +74 -8
  9. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/build.py +2 -2
  10. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/project.py +14 -1
  11. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/publish.py +73 -3
  12. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/sources.py +41 -3
  13. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/engine/duckdb/__init__.py +4 -4
  14. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/events.py +73 -6
  15. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/exceptions.py +17 -0
  16. local_data_platform-0.1.4/src/local_data_platform/gold/__init__.py +9 -0
  17. local_data_platform-0.1.4/src/local_data_platform/gold/build.py +424 -0
  18. local_data_platform-0.1.4/src/local_data_platform/gold/catalog.py +595 -0
  19. local_data_platform-0.1.4/src/local_data_platform/gold/cli.py +428 -0
  20. local_data_platform-0.1.4/src/local_data_platform/gold/release.py +698 -0
  21. local_data_platform-0.1.4/src/local_data_platform/gold/render.py +414 -0
  22. local_data_platform-0.1.4/src/local_data_platform/gold/serve.py +860 -0
  23. local_data_platform-0.1.4/src/local_data_platform/gold/status.py +274 -0
  24. local_data_platform-0.1.4/src/local_data_platform/gold/sync.py +133 -0
  25. local_data_platform-0.1.4/src/local_data_platform/html.py +294 -0
  26. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/lake.py +51 -11
  27. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/__init__.py +1 -1
  28. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/cli.py +29 -6
  29. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/discovery.py +59 -14
  30. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/guard.py +64 -10
  31. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/sandbox.py +99 -5
  32. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/tools.py +191 -179
  33. local_data_platform-0.1.4/src/local_data_platform/metrics/__init__.py +61 -0
  34. local_data_platform-0.1.4/src/local_data_platform/metrics/check.py +221 -0
  35. local_data_platform-0.1.4/src/local_data_platform/metrics/cli.py +180 -0
  36. local_data_platform-0.1.4/src/local_data_platform/metrics/compiler.py +427 -0
  37. local_data_platform-0.1.4/src/local_data_platform/metrics/load.py +258 -0
  38. local_data_platform-0.1.4/src/local_data_platform/metrics/model.py +621 -0
  39. local_data_platform-0.1.4/src/local_data_platform/metrics/render.py +180 -0
  40. local_data_platform-0.1.4/src/local_data_platform/metrics/runner.py +177 -0
  41. local_data_platform-0.1.4/src/local_data_platform/metrics/sql.py +213 -0
  42. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/cli.py +13 -1
  43. local_data_platform-0.1.4/src/local_data_platform/orchestration/gold_task.py +197 -0
  44. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/tasks.py +48 -22
  45. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/__init__.py +91 -9
  46. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/builders.py +13 -7
  47. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/duckdb_to_iceberg/__init__.py +3 -2
  48. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/spec.py +151 -15
  49. local_data_platform-0.1.4/src/local_data_platform/store/source/duckdb/__init__.py +269 -0
  50. local_data_platform-0.1.3/src/local_data_platform/store/source/duckdb/__init__.py +0 -155
  51. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/LICENSE +0 -0
  52. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/catalog/__init__.py +0 -0
  53. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/catalog/local/__init__.py +0 -0
  54. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/catalog/local/iceberg/__init__.py +0 -0
  55. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/catalog/provider.py +0 -0
  56. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/__init__.py +0 -0
  57. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/cli.py +0 -0
  58. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/demo.py +0 -0
  59. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/engine/__init__.py +0 -0
  60. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/engine/duckdb/row_filter.py +0 -0
  61. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/engine/router.py +0 -0
  62. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/engine/spark/__init__.py +0 -0
  63. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/etl.py +0 -0
  64. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/filelock.py +0 -0
  65. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/__init__.py +0 -0
  66. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/csv/__init__.py +0 -0
  67. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/excel/__init__.py +0 -0
  68. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/iceberg/__init__.py +0 -0
  69. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/iceberg/commit.py +0 -0
  70. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/format/parquet/__init__.py +0 -0
  71. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/fs.py +0 -0
  72. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/github/__init__.py +0 -0
  73. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/integrations/__init__.py +0 -0
  74. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/integrations/airflow.py +0 -0
  75. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/issue/__init__.py +0 -0
  76. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/logger.py +0 -0
  77. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/__init__.py +0 -0
  78. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/_common.py +0 -0
  79. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/cli.py +0 -0
  80. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/orphans.py +0 -0
  81. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/snapshots.py +0 -0
  82. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/__main__.py +0 -0
  83. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/audit.py +0 -0
  84. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/errors.py +0 -0
  85. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/server.py +0 -0
  86. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/__init__.py +0 -0
  87. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/backfills.py +0 -0
  88. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/dbt_task.py +0 -0
  89. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/ledger.py +0 -0
  90. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/recipes.py +0 -0
  91. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/retries.py +0 -0
  92. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/runner.py +0 -0
  93. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/schedule.py +0 -0
  94. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/schedule_cli.py +0 -0
  95. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/windows.py +0 -0
  96. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/orchestration/workflow.py +0 -0
  97. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/paths.py +0 -0
  98. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/builtin.py +0 -0
  99. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/__init__.py +0 -0
  100. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/csv_to_iceberg/__init__.py +0 -0
  101. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/iceberg_to_csv/__init__.py +0 -0
  102. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/iceberg_to_parquet/__init__.py +0 -0
  103. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/__init__.py +0 -0
  104. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/bigquery_to_csv/__init__.py +0 -0
  105. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/csv_to_iceberg/__init__.py +0 -0
  106. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/excel_to_csv/__init__.py +0 -0
  107. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/excel_to_iceberg/__init__.py +0 -0
  108. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/parquet_to_iceberg/__init__.py +0 -0
  109. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/pyarrow/__init__.py +0 -0
  110. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/registry.py +0 -0
  111. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/quality/__init__.py +0 -0
  112. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/quality/checks.py +0 -0
  113. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/quality/report.py +0 -0
  114. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/quality/runner.py +0 -0
  115. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/quality/temporal.py +0 -0
  116. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/__init__.py +0 -0
  117. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/__init__.py +0 -0
  118. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/gcp/__init__.py +0 -0
  119. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/gcp/bigquery/__init__.py +0 -0
  120. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/json/__init__.py +0 -0
  121. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/near/__init__.py +0 -0
  122. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/source/parquet/__init__.py +0 -0
  123. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/target/__init__.py +0 -0
  124. {local_data_platform-0.1.3 → local_data_platform-0.1.4}/src/local_data_platform/store/target/iceberg/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: local-data-platform
3
- Version: 0.1.3
3
+ Version: 0.1.4
4
4
  Summary: A Python API layer for running a complete Iceberg lakehouse data platform on your laptop.
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -67,10 +67,14 @@ It's for learning how a lakehouse works, for building and testing pipelines loca
67
67
  for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
68
68
  tables you build.
69
69
 
70
- > **Status.** This is version 0.1.3, the Orchestration milestone. It adds windows and idempotency keys
71
- > to `ldp run`, workflow files with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries,
72
- > `ldp schedule` for cron and launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)),
73
- > with a [guide](docs/orchestration.md). Version 0.1.2 added `ldp dbt`, which runs a dbt project over
70
+ > **Status.** This is version 0.1.4, the Self Serving Gold Layer milestone. It adds gold tables that
71
+ > carry an owner, a meaning and a description of every column, `ldp gold build` and one release of all
72
+ > the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once in
73
+ > `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
74
+ > [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
75
+ > with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
76
+ > launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
77
+ > [guide](docs/orchestration.md). Version 0.1.2 added `ldp dbt`, which runs a dbt project over
74
78
  > your Iceberg tables and publishes the models you mark back to Iceberg (see
75
79
  > [Transform with dbt](#transform-with-dbt)), the `DUCKDB` and `XLSX` source formats, time travel and
76
80
  > folders of tables in `ldp query`, retries for concurrent appends, and a restaurant example with a
@@ -86,7 +90,7 @@ tables you build.
86
90
  `ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
87
91
 
88
92
  ```bash
89
- pip install "local-data-platform[duckdb]>=0.1.3"
93
+ pip install "local-data-platform[duckdb]>=0.1.4"
90
94
 
91
95
  ldp demo --workdir ldp_demo
92
96
  ```
@@ -126,10 +130,11 @@ More demos, each one a `make` target (see [Development](#development)):
126
130
  | `make demo-robotics` | Humanoid-robot episodes and sensor frames in bronze, silver and gold Iceberg tables, temporal quality checks, a pinned training split and DuckDB research queries |
127
131
  | `make demo-agent` | The read-only MCP server on the demo table, driven by a scripted agent client |
128
132
  | `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
129
- | `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task when the `dbt` extra is installed) |
133
+ | `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
134
+ | `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
130
135
  | `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
131
136
  | `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
132
- | `make demo-all` | All of them; `demo-warehouse` is skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
137
+ | `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
133
138
 
134
139
  ### Extras
135
140
 
@@ -239,6 +244,7 @@ ldp query rides.json "SELECT * FROM rides" --max-rows 20 # print at most 20 ro
239
244
  ldp query rides.json "SELECT count(*) FROM rides" --snapshot 4437 # an older snapshot, from ldp snapshots
240
245
  ldp query rides.json "SELECT count(*) FROM rides" --as-of "2026-07-31 23:59:59" # the newest snapshot at or before that time (UTC)
241
246
  ldp query config/ "SELECT o.id FROM raw.orders o JOIN raw.customers c ON c.id = o.id" # a folder of configs: every table, joinable
247
+ ldp query config/ "SELECT count(*) FROM gold.daily_sales" --release r2 # the gold tables as release r2 names them
242
248
  ldp demo --workdir /tmp/ldp_demo # also --rows N (default 1000) and --seed N (default 42)
243
249
  ```
244
250
 
@@ -261,7 +267,7 @@ Every command follows one exit-code table.
261
267
  | 0 | success, including a run that published and a run that skipped a duplicate |
262
268
  | 1 | an error: a bad config, a usage error, a failed write |
263
269
  | 2 | reserved for an SLA breach that `ldp monitor` reports in 0.1.5 |
264
- | 3 | a data gate held the run back, such as a quality check that blocked a write |
270
+ | 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
265
271
 
266
272
  A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
267
273
  because the data has to change first. Add `-v` to see a traceback. The commands
@@ -289,10 +295,21 @@ call:
289
295
  | `ldp plan CONFIG [--window WINDOW] [--json]` | Validate a config (or a folder of files, each as its kind) and print its spec hash, route, target, sinks, schedule, and the source path and idempotency key of a window | `spec.validate_spec(data)`, `spec.plan(config, window=...)` |
290
296
  | `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
291
297
  | `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
292
- | `ldp datasets pin\|list\|export` | Pin, list and export dataset versions | `datasets.pin`, `list_versions`, `export` |
298
+ | `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
293
299
  | `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
294
300
  | `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
295
301
  | `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
302
+ | `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
303
+ | `ldp gold build DIR [--no-release] [--duckdb FILE] [--json]` | Build the gold tables of a folder in the order that their `inputs` say, and check the ones that are built elsewhere. One line for each table, and a gate that blocks a table stops the build with exit status 3. When every table passed, and the metrics of the folder fit them, it cuts a release, unless `--no-release` says not to. `--duckdb FILE` then writes the DuckDB file of the gold layer. See [Build the gold tables](#build-the-gold-tables) | `gold.build.build_gold(folder, release=True)`, `gold.serve.refresh_after_build(folder, out, result)` |
304
+ | `ldp gold releases DIR [--show RELEASE] [--json]` | List the releases of the gold tables of a folder, newest first, or the tables of one release. See [Release the gold tables](#release-the-gold-tables) | `gold.release.list_releases(folder)`, `current_release(folder)`, `snapshot_for(table, release)` |
305
+ | `ldp gold list DIR [--all-layers] [--json] [--strict]` | One row for each gold table of a folder, with what it means, who owns it, what one row is, its rows, when its data last changed, the status of its last run, the verdict of its last quality check and the release it is in. A table with no contract shows `no description`. `--all-layers` adds the other tables, and `--strict` exits with 1 when a gold table lacks part of its contract, for CI. See [Browse the gold tables](#browse-the-gold-tables) | `gold.catalog.collect(folder)` |
306
+ | `ldp gold export DIR TABLE DEST [--format csv\|parquet\|jsonl\|xlsx] [--release RN\|--live] [--json]` | Write one gold table, as the current release holds it, as a CSV, Parquet, JSONL or Excel file, which opens with no Python and no Iceberg. An Excel file has a second sheet, `about`, with the contract of the table, and needs the `excel` extra. See [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool) | `gold.serve.export_table(folder, table, dest)` |
307
+ | `ldp gold duckdb DIR --out FILE [--release RN\|--live] [--json]` | Write the whole gold layer as one DuckDB file, with a copy of each table of the current release, a comment on each table and column, a `_catalog` table and a `_metrics` table. It opens in the DuckDB command line, the DuckDB UI and the BI tools that read DuckDB. See [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool) | `gold.serve.write_duckdb_file(folder, out)` |
308
+ | `ldp gold show DIR TABLE [--runs N] [--no-samples] [--json]` | Describe one table: its contract, every column with its type and description, its last runs, the releases that include it, a few sample rows and an `ldp query` line that reads it | `gold.catalog.collect(folder, only=table, detail=True)` |
309
+ | `ldp gold docs DIR --out FILE [--format html\|md] [--all-layers] [--no-samples]` | Write the catalog as one self-contained page with a search box, or as Markdown for a MkDocs site | `gold.render.html_page(catalog)`, `gold.render.markdown_page(catalog)` |
310
+ | `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
311
+ | `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
312
+ | `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
296
313
  | `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
297
314
 
298
315
  ## Config schema
@@ -310,13 +327,16 @@ and a `metadata` block:
310
327
  | `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
311
328
  | `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
312
329
  | `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
330
+ | `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
331
+ | `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
313
332
 
314
333
  A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
315
334
  is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
316
- the second kind. A kind is written exactly as shown, so `workflow` fails with a message that names `Workflow`, and
317
- `Metrics`, `Report` and `Profiles` are reserved for later milestones. `ldp plan` validates each file of a folder as its
318
- kind, and `ldp query`, `ldp mcp --config` and `ldp dbt` read the tables of the `Pipeline` files only, so a folder can
319
- hold workflow files next to its configs. `ldp schema --kind Workflow` prints the schema of another kind.
335
+ the second kind and `Metrics` the third, see [Define metrics](#define-metrics). A kind is written exactly as shown, so
336
+ `workflow` fails with a message that names `Workflow`, and `Report` and `Profiles` are reserved for later milestones.
337
+ `ldp plan` validates each file of a folder as its kind, and `ldp query`, `ldp mcp --config` and `ldp dbt` read the
338
+ tables of the `Pipeline` files only, so a folder can hold workflow and metrics files next to its configs.
339
+ `ldp schema --kind Metrics` prints the schema of another kind.
320
340
 
321
341
  Partition transforms are `identity`, `year`, `month`, `day`, `hour`, `bucket[N]` and
322
342
  `truncate[W]`. On pyiceberg 0.10 and later, writing with any transform except `identity` needs the
@@ -388,9 +408,10 @@ ldp runs .ldp/events.jsonl # the workf
388
408
  ```
389
409
 
390
410
  - Each task has an `id`, an optional `depends_on`, and exactly one kind: `pipeline` (the path of a dataset config,
391
- relative to the workflow file), `dbt` (a dbt project, built for the window, see below) or `maintain` (a config's Iceberg
392
- table and the options of `ldp maintain`, a dry run unless `apply` is true). Later milestones add kinds without changing the
393
- file format.
411
+ relative to the workflow file), `dbt` (a dbt project, built for the window, see below), `maintain` (a config's Iceberg
412
+ table and the options of `ldp maintain`, a dry run unless `apply` is true) or `gold` (the gold tables of a folder, built
413
+ for the window, with a release cut at the end, see [Release the gold tables](#release-the-gold-tables)). Later
414
+ milestones add kinds without changing the file format.
394
415
  - Tasks run one at a time, in dependency order. A pipeline task runs with the window of the workflow, so its idempotency key
395
416
  is the one `ldp plan CONFIG --window` prints.
396
417
  - A task that fails, or that a quality check blocks, makes the tasks that depend on it `upstream_failed`, and they do not run.
@@ -539,10 +560,23 @@ snapshots. It is how tables built with SQL (or with dbt, through `ldp dbt`) reac
539
560
  }
540
561
  ```
541
562
 
542
- Use `relation` (`table`, `schema.table` or `catalog.schema.table`) or `sql`, not both. The file is
543
- opened read-only and closed before the Iceberg write starts, and a `sql` source must be exactly one
544
- `SELECT` or `WITH` query. It needs the `duckdb` extra. `ldp plan` reports a missing file or a
545
- source with both or neither of `relation` and `sql`.
563
+ Use `relation` (`table`, `schema.table` or `catalog.schema.table`), `sql` or `query_file`, one of them. The file is
564
+ opened read-only and closed before the Iceberg write starts, and a query must be exactly one `SELECT` or `WITH` query.
565
+ A `query_file` is a file that holds the query, in place of `sql`. It needs the `duckdb` extra. `ldp plan` reports a
566
+ missing file or a source with more than one or none of the three.
567
+
568
+ A source can also have `inputs` in place of `path`, which is a list of dataset configs. It then reads no file. It
569
+ opens an in-memory DuckDB, registers the Iceberg table that each input config writes under that table's name, with the
570
+ session time zone set to UTC, and runs the query on them. This is how a gold table is one SQL file over silver tables,
571
+ and [`ldp gold build`](#build-the-gold-tables) builds them in order.
572
+
573
+ ```json
574
+ "source": {"name": "revenue_by_restaurant", "format": "DUCKDB", "query_file": "models/revenue_by_restaurant.sql",
575
+ "inputs": ["silver_orders.json"]}
576
+ ```
577
+
578
+ A query that reads a table that is not in `inputs` fails with DuckDB's own unknown table error, so `inputs` is the whole
579
+ lineage of the table, and every input is an input dataset in the run's OpenLineage events.
546
580
 
547
581
  ### An Excel sheet as a source
548
582
 
@@ -572,6 +606,415 @@ sheet. It needs the `excel` extra, and `ldp plan` reports a missing file or a `.
572
606
  atomically, so a folder that Google Drive syncs never holds a half-written file; see the recipe
573
607
  [Excel to CSV in a Google Drive folder](docs/recipes.md#excel-to-csv-in-a-google-drive-folder).
574
608
 
609
+ ### Table contracts
610
+
611
+ A table says what it means, who owns it, what one row is and what each column holds, in a `metadata.docs` block next to
612
+ the pipeline that builds it. After a run publishes, LDP writes the block into the Iceberg table, as the properties
613
+ `comment`, `ldp.owner`, `ldp.layer`, `ldp.grain` and `ldp.primary-key` and the doc of each column. Anyone who finds the
614
+ table reads it there, in `ldp mcp`, in DuckDB or in any other client.
615
+
616
+ ```json
617
+ "docs": {
618
+ "layer": "gold",
619
+ "description": "Orders and revenue for each restaurant on each day it had an order.",
620
+ "owner": "restaurant operations",
621
+ "grain": "one row for each restaurant on each day it had an order",
622
+ "primary_key": ["restaurant_id", "day"],
623
+ "columns": {"restaurant_id": "The restaurant.", "day": "The day.", "orders": "How many orders.",
624
+ "revenue": "What they cost."}
625
+ }
626
+ ```
627
+
628
+ - A `gold` table has to keep its promise. It needs a `description`, a `grain` and an owner (`owner`, or the `who` of the
629
+ config). `ldp run` stops it before anything is written when a column has no description, or when `columns` describes
630
+ a column that the table does not have, and the exit status is 3, as for a failed quality check. A table of another
631
+ layer may document as much or as little as it likes.
632
+ - The contract is one metadata-only commit, so it adds no snapshot, and a table that already has it gets no commit. It
633
+ only adds and changes text, and a run that wrote nothing does not touch the table.
634
+ - `ldp gold sync CONFIG` writes the contract of a config into a table that exists, with no new data. With
635
+ `--dbt-manifest target/manifest.json` it reads the contract from the manifest, for a table that dbt built.
636
+ - `ldp dbt build` fills the block from the dbt manifest. A model that is published to the `gold` namespace is a gold
637
+ table, so it needs `owner` and `grain` in its meta, a description, and a description of each column.
638
+ - Editing the block does not change the spec hash. The design is in [the v0.1.4 contract](docs/design/v0_1_4.md).
639
+
640
+ ### Build the gold tables
641
+
642
+ A gold table is one SQL file over other tables. Its config has a `DUCKDB` source with `inputs` and a `query_file`, and
643
+ a `metadata.docs` block with `"layer": "gold"`. `ldp gold build DIR` finds every gold config of a folder, orders them by
644
+ their `inputs`, since a gold table may read another, and builds them one at a time.
645
+
646
+ ```bash
647
+ ldp gold build examples/robot_episodes/data
648
+ ```
649
+
650
+ ```text
651
+ gold_episode_stats built 96 rows snapshot 3844274472678295321 4 of 4 checks passed
652
+ gold_robot_daily built 16 rows snapshot 1985638678580193697 2 of 2 checks passed
653
+ gold build examples/robot_episodes/data: 2 built, 0 checked, 0 blocked, 0 failed, 0 not run in 0.6s
654
+ metrics: 1 semantic model and 5 metrics fit the gold tables
655
+ release r1 cut with 2 tables and 112 rows in robots
656
+ ```
657
+
658
+ The last line is the release that the build cuts, see [Release the gold tables](#release-the-gold-tables).
659
+
660
+ - A cycle between configs is refused before anything runs, and the message names it.
661
+ - The build stops at the first table that a gate blocks or that fails, and the tables after it are `not run`, because
662
+ they could read what it did not write. The exit status is 3 when a gate held a table back, and 1 on any other
663
+ failure. A table whose contract does not hold is held back by a gate too.
664
+ - A gold config with no `inputs`, such as a dbt model that `ldp dbt build` published or a table that Spark or a script
665
+ wrote, is built elsewhere. The build does not run it. It checks that the table exists and runs its quality checks on
666
+ the current snapshot, and it writes nothing. A failing check blocks the build, unless the config says
667
+ `on_failure` `warn`.
668
+ - The runs are recorded in `_ldp.runs` and `_ldp.quality_results`, through the `iceberg` sink, unless the config names
669
+ sinks of its own, so the freshness and the gate status of a gold table show in `describe_table` of `ldp mcp`.
670
+ - A folder with a `metrics.json` has its metrics checked, see [Define metrics](#define-metrics). A file that is not
671
+ valid stops the build before anything is built. When every table was built or checked, the names and expressions are
672
+ checked against the tables, and the build says so with a `metrics:` line. Metrics that do not fit cut no release and
673
+ the exit status is 1, and the readers still see the last release.
674
+
675
+ ### Release the gold tables
676
+
677
+ A gold table is rebuilt on its own, so two tables that are read at the same moment can come from different builds. A
678
+ release fixes that. When every gold table of a build was built or checked, `ldp gold build` cuts release `r1`, and then
679
+ `r2` and so on. A release names the snapshot that each gold table ended the build with, so a reader that reads the
680
+ current release sees tables that agree. A build that a gate blocks cuts no release, and the last release stays current.
681
+ The build says which release readers still see.
682
+
683
+ ```bash
684
+ ldp gold build examples/robot_episodes/data # builds, and then cuts a release
685
+ ldp gold releases examples/robot_episodes/data # the releases, newest first
686
+ ldp gold releases examples/robot_episodes/data --show r1
687
+ ldp query examples/robot_episodes/data "SELECT count(*) FROM robots.gold_episode_stats" --release r1
688
+ ```
689
+
690
+ ```text
691
+ gold_episode_stats built 96 rows snapshot 7902942047508908069 4 of 4 checks passed
692
+ gold_robot_daily built 16 rows snapshot 7453607785734378588 2 of 2 checks passed
693
+ gold build examples/robot_episodes/data: 2 built, 0 checked, 0 blocked, 0 failed, 0 not run in 0.6s
694
+ metrics: 1 semantic model and 5 metrics fit the gold tables
695
+ release r1 cut with 2 tables and 112 rows in robots
696
+ ```
697
+
698
+ - A release is a file in the warehouse, `<warehouse>/.ldp/releases/<namespace>/r1.json`. It lists each table with its
699
+ snapshot, the dataset version that pins it, the run that built it and its rows. The file is written to a temporary
700
+ path and renamed, so a reader sees all of it or none of it.
701
+ - Each table is pinned with `datasets.pin` as the dataset `<namespace>.<table>`, with the property `release=r1`. A pin
702
+ tags its snapshot, and snapshot expiry never removes a tagged snapshot, so `ldp maintain` keeps every released
703
+ snapshot. That keeps its files too, so a table that is rebuilt in full for each release keeps a copy for each of them.
704
+ - A release covers one namespace of one warehouse, because that is where its file is. A folder whose gold tables are in
705
+ more than one is refused before anything is built. Give each its own folder, or build with `--no-release`.
706
+ - A build that finds the tables at the snapshots of the current release, as a workflow does when it runs a window that
707
+ it already ran, cuts no new release.
708
+ - `ldp query DIR SQL --release r1` reads the gold tables as r1 names them. A table that the release does not hold, and
709
+ a table in a namespace that has no releases, is read at its current snapshot. Without `--release`, `ldp query` reads the
710
+ tables as they are now.
711
+ - Code that reads the gold layer asks for the release with `local_data_platform.gold.release`. `current_release(dir)`
712
+ gives the release readers see, `snapshot_for(table, release)` gives the snapshot to read, and `register_release(engine,
713
+ dir)` registers every gold table in a `DuckDBEngine` at its snapshot in the release. Pass `release="r1"` for an older
714
+ one and `live=True` to read the tables as they are.
715
+ - `ldp runs` shows a release in the history of each table, as a run with the status `released`.
716
+
717
+ A workflow can build and release the gold layer as one of its tasks, after the pipelines and the dbt project that
718
+ feed it. The task is one line.
719
+
720
+ ```json
721
+ {"id": "gold", "depends_on": ["orders", "dbt"], "gold": {"dir": "config"}}
722
+ ```
723
+
724
+ The task builds the gold tables of `config` for the window of the run, as a pipeline task runs a config, so running the
725
+ same window again finds the tables already published and cuts no second release. It ends `blocked_quality` when a gate
726
+ holds a table back, and then no release is cut.
727
+
728
+ ### Browse the gold tables
729
+
730
+ A person who is not an engineer should find the right gold table without reading pipeline code, and so should an agent
731
+ that works for them. `ldp gold list` prints one row for each gold table of a folder, and `ldp gold show` describes one.
732
+
733
+ ```bash
734
+ ldp gold list examples/robot_episodes/data
735
+ ```
736
+
737
+ ```text
738
+ 2 gold tables in examples/robot_episodes/data
739
+ table description owner grain rows refreshed_utc run quality release
740
+ ------------------------- ------------------------------------------------ -------------------------- ------------------------- ---- ------------------- --------- ------------- -------
741
+ robots.gold_episode_stats One row for each promoted episode, with its o... robot learning researchers one row per episode 96 2026-10-16 09:17:03 published 4 of 4 passed r1
742
+ robots.gold_robot_daily Fleet health for each robot on each day: how ... fleet operations one row per robot per day 16 2026-10-16 09:17:03 published 2 of 2 passed r1
743
+ (2 rows)
744
+ ```
745
+
746
+ ```bash
747
+ ldp gold show examples/robot_episodes/data gold_robot_daily
748
+ ```
749
+
750
+ ```text
751
+ robots.gold_robot_daily (gold)
752
+ Fleet health for each robot on each day: how many episodes it recorded, how many succeeded, and how healthy its frames were.
753
+ Owner fleet operations
754
+ Grain one row per robot per day
755
+ Primary key robot_id, recorded_on
756
+ Refresh after each promotion
757
+ Tags robotics, fleet
758
+ Config gold_robot_daily.json
759
+ Rows 16
760
+ Refreshed 2026-10-16 09:17:03 UTC
761
+ Last run published, run 01a14400-2856-7074-a459-21ae4afcae71, 2026-10-16 09:17:03 UTC
762
+ Quality 2 of 2 passed
763
+ Release r1
764
+
765
+ Columns (9)
766
+ ...
767
+ Metrics
768
+ ...
769
+ Releases that include it (1)
770
+ ...
771
+ Read it
772
+ ldp query examples/robot_episodes/data "SELECT * FROM robots.gold_robot_daily LIMIT 10"
773
+ ```
774
+
775
+ `show` also prints every column with its type and description, the metrics defined on the table, the last 5 runs, the
776
+ releases that include the table and 5 sample rows. The last line is an `ldp query` line that reads the table, ready to copy. `TABLE` is a table with its
777
+ namespace, or its bare name when only one table has it, and it can be a table of any layer.
778
+
779
+ - The tables are the Iceberg tables that the dataset configs of the folder write. A gold table is one whose
780
+ `metadata.docs.layer` is `gold`, or whose Iceberg table says so, or that is in a namespace called `gold`.
781
+ `--all-layers` adds the bronze and silver tables and the tables that have no layer, and `--json` prints the same as
782
+ data.
783
+ - The contract is the one in the `metadata.docs` of the config and the one that the Iceberg table carries. Where both
784
+ have a value the config wins, because it is what the next run writes, and `show` says so when the table does not carry
785
+ it yet. A table with no contract shows `no description` and does not fail the listing, and its description is `null`
786
+ in JSON.
787
+ - `--strict` exits with 1 when a gold table has no description, no owner or no grain, or has a column without a
788
+ description, and it names the table and what it lacks. A table of another layer is never held to it. Run it in CI, so
789
+ that a gold table that nobody documented does not get in. A table that does not exist yet is listed as `not built`.
790
+ - The last run and the last quality check are what `describe_table` of `ldp mcp` reports, from the same code. A release is
791
+ not a run, so a table that was released still shows the status of its last build.
792
+ - `release` is the release that readers see now. `r1 (behind live)` says that the table was built again after r1, so
793
+ readers of the release do not see the newer data yet. The rows and the time are the table as it is now, and the time
794
+ leaves out the snapshots that `ldp maintain` makes.
795
+ - Nothing is written, not even a catalog or a namespace that does not exist yet.
796
+
797
+ `ldp gold docs` writes the same content as one page, or as Markdown for a MkDocs site.
798
+
799
+ ```bash
800
+ ldp gold docs examples/robot_episodes/data --out catalog.html
801
+ ldp gold docs examples/robot_episodes/data --out docs/catalog.md --format md
802
+ ```
803
+
804
+ ```text
805
+ wrote catalog.html, 2 tables, 18.8 KB. Open it as a page. It needs no network.
806
+ wrote docs/catalog.md, 2 tables, 7.4 KB. Open it as a Markdown file.
807
+ ```
808
+
809
+ - The page is one file with a search box, a light and a dark theme, and its CSS and its one script inline. It asks for
810
+ nothing from the network, and its Content-Security-Policy makes the browser refuse to try, so it opens from a folder or
811
+ from an e-mail. It is built with `local_data_platform.html`, which later pages share.
812
+ - Every value that a person wrote, such as the description of a table, is escaped. It can never become markup in the page,
813
+ and the Markdown escapes it too.
814
+ - The sample rows of each table are in the file. Leave them out with `--no-samples` when the file will be shared and the
815
+ rows must not be.
816
+ - In code, `local_data_platform.gold.catalog.collect(folder)` returns one entry for each table, and
817
+ `entry.document()` is the text of an entry for a search to index.
818
+
819
+ `ldp mcp` says the same things to an agent. `list_tables` returns the layer, description, owner and grain of each table, and
820
+ `describe_table` returns the contract. `ldp mcp --config DIR --layer gold` serves only the gold tables, so no tool and no
821
+ SQL can reach the tables upstream of them.
822
+
823
+ ### Define metrics
824
+
825
+ Revenue, orders and average ticket should mean one thing in every report, export and answer of an agent. A
826
+ `metrics.json` file next to the configs of the gold tables defines each number once, with what it means, who answers
827
+ for it and how a person reads it. The robot example has one, `examples/robot_episodes/configs/metrics.json`, and so has the
828
+ restaurant example, `examples/restaurant/config/metrics.json`, which defines `churn_rate` and `active_customers` on the
829
+ churn model of the warehouse demo.
830
+
831
+ ```json
832
+ {
833
+ "apiVersion": "ldp/v1", "kind": "Metrics", "identifier": "robot_fleet_metrics", "who": "fleet operations",
834
+ "semantic_models": [{
835
+ "name": "robot_daily", "table": "robots.gold_robot_daily",
836
+ "time_dimension": {"column": "recorded_on", "grains": ["day", "week", "month", "quarter", "year"]},
837
+ "dimensions": [{"name": "robot", "column": "robot_id"}],
838
+ "measures": [{"name": "episode_sum", "agg": "sum", "expr": "episodes"},
839
+ {"name": "success_sum", "agg": "sum", "expr": "successes"}]}],
840
+ "metrics": [
841
+ {"name": "episodes", "type": "simple", "measure": "episode_sum",
842
+ "description": "How many episodes the robots recorded."},
843
+ {"name": "success_rate", "type": "ratio", "numerator": "success_sum", "denominator": "episode_sum",
844
+ "format": "percent", "description": "The share of episodes that succeeded."}]
845
+ }
846
+ ```
847
+
848
+ A semantic model names one gold table, its `time_dimension` (a date column and the grains that its numbers may be
849
+ reported at), its `dimensions` and its `measures`. A measure is an aggregate, `sum`, `count`, `count_distinct`, `avg`,
850
+ `min` or `max`, over a SQL expression of the columns of the table. A metric is `simple`, which is one measure, or a
851
+ `ratio`, which is one measure over another. It has a `description`, an `owner` (its own, or the `who` of the file) and
852
+ a `format` of `number`, `percent` or `currency`. The terms follow MetricFlow, so an exporter can follow later.
853
+ `ldp plan DIR` validates the file as `Metrics`, and `ldp schema --kind Metrics` prints its schema.
854
+
855
+ ```bash
856
+ ldp metrics list examples/robot_episodes/data
857
+ ldp metrics query examples/robot_episodes/data episodes,success_rate,frame_drop_rate --by robot --grain week
858
+ ```
859
+
860
+ ```text
861
+ metric description owner format table dimensions grains
862
+ ----------------- ------------------------------------------------ -------------------------- ------- ----------------------- ---------- -------------------------------
863
+ episodes How many episodes the robots recorded. fleet operations number robots.gold_robot_daily robot day, week, month, quarter, year
864
+ success_rate The share of episodes that succeeded. fleet operations percent robots.gold_robot_daily robot day, week, month, quarter, year
865
+ frame_drop_rate The share of frames that were dropped. fleet operations percent robots.gold_robot_daily robot day, week, month, quarter, year
866
+ worst_skew_ms The largest RGB to depth gap of any frame, in... fleet operations number robots.gold_robot_daily robot day, week, month, quarter, year
867
+ active_robot_days Days with a recording, added up over robots. robot learning researchers number robots.gold_robot_daily robot day, week, month, quarter, year
868
+ (5 rows)
869
+
870
+ week robot episodes success_rate frame_drop_rate
871
+ ---------- ----- -------- ------------ ---------------
872
+ 2026-08-31 h01 12 83.3% 0.3%
873
+ 2026-08-31 h02 12 75.0% 0.5%
874
+ 2026-08-31 h03 12 83.3% 0.3%
875
+ 2026-08-31 h04 12 91.7% 0.4%
876
+ 2026-08-31 h05 12 41.7% 0.4%
877
+ 2026-08-31 h06 12 66.7% 0.3%
878
+ 2026-08-31 h07 12 75.0% 0.4%
879
+ 2026-08-31 h08 12 66.7% 0.0%
880
+ (8 rows)
881
+ read release r1, cut 2026-10-20 08:34:32 UTC
882
+ ```
883
+
884
+ `--filter dimension=value` keeps the rows of a dimension, `--from` and `--to` take days that are both included, and
885
+ `--sql` prints what would run, with the value of each parameter, and runs nothing.
886
+
887
+ ```bash
888
+ ldp metrics query examples/robot_episodes/data success_rate --filter robot=h05 --grain month --from 2026-09-01 --sql
889
+ ```
890
+
891
+ ```text
892
+ SELECT
893
+ CAST(date_trunc('month', "recorded_on") AS DATE) AS "month",
894
+ sum(successes) / NULLIF(sum(episodes), 0) AS "success_rate"
895
+ FROM "robots"."gold_robot_daily"
896
+ WHERE "robot_id" = $1 AND "recorded_on" >= $2
897
+ GROUP BY 1
898
+ ORDER BY 1
899
+ -- $1 = "h05"
900
+ -- $2 = DATE '2026-09-01'
901
+ ```
902
+
903
+ - **A ratio is a ratio of sums.** `success_rate` is the sum of the successes over the sum of the episodes of the group,
904
+ and never the average of the daily rates, which is a different number when the days have different counts. A group
905
+ with no episodes has no value and not infinity.
906
+ - **A value is never SQL.** A filter value, a date and every other value that a person gives is bound as a parameter of
907
+ the type of its column, so `--filter robot="h05' OR '1'='1"` matches no robot. Names go into the SQL only after they
908
+ were checked against the definitions, and an unknown metric, dimension or grain is refused with the valid names and the
909
+ closest one.
910
+ - **A query reads one table.** Metrics of two semantic models are refused with the table of each, because that would need
911
+ a join, which this version does not have.
912
+ - **`grains` is where the numbers are right.** A number that counts customers at the start of a month is wrong at any
913
+ other grain, so its model allows only `month`, and another grain is refused. A query with no grain adds the whole
914
+ period into one row, which is the same step as adding up years, so it needs `year` to be allowed. The churn of the
915
+ restaurant example is such a number. After `make demo-warehouse`,
916
+ `ldp metrics query ldp_demo/restaurant/config churn_rate,active_customers --grain month` prints it for every month,
917
+ and the [recipe](docs/recipes.md) for monthly churn from an orders table shows the definition in SQL.
918
+ - **The format is for people.** `percent` shows `0.5` as `50.0%` and `currency` has two decimals, on a terminal. `--raw`
919
+ shows what was computed, and `--out FILE`, `--json` and the Python function always give the values as computed.
920
+ `--out` writes CSV, Parquet or JSONL by the ending of the name.
921
+ - **The tables are read as the current release holds them.** `--release r1` reads an older one and `--live` reads the
922
+ tables as they are now, as `ldp query` does. The line under the table says which was read. A release pins the data and
923
+ not the definitions, so an old release that is read with a definition that needs a column that it lacks fails with a
924
+ message that says so.
925
+ - **The definitions are checked against the tables.** `ldp metrics validate DIR` checks that every table is a gold table
926
+ that exists, that every dimension and the time column are columns, that the time column is a `DATE` or a `TIMESTAMP`,
927
+ that each expression is one plain SQL expression that works on the table, and that every metric is a number. It lists
928
+ every problem and exits with 1. `ldp gold build` runs the same check and cuts no release when metrics do not fit.
929
+ - **In code**, `local_data_platform.metrics.query(folder, "episodes,success_rate", by="robot", grain="week")` returns a
930
+ `pyarrow.Table`, and `metrics.compile_for(...)` returns the SQL and its parameters without running anything.
931
+
932
+ `ldp mcp` has two more tools for the same numbers, `list_metrics` and `query_metric`, see
933
+ [MCP server for agents](#mcp-server-for-agents). The gold catalog shows the metrics of each table, in `ldp gold show`,
934
+ on the page of `ldp gold docs`, and in its search.
935
+
936
+ ### Open the gold tables in a spreadsheet or a BI tool
937
+
938
+ A person who works in a spreadsheet or a desktop SQL tool should be able to use a gold table without Python, Iceberg or an
939
+ engineer. `ldp gold export` writes one table as a file, and `ldp gold duckdb` writes the whole gold layer as one DuckDB file.
940
+ Both read the gold layer as every reader does, so by default it is the current release, `--release r1` reads an older one
941
+ and `--live` reads the tables as they are now.
942
+
943
+ ```bash
944
+ ldp gold export examples/robot_episodes/data gold_robot_daily daily.csv
945
+ ldp gold export examples/robot_episodes/data gold_robot_daily daily.xlsx
946
+ ```
947
+
948
+ ```text
949
+ wrote daily.csv, 16 rows of robots.gold_robot_daily, read from release r1, cut 2026-10-23 11:26:36 UTC, snapshot 1317331787740633770
950
+ wrote daily.xlsx, 16 rows of robots.gold_robot_daily, read from release r1, cut 2026-10-23 11:26:36 UTC, snapshot 1317331787740633770
951
+ ```
952
+
953
+ The format comes from the end of the name, or from `--format`.
954
+
955
+ - **CSV, Parquet and JSONL.** The three are written by `datasets.write_rows`, the same function as `ldp datasets export`,
956
+ which has `--format csv` too. A CSV has a header and a line for each row. A list, a struct or a map is written as JSON text,
957
+ bytes as hexadecimal and a number that is not finite as no value. A Parquet file has the table, release and snapshot in its
958
+ metadata, under `ldp.gold`. A name that starts with `s3://` or `gs://` is written there.
959
+ - **Excel.** The file needs the `excel` extra, `pip install "local-data-platform[excel]"`, and says which if it is missing.
960
+ The first sheet has the rows, with a bold header that stays in view and a filter. The second sheet is called `about` and
961
+ has the description, owner, grain and key of the table, its release, snapshot id and build time, its rows, and every
962
+ column with a word for its type and its description. A text is always a text and never a formula, even when it starts
963
+ with `=`. A time with a zone is written in UTC, because a sheet has no zones. An integer beyond 2^53 is written as text,
964
+ because a sheet would round it. A table with more than 1,048,575 rows does not fit a sheet, with its header row, and is
965
+ refused before it is read, with a hint to use CSV. A table with more than 16,384 columns and a value of more than 32,767
966
+ characters are refused too, with the same hint, when the file is written, and no file is left.
967
+ - **A CSV and a formula.** Excel runs a CSV value that starts with `=`, `+`, `-` or `@` as a formula when a person opens
968
+ the file. LDP writes the values as they are, so use Excel for text that people typed and that you do not trust.
969
+ - **A table that the release does not hold.** When the current release was cut before the table existed, the table is read
970
+ as it is, and the line says why. `--release r1` for a table that r1 does not hold is refused.
971
+
972
+ ```bash
973
+ ldp gold duckdb examples/robot_episodes/data --out gold.duckdb
974
+ ```
975
+
976
+ ```text
977
+ wrote gold.duckdb, 2 tables and 5 metrics, from release r1, cut 2026-10-23 11:26:36 UTC, 1292.0 KB
978
+ ```
979
+
980
+ The file has a copy of every gold table of the release, in the `main` schema under its own name, so `SELECT * FROM
981
+ gold_robot_daily` works. It has no schema called `gold`, because DuckDB names a file's database after the file, and
982
+ `gold.daily_sales` in a file called `gold.duckdb` would be ambiguous. Three more things are in it.
983
+
984
+ - **Comments.** `COMMENT ON TABLE` has the description of each table and `COMMENT ON COLUMN` the description of each column,
985
+ from the contract, so a tool that shows comments shows what the data means.
986
+ - **`_catalog`.** One row for each table with `table_name`, `description`, `owner`, `grain`, `release`, `snapshot_id` and
987
+ `built_at`, which is when that snapshot was written, in UTC.
988
+ - **`_metrics`.** One row for each metric of `metrics.json` with its `metric`, `description`, `owner`, `type`, `format`,
989
+ `table_name`, `definition`, `dimensions`, `time_column` and `grains`. The `definition` is the SQL that computes it from
990
+ its table, such as `sum(dropped_frames) / NULLIF(sum(frames), 0)`. The table is empty when the folder has no metrics file.
991
+
992
+ Open it in the DuckDB command line, with `duckdb gold.duckdb`, or in the DuckDB UI, with `duckdb -ui gold.duckdb`. The UI
993
+ fetches its web pages from ui.duckdb.org, so it needs the network.
994
+
995
+ ```sql
996
+ SELECT table_name, owner, grain, release FROM _catalog;
997
+ SELECT metric, definition FROM _metrics;
998
+ SELECT robot_id, round(avg(success_rate), 2) AS success_rate FROM gold_robot_daily GROUP BY 1 ORDER BY 1;
999
+ ```
1000
+
1001
+ - **It needs no extension.** The file has plain tables, so a DuckDB with no extension reads it, and no `iceberg_scan` is in
1002
+ it. It is written in the storage version of DuckDB 1.0, so a DuckDB 1.x opens it as well as a DuckDB 2.0. DuckDB 2.0
1003
+ writes a newer version by default, which a DuckDB 1.x cannot open.
1004
+ - **A reader never sees half a file.** It is written to a temporary path next to the file and renamed. A failure leaves the
1005
+ file that was there, and no temporary file.
1006
+ - **It holds one release.** A gold table that the release does not hold is left out, and the output names it, so that the
1007
+ file is as consistent as the release is. With `--live` it holds every table as it is now and `_catalog.release` has no
1008
+ value.
1009
+ - **`ldp gold build DIR --duckdb gold.duckdb`** writes the file after the build cuts a release. When the build cuts none,
1010
+ because the tables are at the snapshots of the current release, the file is written only if it is missing or holds
1011
+ another release. A build that a gate blocks or that fails leaves the file alone, so the file that readers have is the
1012
+ one of the last good release. A file that cannot be written after the release was cut is an error with exit status 1,
1013
+ and the release stays cut. `--duckdb` cannot be used with `--no-release`.
1014
+ - **Windows.** Replacing a file that a program has open fails on Windows. `{release}` in the name is replaced by the release,
1015
+ so `--out "gold_{release}.duckdb"` writes a new file for each release and never replaces one.
1016
+ - **Memory.** A table is read whole before it is written, as `ldp datasets export` does, so a gold table has to fit in memory.
1017
+
575
1018
  ## Catalogs and object storage
576
1019
 
577
1020
  The catalog block's `type` picks the catalog. It defaults to `local`, so a catalog block without
@@ -783,6 +1226,12 @@ A table or view model overwrites its table, and an incremental model with a `uni
783
1226
  published when dbt fails. Everything after the project folder goes to dbt unchanged, and the options of `ldp dbt`
784
1227
  itself go before it.
785
1228
 
1229
+ A published model also carries what the project says about it. Its description, the description of each column, its
1230
+ tags, its `owner` and `grain` meta and its layer become the table's [contract](docs/design/v0_1_4.md) and go into the
1231
+ Iceberg table. A model that is published to the `gold` namespace is a gold table, so it needs an owner, a grain, a
1232
+ description and a description of every column, and `ldp dbt build` stops before it publishes anything if one is
1233
+ missing.
1234
+
786
1235
  - The [warehousing guide](docs/warehousing.md) walks through it on a restaurant chain's Excel and CSV exports, from
787
1236
  the raw tables to tested models, churn, an incremental load and time travel. `make demo-warehouse` runs the same
788
1237
  example offline in about 25 seconds.
@@ -797,9 +1246,21 @@ make demo-orchestration # the restaurant's nightly drops, a bad row, a backf
797
1246
  ```
798
1247
 
799
1248
  - The [orchestration guide](docs/orchestration.md) walks through it on the restaurant example, command by command, from a window and its key to a workflow file, a backfill that stops at a bad row, a replaced day, a retried task, launchd and cron, and Airflow, Dagster and Prefect.
800
- - `make demo-orchestration` runs `examples/restaurant/run.py --layer orchestration`. It drops three nights of orders into a folder, with a bad row in the third. `ldp backfill` publishes two days and a data gate holds the third back, with exit status 3. After the fix, the same command runs only the third, and a third run skips every day. `ldp schedule tick` then loads the fourth night, and `ldp runs`, `ldp commits`, `ldp schedule list` and `ldp schedule install` show what happened. It needs no extra, runs the `models` task when the `dbt` extra is installed, and prints a skip line when it is not.
1249
+ - `make demo-orchestration` runs `examples/restaurant/run.py --layer orchestration`. It drops three nights of orders into a folder, with a bad row in the third. `ldp backfill` publishes two days and a data gate holds the third back, with exit status 3. After the fix, the same command runs only the third, and a third run skips every day. `ldp schedule tick` then loads the fourth night, and `ldp runs`, `ldp commits`, `ldp schedule list` and `ldp schedule install` show what happened. It needs no extra, runs the `models` and `gold` tasks when the `dbt` extra is installed, and prints a skip line when it is not.
801
1250
  - The pieces are described above, under [Windows](#windows), [Workflow files](#workflow-files), [Backfill a range of windows](#backfill-a-range-of-windows), [Run on a schedule](#run-on-a-schedule) and [Run from Airflow](#run-from-airflow), and the exit statuses are in the [table](#use-it-from-the-command-line) of the command line section.
802
1251
 
1252
+ ## Gold layer
1253
+
1254
+ A gold table is a table that people read. It says who owns it, what it means, what one row is and what each column holds, it is checked before it is written, and all the gold tables are published together as one release, so every reader sees the same build. A metric such as the average ticket is defined once. People who do not write code get the tables as a catalog page, an Excel file and one DuckDB file, and an AI agent can be limited to the gold tables.
1255
+
1256
+ ```bash
1257
+ make demo-gold # the restaurant's data mart, a faulty batch that the gate stops, a release, a metric and files for people
1258
+ ```
1259
+
1260
+ - The [gold layer guide](docs/gold.md) walks through it on the restaurant example, command by command, from the three layers and a gate that stops a bad batch to the contract of a gold table, a release that a gate can hold back, the catalog, metrics, files for people and `ldp mcp --layer gold`. It has a section for people who are not engineers, and every command in it runs in the tests.
1261
+ - `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
1262
+ - The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
1263
+
803
1264
  ## MCP server for agents
804
1265
 
805
1266
  `local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
@@ -825,6 +1286,8 @@ on stdio and read-only. Install the `mcp` extra, then point your MCP client at `
825
1286
  | `sample_rows(table, n)` | A sample of rows |
826
1287
  | `table_history(table)` | The table's snapshot history |
827
1288
  | `get_dataset(name)` | A pinned dataset version |
1289
+ | `list_metrics()` | The metrics that the `metrics.json` files of the `--config` folders define, each with what it means, its owner, format, table, dimensions and grains |
1290
+ | `query_metric(metrics, by, grain, filters, start, end, max_rows)` | Some metrics of one table, computed from the current release of the gold tables as the definitions say, with every value bound as a parameter |
828
1291
 
829
1292
  The guardrails are on by default. A query must be exactly one `SELECT` or `WITH` statement, as
830
1293
  DuckDB's parser sees it, so `COPY`, `ATTACH`, `INSTALL`, `SET`, `read_csv('/etc/passwd')` and
@@ -834,7 +1297,10 @@ Results are capped in rows and time, `--allow` limits the tables, and every call
834
1297
  tool, the SQL, row counts, duration and any error, never the result data. Records go to a JSONL file
835
1298
  as each call ends, and to the `_ldp.audit` Iceberg table in the first served catalog when the
836
1299
  server stops (`--no-iceberg-audit` turns that off).
837
- `make demo-agent` runs a scripted client against the demo table. See
1300
+ `query_metric` runs in the same sandbox as `query`, with the same row cap, timeout and audit, and its audit record
1301
+ has the arguments and the compiled SQL. It reads the current release of the gold tables, and `query` reads the tables
1302
+ as they are now. `python examples/agent_client.py` builds a small lake with a gold table, three metrics and a release,
1303
+ and calls both tools. `make demo-agent` runs a scripted client against the demo table. See
838
1304
  [MCP server for agents](docs/agents.md).
839
1305
 
840
1306
  ## Reproducible datasets and robotics
@@ -851,7 +1317,7 @@ episodes = Iceberg("gold_episode_stats", {"identifier": "robots", "warehouse_pat
851
1317
  train = datasets.pin(episodes, "humanoid_train", row_filter="success = true",
852
1318
  selected_fields=["episode_id", "robot_id", "task", "frame_count"])
853
1319
  rows = datasets.load(train) # the same rows at any later time
854
- datasets.export(train, "exports/humanoid_train_v1.parquet") # or format="jsonl"
1320
+ datasets.export(train, "exports/humanoid_train_v1.parquet") # or format="jsonl" or "csv"
855
1321
  ```
856
1322
 
857
1323
  Each pin also tags the snapshot (`ldp_ds_<name>_v<N>`), so snapshot expiry keeps it.
@@ -972,8 +1438,9 @@ make install # create .venv and install the package with the dev and docs e
972
1438
  make lint # flake8
973
1439
  make test # pytest, offline: the Spark and REST integration tests skip
974
1440
  make demo # run the demo into ./ldp_demo
975
- make demo-all # every demo; demo-warehouse needs the dbt and excel extras, demo-spark and demo-rest need scala-cli
1441
+ make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
976
1442
  make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
1443
+ make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
977
1444
  make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
978
1445
  make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
979
1446
  make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
@@ -1019,8 +1486,8 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1019
1486
  with no rows, leaves the models before it published. A model published with an upsert that gains or loses
1020
1487
  a column needs one `ldp run CONFIG --mode overwrite`.
1021
1488
  - **The restaurant example is in the repo, not in the package.** `pip install` does not include
1022
- `examples/`, so the warehousing and orchestration guides and `make demo-warehouse` and `make demo-orchestration`
1023
- need a clone.
1489
+ `examples/`, so the warehousing and orchestration guides and `make demo-warehouse`, `make demo-orchestration` and
1490
+ `make demo-gold` need a clone.
1024
1491
  - **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
1025
1492
  respects DuckDB's single writer. There are no parallel tasks.
1026
1493
  - **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
@@ -1030,16 +1497,19 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1030
1497
  model that reads them is limited to the window, and the models of the restaurant example are not.
1031
1498
  - **Airflow runs one config for each task.** DAGs built from workflow files, an `LdpTaskOperator` and managed Airflow
1032
1499
  recipes are planned for 0.2.0.
1500
+ - **A release covers one namespace of one warehouse.** `ldp gold build` refuses a folder whose gold tables are in more
1501
+ than one place, and says to give each place its own folder of configs.
1502
+ - **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
1503
+ question, so ask for them in separate queries.
1033
1504
 
1034
1505
  ## Roadmap
1035
1506
 
1036
1507
  From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
1037
- of a data platform, and every layer runs on a laptop first. 0.1.3 Orchestration is released, and these
1508
+ of a data platform, and every layer runs on a laptop first. 0.1.4 Self Serving Gold Layer is released, and these
1038
1509
  are the milestones after it:
1039
1510
 
1040
1511
  | Version | Milestone | What it adds |
1041
1512
  |---|---|---|
1042
- | 0.1.4 | Self Serving Gold Layer | Gold tables with owners and docs, releases, metrics, a catalog and exports |
1043
1513
  | 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
1044
1514
  | 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
1045
1515
  | 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |