local-data-platform 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. local_data_platform-0.1.4/PKG-INFO +1535 -0
  2. local_data_platform-0.1.4/README.md +1477 -0
  3. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/pyproject.toml +4 -1
  4. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/__init__.py +1 -1
  5. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/catalog/provider.py +38 -4
  6. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/cli.py +128 -25
  7. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/config.py +63 -4
  8. local_data_platform-0.1.4/src/local_data_platform/contracts.py +479 -0
  9. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/datasets.py +74 -8
  10. local_data_platform-0.1.4/src/local_data_platform/dbt_layer/build.py +349 -0
  11. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/cli.py +13 -95
  12. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/project.py +21 -4
  13. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/publish.py +87 -6
  14. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/sources.py +41 -3
  15. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/engine/duckdb/__init__.py +4 -4
  16. local_data_platform-0.1.4/src/local_data_platform/etl.py +275 -0
  17. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/events.py +170 -15
  18. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/exceptions.py +17 -0
  19. local_data_platform-0.1.4/src/local_data_platform/filelock.py +99 -0
  20. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/iceberg/__init__.py +50 -40
  21. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/iceberg/commit.py +115 -8
  22. local_data_platform-0.1.4/src/local_data_platform/gold/__init__.py +9 -0
  23. local_data_platform-0.1.4/src/local_data_platform/gold/build.py +424 -0
  24. local_data_platform-0.1.4/src/local_data_platform/gold/catalog.py +595 -0
  25. local_data_platform-0.1.4/src/local_data_platform/gold/cli.py +428 -0
  26. local_data_platform-0.1.4/src/local_data_platform/gold/release.py +698 -0
  27. local_data_platform-0.1.4/src/local_data_platform/gold/render.py +414 -0
  28. local_data_platform-0.1.4/src/local_data_platform/gold/serve.py +860 -0
  29. local_data_platform-0.1.4/src/local_data_platform/gold/status.py +274 -0
  30. local_data_platform-0.1.4/src/local_data_platform/gold/sync.py +133 -0
  31. local_data_platform-0.1.4/src/local_data_platform/html.py +294 -0
  32. local_data_platform-0.1.4/src/local_data_platform/integrations/__init__.py +6 -0
  33. local_data_platform-0.1.4/src/local_data_platform/integrations/airflow.py +249 -0
  34. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/lake.py +51 -11
  35. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/cli.py +60 -13
  36. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/__init__.py +1 -1
  37. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/cli.py +29 -6
  38. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/discovery.py +67 -16
  39. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/guard.py +64 -10
  40. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/sandbox.py +99 -5
  41. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/tools.py +191 -179
  42. local_data_platform-0.1.4/src/local_data_platform/metrics/__init__.py +61 -0
  43. local_data_platform-0.1.4/src/local_data_platform/metrics/check.py +221 -0
  44. local_data_platform-0.1.4/src/local_data_platform/metrics/cli.py +180 -0
  45. local_data_platform-0.1.4/src/local_data_platform/metrics/compiler.py +427 -0
  46. local_data_platform-0.1.4/src/local_data_platform/metrics/load.py +258 -0
  47. local_data_platform-0.1.4/src/local_data_platform/metrics/model.py +621 -0
  48. local_data_platform-0.1.4/src/local_data_platform/metrics/render.py +180 -0
  49. local_data_platform-0.1.4/src/local_data_platform/metrics/runner.py +177 -0
  50. local_data_platform-0.1.4/src/local_data_platform/metrics/sql.py +213 -0
  51. local_data_platform-0.1.4/src/local_data_platform/orchestration/__init__.py +73 -0
  52. local_data_platform-0.1.4/src/local_data_platform/orchestration/backfills.py +541 -0
  53. local_data_platform-0.1.4/src/local_data_platform/orchestration/cli.py +294 -0
  54. local_data_platform-0.1.4/src/local_data_platform/orchestration/dbt_task.py +395 -0
  55. local_data_platform-0.1.4/src/local_data_platform/orchestration/gold_task.py +197 -0
  56. local_data_platform-0.1.4/src/local_data_platform/orchestration/ledger.py +164 -0
  57. local_data_platform-0.1.4/src/local_data_platform/orchestration/recipes.py +134 -0
  58. local_data_platform-0.1.4/src/local_data_platform/orchestration/retries.py +182 -0
  59. local_data_platform-0.1.4/src/local_data_platform/orchestration/runner.py +394 -0
  60. local_data_platform-0.1.4/src/local_data_platform/orchestration/schedule.py +681 -0
  61. local_data_platform-0.1.4/src/local_data_platform/orchestration/schedule_cli.py +219 -0
  62. local_data_platform-0.1.4/src/local_data_platform/orchestration/tasks.py +607 -0
  63. local_data_platform-0.1.4/src/local_data_platform/orchestration/windows.py +646 -0
  64. local_data_platform-0.1.4/src/local_data_platform/orchestration/workflow.py +593 -0
  65. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/__init__.py +303 -34
  66. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/builders.py +13 -7
  67. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/duckdb_to_iceberg/__init__.py +3 -2
  68. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/pyarrow/__init__.py +3 -2
  69. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/registry.py +8 -2
  70. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/quality/checks.py +6 -3
  71. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/spec.py +611 -56
  72. local_data_platform-0.1.4/src/local_data_platform/store/source/duckdb/__init__.py +269 -0
  73. local_data_platform-0.1.2/PKG-INFO +0 -800
  74. local_data_platform-0.1.2/README.md +0 -743
  75. local_data_platform-0.1.2/src/local_data_platform/etl.py +0 -141
  76. local_data_platform-0.1.2/src/local_data_platform/store/source/duckdb/__init__.py +0 -155
  77. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/LICENSE +0 -0
  78. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/catalog/__init__.py +0 -0
  79. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/catalog/local/__init__.py +0 -0
  80. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/catalog/local/iceberg/__init__.py +0 -0
  81. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/dbt_layer/__init__.py +0 -0
  82. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/demo.py +0 -0
  83. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/engine/__init__.py +0 -0
  84. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/engine/duckdb/row_filter.py +0 -0
  85. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/engine/router.py +0 -0
  86. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/engine/spark/__init__.py +0 -0
  87. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/__init__.py +0 -0
  88. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/csv/__init__.py +0 -0
  89. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/excel/__init__.py +0 -0
  90. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/format/parquet/__init__.py +0 -0
  91. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/fs.py +0 -0
  92. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/github/__init__.py +0 -0
  93. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/issue/__init__.py +0 -0
  94. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/logger.py +0 -0
  95. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/__init__.py +0 -0
  96. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/_common.py +0 -0
  97. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/orphans.py +0 -0
  98. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/maintenance/snapshots.py +0 -0
  99. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/__main__.py +0 -0
  100. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/audit.py +0 -0
  101. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/errors.py +0 -0
  102. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/mcp_server/server.py +0 -0
  103. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/paths.py +0 -0
  104. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/builtin.py +0 -0
  105. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/__init__.py +0 -0
  106. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/csv_to_iceberg/__init__.py +0 -0
  107. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/iceberg_to_csv/__init__.py +0 -0
  108. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/egression/iceberg_to_parquet/__init__.py +0 -0
  109. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/__init__.py +0 -0
  110. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/bigquery_to_csv/__init__.py +0 -0
  111. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/csv_to_iceberg/__init__.py +0 -0
  112. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/excel_to_csv/__init__.py +0 -0
  113. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/excel_to_iceberg/__init__.py +0 -0
  114. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/pipeline/ingestion/parquet_to_iceberg/__init__.py +0 -0
  115. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/quality/__init__.py +0 -0
  116. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/quality/report.py +0 -0
  117. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/quality/runner.py +0 -0
  118. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/quality/temporal.py +0 -0
  119. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/__init__.py +0 -0
  120. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/__init__.py +0 -0
  121. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/gcp/__init__.py +0 -0
  122. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/gcp/bigquery/__init__.py +0 -0
  123. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/json/__init__.py +0 -0
  124. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/near/__init__.py +0 -0
  125. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/source/parquet/__init__.py +0 -0
  126. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/target/__init__.py +0 -0
  127. {local_data_platform-0.1.2 → local_data_platform-0.1.4}/src/local_data_platform/store/target/iceberg/__init__.py +0 -0
@@ -0,0 +1,1535 @@
1
+ Metadata-Version: 2.4
2
+ Name: local-data-platform
3
+ Version: 0.1.4
4
+ Summary: A Python API layer for running a complete Iceberg lakehouse data platform on your laptop.
5
+ License: MIT
6
+ License-File: LICENSE
7
+ Author: Tushar Choudhary
8
+ Requires-Python: >=3.12,<4
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: Programming Language :: Python :: 3.14
14
+ Classifier: Programming Language :: Python :: 3.15
15
+ Provides-Extra: bigquery
16
+ Provides-Extra: dbt
17
+ Provides-Extra: dev
18
+ Provides-Extra: docs
19
+ Provides-Extra: duckdb
20
+ Provides-Extra: excel
21
+ Provides-Extra: glue
22
+ Provides-Extra: mcp
23
+ Provides-Extra: s3
24
+ Provides-Extra: spark
25
+ Requires-Dist: boto3 (>=1.34) ; extra == "s3"
26
+ Requires-Dist: build (>=1.2) ; extra == "dev"
27
+ Requires-Dist: cronsim (>=2.6,<3)
28
+ Requires-Dist: dbt-core (>=1.12,<2) ; extra == "dbt"
29
+ Requires-Dist: dbt-duckdb (>=1.11,<2) ; extra == "dbt"
30
+ Requires-Dist: defusedxml (>=0.7) ; extra == "dev"
31
+ Requires-Dist: defusedxml (>=0.7) ; extra == "excel"
32
+ Requires-Dist: duckdb (>=1.1) ; extra == "dbt"
33
+ Requires-Dist: duckdb (>=1.1) ; extra == "dev"
34
+ Requires-Dist: duckdb (>=1.1) ; extra == "duckdb"
35
+ Requires-Dist: duckdb (>=1.1) ; extra == "mcp"
36
+ Requires-Dist: flake8 (>=7) ; extra == "dev"
37
+ Requires-Dist: google-auth (>=2.20) ; extra == "bigquery"
38
+ Requires-Dist: google-cloud-bigquery (>=3.20) ; extra == "bigquery"
39
+ Requires-Dist: mcp (>=2.2,<3) ; extra == "dev"
40
+ Requires-Dist: mcp (>=2.2,<3) ; extra == "mcp"
41
+ Requires-Dist: mkdocs (>=1.6.0) ; extra == "docs"
42
+ Requires-Dist: mkdocs-material (==9.5.21) ; extra == "docs"
43
+ Requires-Dist: mkdocstrings[python] (>=0.25.0) ; extra == "docs"
44
+ Requires-Dist: moto[server] (>=5.0) ; extra == "dev"
45
+ Requires-Dist: openpyxl (>=3.1) ; extra == "dev"
46
+ Requires-Dist: openpyxl (>=3.1) ; extra == "excel"
47
+ Requires-Dist: pyarrow (>=17)
48
+ Requires-Dist: pyiceberg[glue] (>=0.11,<0.13) ; extra == "glue"
49
+ Requires-Dist: pyiceberg[pyarrow,pyiceberg-core,sql-sqlite] (>=0.11,<0.13)
50
+ Requires-Dist: pymdown-extensions (>=10.8.1) ; extra == "docs"
51
+ Requires-Dist: pyspark (>=4.0,<4.2) ; extra == "spark"
52
+ Requires-Dist: pytest (>=8) ; extra == "dev"
53
+ Requires-Dist: requests (>=2.31)
54
+ Project-URL: Documentation, https://local-data-platform.readthedocs.io
55
+ Project-URL: Repository, https://github.com/tusharchou/local-data-platform
56
+ Description-Content-Type: text/markdown
57
+
58
+ # Local Data Platform
59
+
60
+ **local-data-platform** (`ldp`) is a Python library for running an Apache Iceberg lakehouse on your
61
+ laptop, and for taking the same pipelines to a shared catalog and object storage when you need to.
62
+ You describe a dataset in a JSON config, and `ldp` loads it into an Iceberg table, runs data quality
63
+ checks before the write, and lets you query the table with DuckDB SQL and read earlier snapshots.
64
+ The default setup needs no server and no cloud account.
65
+
66
+ It's for learning how a lakehouse works, for building and testing pipelines locally before you pay
67
+ for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
68
+ tables you build.
69
+
70
+ > **Status.** This is version 0.1.4, the Self Serving Gold Layer milestone. It adds gold tables that
71
+ > carry an owner, a meaning and a description of every column, `ldp gold build` and one release of all
72
+ > the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once in
73
+ > `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
74
+ > [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
75
+ > with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
76
+ > launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
77
+ > [guide](docs/orchestration.md). Version 0.1.2 added `ldp dbt`, which runs a dbt project over
78
+ > your Iceberg tables and publishes the models you mark back to Iceberg (see
79
+ > [Transform with dbt](#transform-with-dbt)), the `DUCKDB` and `XLSX` source formats, time travel and
80
+ > folders of tables in `ldp query`, retries for concurrent appends, and a restaurant example with a
81
+ > [guide](docs/warehousing.md). Version 0.1.1 was the first release on PyPI since 0.1.0. It carries
82
+ > the hardening work (contract in [`docs/design/v0_1_1.md`](docs/design/v0_1_1.md)) and the platform
83
+ > work ("multi-writer, any catalog, object storage, agent-ready", contract in
84
+ > [`docs/design/v0_1_1_platform.md`](docs/design/v0_1_1_platform.md)), which works from Python and
85
+ > through the [platform `ldp` commands](#platform-commands). The version number follows the
86
+ > milestones in the [roadmap](docs/roadmap.md).
87
+
88
+ ## Quickstart
89
+
90
+ `ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
91
+
92
+ ```bash
93
+ pip install "local-data-platform[duckdb]>=0.1.4"
94
+
95
+ ldp demo --workdir ldp_demo
96
+ ```
97
+
98
+ To work on the library itself, install it from a clone of this repo. `make install` creates
99
+ `.venv` and installs the package in editable mode with the `dev` and `docs` extras:
100
+
101
+ ```bash
102
+ git clone https://github.com/tusharchou/local-data-platform.git
103
+ cd local-data-platform
104
+ make install
105
+ source .venv/bin/activate
106
+
107
+ ldp demo --workdir ldp_demo
108
+ ```
109
+
110
+ `ldp demo` generates 1,000 synthetic taxi rides (always the same ones), then shows each feature
111
+ using only the public API:
112
+
113
+ 1. It loads a config from JSON.
114
+ 2. It appends twice, so you can see the duplicates, then overwrites twice to show that the load is
115
+ idempotent.
116
+ 3. It upserts on a key column.
117
+ 4. It writes a day-partitioned table.
118
+ 5. It runs quality checks that pass, then shows a bad batch being blocked before the write.
119
+ 6. It queries the table with DuckDB SQL.
120
+ 7. It reads an earlier snapshot (time travel).
121
+ 8. It exports the table back to CSV.
122
+
123
+ Everything it writes goes into the `--workdir` folder, so you can delete it afterwards. The
124
+ [quickstart guide](docs/quickstart.md) explains each step and how to write your own config.
125
+
126
+ More demos, each one a `make` target (see [Development](#development)):
127
+
128
+ | Target | What it shows |
129
+ |---|---|
130
+ | `make demo-robotics` | Humanoid-robot episodes and sensor frames in bronze, silver and gold Iceberg tables, temporal quality checks, a pinned training split and DuckDB research queries |
131
+ | `make demo-agent` | The read-only MCP server on the demo table, driven by a scripted agent client |
132
+ | `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
133
+ | `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
134
+ | `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
135
+ | `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
136
+ | `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
137
+ | `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
138
+
139
+ ### Extras
140
+
141
+ | Extra | Adds | You need it for |
142
+ |---|---|---|
143
+ | `duckdb` | DuckDB | SQL over Iceberg tables (`DuckDBEngine`, `ldp query`, the demo's SQL step) |
144
+ | `mcp` | The MCP SDK and DuckDB | The read-only MCP server for agents (`ldp mcp`) |
145
+ | `s3` | boto3 | SigV4-signed REST catalogs on AWS, such as S3 Tables. `s3://` IO itself needs nothing extra: pyarrow does it |
146
+ | `glue` | `pyiceberg[glue]` (boto3) | The AWS Glue catalog |
147
+ | `bigquery` | Google Cloud BigQuery client | BigQuery as a source |
148
+ | `spark` | PySpark | The experimental `SparkEngine`, which also needs a JDK 17+ |
149
+ | `excel` | openpyxl and defusedxml | Excel workbooks (`.xlsx`) as a source, for `ExcelToCSV` and `ExcelToIceberg` |
150
+ | `dbt` | dbt-core, the dbt-duckdb adapter and DuckDB | `ldp dbt`, which runs a dbt project over your Iceberg tables and publishes the models you mark back to Iceberg ([how it works](docs/design/warehousing.md)), and the tests marked `dbt` |
151
+ | `dev` | pytest, flake8, build, DuckDB, `moto[server]`, the MCP SDK, openpyxl | Working on the library |
152
+ | `docs` | MkDocs and plugins | Building the docs |
153
+
154
+ ## Use it from Python
155
+
156
+ A config names a source, a target and optional quality checks. Paths are relative to the config
157
+ file's folder. Save this as `rides.json`, next to a `data/rides.csv` with `ride_id`, `pickup_ts`
158
+ and `fare` columns:
159
+
160
+ ```json
161
+ {
162
+ "identifier": "rides",
163
+ "metadata": {
164
+ "source": {"name": "rides", "format": "CSV", "path": "data/rides.csv"},
165
+ "target": {
166
+ "name": "rides",
167
+ "format": "ICEBERG",
168
+ "catalog": {"identifier": "nyc", "warehouse_path": "warehouse"},
169
+ "write_mode": "upsert",
170
+ "join_cols": ["ride_id"],
171
+ "partition_by": [{"column": "pickup_ts", "transform": "day"}]
172
+ },
173
+ "quality": {
174
+ "on_failure": "fail",
175
+ "checks": [
176
+ {"check": "not_null", "columns": ["ride_id", "pickup_ts"]},
177
+ {"check": "unique", "columns": ["ride_id"]},
178
+ {"check": "range", "column": "fare", "min": 0}
179
+ ]
180
+ }
181
+ }
182
+ }
183
+ ```
184
+
185
+ Run it, query it and time-travel:
186
+
187
+ ```python
188
+ from local_data_platform import Config
189
+ from local_data_platform.engine.duckdb import DuckDBEngine
190
+ from local_data_platform.format.iceberg import Iceberg
191
+ from local_data_platform.pipeline.registry import create_pipeline
192
+
193
+ config = Config.from_json("rides.json")
194
+ result = create_pipeline(config).run() # picks CSVToIceberg from the config's formats
195
+ print(result.rows_read, result.rows_written, result.quality.passed)
196
+
197
+ table = Iceberg("rides", config.target["catalog"], base_dir=config.base_dir)
198
+
199
+ with DuckDBEngine() as duck:
200
+ duck.register_iceberg(table, "rides")
201
+ print(duck.query("SELECT count(*) AS n, avg(fare) AS avg_fare FROM rides").to_pylist())
202
+
203
+ first = table.snapshots()[0]["snapshot_id"]
204
+ print(table.get(snapshot_id=first).num_rows) # the table as of its first snapshot
205
+ ```
206
+
207
+ If a check fails and `on_failure` is `"fail"`, `run()` raises `DataQualityError` and writes
208
+ nothing; the failing checks are in `error.report`. With `"warn"` it logs them and writes anyway.
209
+
210
+ You can also build a pipeline in code, with your own transforms:
211
+
212
+ ```python
213
+ import pyarrow.compute as pc
214
+
215
+ from local_data_platform.format.csv import CSV
216
+ from local_data_platform.format.iceberg import Iceberg
217
+ from local_data_platform.pipeline import Pipeline
218
+ from local_data_platform.quality import NotNull, Unique
219
+
220
+ pipeline = Pipeline(
221
+ source=CSV("rides", "data/rides.csv"),
222
+ target=Iceberg("rides", {"identifier": "nyc", "warehouse_path": "warehouse"}, write_mode="overwrite"),
223
+ transforms=[lambda df: df.filter(pc.field("fare") >= 0)],
224
+ checks=[NotNull(["ride_id"]), Unique(["ride_id"])],
225
+ )
226
+ print(pipeline.run())
227
+ ```
228
+
229
+ ## Use it from the command line
230
+
231
+ ```bash
232
+ ldp --version
233
+ ldp pipelines # list the registered source/target routes
234
+ ldp run rides.json # run the config's pipeline
235
+ ldp run rides.json --mode overwrite # override the config's write_mode
236
+ ldp run rides.json --window 2026-08-01/2026-08-02 # staged and exactly once: a second run for the window skips
237
+ ldp run orders.json --window yesterday # a window by name: also last-week and last-month, in the config's time zone
238
+ ldp run rides.json --window 2026-08-01/2026-08-02 --run-id RUN_ID --attempt 2 # a retry of that run
239
+ ldp run rides.json --idempotency-key sales-2026-08-01 # a key of your own, for an orchestrator
240
+ ldp run rides.json --window 2026-08-01/2026-08-02 --json # the result as JSON on stdout, for scripts
241
+ ldp snapshots rides.json # list the target table's snapshots
242
+ ldp query rides.json "SELECT count(*) FROM rides" # SQL over the target, named after target.name
243
+ ldp query rides.json "SELECT * FROM rides" --max-rows 20 # print at most 20 rows (default 100)
244
+ ldp query rides.json "SELECT count(*) FROM rides" --snapshot 4437 # an older snapshot, from ldp snapshots
245
+ ldp query rides.json "SELECT count(*) FROM rides" --as-of "2026-07-31 23:59:59" # the newest snapshot at or before that time (UTC)
246
+ ldp query config/ "SELECT o.id FROM raw.orders o JOIN raw.customers c ON c.id = o.id" # a folder of configs: every table, joinable
247
+ ldp query config/ "SELECT count(*) FROM gold.daily_sales" --release r2 # the gold tables as release r2 names them
248
+ ldp demo --workdir /tmp/ldp_demo # also --rows N (default 1000) and --seed N (default 42)
249
+ ```
250
+
251
+ With `--window` or `--idempotency-key` a run is staged and exactly once. It prints its window, its
252
+ idempotency key, its run id and attempt, its status (`published` or `skipped_duplicate`) and the snapshot that holds
253
+ its rows, and running it again for the same window finds that snapshot and writes nothing. `ldp plan CONFIG --window W`
254
+ prints the same key, and `--idempotency-key auto` asks for it. The key is looked for in the whole history that the table
255
+ still holds, before the source is read, so a window that ran weeks ago is skipped without being read too. A retry
256
+ reuses the run id with the next attempt number,
257
+ and the new attempt fences the older ones, so a staged branch that an earlier attempt left behind never reaches `main`.
258
+ `--attempt` needs `--run-id`, and both need a window or a key. Without these options `ldp run` writes directly, as it
259
+ always did. A window is `START/END`, or `yesterday`, `last-week` (the last full Monday-to-Monday week) or `last-month`,
260
+ counted in the time zone of the config, and `ldp run` prints the window it stands for. What a window reads, and how a
261
+ config says so, is in [Windows](#windows).
262
+
263
+ Every command follows one exit-code table.
264
+
265
+ | Exit | Meaning |
266
+ |---|---|
267
+ | 0 | success, including a run that published and a run that skipped a duplicate |
268
+ | 1 | an error: a bad config, a usage error, a failed write |
269
+ | 2 | reserved for an SLA breach that `ldp monitor` reports in 0.1.5 |
270
+ | 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
271
+
272
+ A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
273
+ because the data has to change first. Add `-v` to see a traceback. The commands
274
+ that read a table (`snapshots`, `query`, `commits`, `maintain`, `datasets pin`, `catalog test`) fail
275
+ without creating anything when the config's catalog is a SQLite file that doesn't exist yet (a
276
+ `local` catalog, or `sql` on SQLite). `ldp run`, `ldp snapshots` and `ldp query` also work with the
277
+ other [catalog types](#catalogs-and-object-storage); they have been run against `sql` (SQLite) and
278
+ `rest` catalogs.
279
+
280
+ ### Platform commands
281
+
282
+ `ldp --help` lists them, and `ldp <command> --help` shows each one's options. Each is also a Python
283
+ call:
284
+
285
+ | Command | What it does | From Python |
286
+ |---|---|---|
287
+ | `ldp catalog test SPEC` | Connect to a catalog (a spec file, or a config's catalog block) and list its namespaces and tables | `catalog.provider.create_catalog(spec).list_namespaces()` |
288
+ | `ldp schema [--kind KIND]` | Print the `ldp/v1` JSON Schema of a kind of file, the dataset config by default | `spec.json_schema()`, `spec.kind_schema(kind)` |
289
+ | `ldp workflow plan FILE [--window W] [--json]` | Check a workflow file, or a dataset config with a schedule, and every config it names, and print its tasks in the order they run with each pipeline task's idempotency key for the window | `orchestration.plan_workflow(path, window=...)` |
290
+ | `ldp backfill FILE --from START [--to END] [--dry-run] [--replace] [--run-backwards] [--max-windows N] [--stop-on-failure] [--json]` | Run a workflow, or a config with a window, for every window of a range, oldest first, each pipeline under the key of its window. A window that is done is skipped without being read, so running it again resumes where it stopped; `--dry-run` lists the windows with their keys and states, and `--replace` writes them again under a replace key; exit status 0, 1, or 3 for a data gate | `orchestration.backfill(path, start, end, ...)` |
291
+ | `ldp workflow run FILE --window W [--json]` | Run the tasks of a workflow one at a time for a window. A failed or blocked task stops the tasks that depend on it and the rest still run, and a second run skips what published or was built; exit status 0, 1, or 3 for a data gate | `orchestration.run_workflow(path, window=...)` |
292
+ | `ldp schedule tick DIR [--now TIME] [--max-windows N] [--json]` | Run every window that is due and not done, for every workflow and config under DIR that has a schedule, then exit. cron or launchd calls it every 5 minutes. A window that failed is carried on by the next tick as the same run with the next attempt, and a tick that starts while another runs does nothing; exit status 0, 1, or 3 for a data gate | `orchestration.schedule.tick(dir, now=...)` |
293
+ | `ldp schedule list DIR [--now TIME] [--max-windows N] [--json]` | List every scheduled workflow and config under DIR with its cron expression, time zone, next fire in UTC and in local time, last window and how it ended, the windows that are due, and the windows it missed with the `ldp backfill` command that fills them | `orchestration.schedule.list_scheduled(dir)` |
294
+ | `ldp schedule install --print {cron,launchd} DIR [--ldp PATH]` | Print the crontab line or the launchd plist that ticks DIR every 5 minutes. It never writes to a system location | `orchestration.recipes.recipe(kind, dir)` |
295
+ | `ldp plan CONFIG [--window WINDOW] [--json]` | Validate a config (or a folder of files, each as its kind) and print its spec hash, route, target, sinks, schedule, and the source path and idempotency key of a window | `spec.validate_spec(data)`, `spec.plan(config, window=...)` |
296
+ | `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
297
+ | `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
298
+ | `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
299
+ | `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
300
+ | `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
301
+ | `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
302
+ | `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
303
+ | `ldp gold build DIR [--no-release] [--duckdb FILE] [--json]` | Build the gold tables of a folder in the order that their `inputs` say, and check the ones that are built elsewhere. One line for each table, and a gate that blocks a table stops the build with exit status 3. When every table passed, and the metrics of the folder fit them, it cuts a release, unless `--no-release` says not to. `--duckdb FILE` then writes the DuckDB file of the gold layer. See [Build the gold tables](#build-the-gold-tables) | `gold.build.build_gold(folder, release=True)`, `gold.serve.refresh_after_build(folder, out, result)` |
304
+ | `ldp gold releases DIR [--show RELEASE] [--json]` | List the releases of the gold tables of a folder, newest first, or the tables of one release. See [Release the gold tables](#release-the-gold-tables) | `gold.release.list_releases(folder)`, `current_release(folder)`, `snapshot_for(table, release)` |
305
+ | `ldp gold list DIR [--all-layers] [--json] [--strict]` | One row for each gold table of a folder, with what it means, who owns it, what one row is, its rows, when its data last changed, the status of its last run, the verdict of its last quality check and the release it is in. A table with no contract shows `no description`. `--all-layers` adds the other tables, and `--strict` exits with 1 when a gold table lacks part of its contract, for CI. See [Browse the gold tables](#browse-the-gold-tables) | `gold.catalog.collect(folder)` |
306
+ | `ldp gold export DIR TABLE DEST [--format csv\|parquet\|jsonl\|xlsx] [--release RN\|--live] [--json]` | Write one gold table, as the current release holds it, as a CSV, Parquet, JSONL or Excel file, which opens with no Python and no Iceberg. An Excel file has a second sheet, `about`, with the contract of the table, and needs the `excel` extra. See [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool) | `gold.serve.export_table(folder, table, dest)` |
307
+ | `ldp gold duckdb DIR --out FILE [--release RN\|--live] [--json]` | Write the whole gold layer as one DuckDB file, with a copy of each table of the current release, a comment on each table and column, a `_catalog` table and a `_metrics` table. It opens in the DuckDB command line, the DuckDB UI and the BI tools that read DuckDB. See [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool) | `gold.serve.write_duckdb_file(folder, out)` |
308
+ | `ldp gold show DIR TABLE [--runs N] [--no-samples] [--json]` | Describe one table: its contract, every column with its type and description, its last runs, the releases that include it, a few sample rows and an `ldp query` line that reads it | `gold.catalog.collect(folder, only=table, detail=True)` |
309
+ | `ldp gold docs DIR --out FILE [--format html\|md] [--all-layers] [--no-samples]` | Write the catalog as one self-contained page with a search box, or as Markdown for a MkDocs site | `gold.render.html_page(catalog)`, `gold.render.markdown_page(catalog)` |
310
+ | `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
311
+ | `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
312
+ | `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
313
+ | `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
314
+
315
+ ## Config schema
316
+
317
+ A config has an `identifier`, optional descriptive fields (`who`, `what`, `where`, `when`, `how`)
318
+ and a `metadata` block:
319
+
320
+ | Key | What it holds |
321
+ |---|---|
322
+ | `metadata.source` | `name`, `format` (`CSV`, `PARQUET`, `ICEBERG`, `JSON`, `DUCKDB` or `XLSX`), and a `path` (local, `s3://` or `gs://`, and it may have `{window_start:FORMAT}` placeholders, see [Windows](#windows)) or, for Iceberg, a `catalog`. A `DUCKDB` source also has one of `relation` (a table or view) and `sql` (one `SELECT` query), see [A DuckDB file as a source](#a-duckdb-file-as-a-source). An `XLSX` source may have `sheet` and `header_row`, see [An Excel sheet as a source](#an-excel-sheet-as-a-source) |
323
+ | `metadata.target` | The same, plus for Iceberg `write_mode` (`append`, `overwrite` or `upsert`), `join_cols` for upserts, and `partition_by` |
324
+ | `metadata.target.catalog` | `type` (`local` by default, `sql`, `rest` or `glue`) and that type's keys; see [Catalogs](#catalogs-and-object-storage) |
325
+ | `metadata.quality` | `on_failure` (`fail` or `warn`) and a list of `checks`: `row_count`, `not_null`, `unique`, `accepted_values`, `range`, `freshness`, `schema`, and the temporal checks `monotonic`, `max_skew`, `rate_below` and `max_gap` |
326
+ | `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`. The default sends them nowhere |
327
+ | `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
328
+ | `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
329
+ | `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
330
+ | `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
331
+ | `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
332
+
333
+ A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
334
+ is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
335
+ the second kind and `Metrics` the third, see [Define metrics](#define-metrics). A kind is written exactly as shown, so
336
+ `workflow` fails with a message that names `Workflow`, and `Report` and `Profiles` are reserved for later milestones.
337
+ `ldp plan` validates each file of a folder as its kind, and `ldp query`, `ldp mcp --config` and `ldp dbt` read the
338
+ tables of the `Pipeline` files only, so a folder can hold workflow and metrics files next to its configs.
339
+ `ldp schema --kind Metrics` prints the schema of another kind.
340
+
341
+ Partition transforms are `identity`, `year`, `month`, `day`, `hour`, `bucket[N]` and
342
+ `truncate[W]`. On pyiceberg 0.10 and later, writing with any transform except `identity` needs the
343
+ `pyiceberg-core` package, which `ldp` depends on, so a normal install has it. If it's missing, the
344
+ demo's step 4 says so and partitions by `identity(pickup_date)` instead. The full schema, with an
345
+ example of every check, is in the [v0.1.1 hardening contract](docs/design/v0_1_1.md#config-schema).
346
+ `local_data_platform.spec.json_schema()` returns it as JSON Schema, and `ldp schema` prints it. The configs in
347
+ [`examples/`](examples/README.md) are real ones you can run.
348
+
349
+ ### Windows
350
+
351
+ A run can be responsible for one window of time, `[start, end)`, which is what `--window` gives it. A config says how a
352
+ window picks what the run reads, and when the config runs:
353
+
354
+ ```json
355
+ {
356
+ "identifier": "daily_orders",
357
+ "when": "daily",
358
+ "metadata": {
359
+ "schedule": "5 2 * * *",
360
+ "window": {"size": "day", "column": "ordered_at", "timezone": "Europe/Paris"},
361
+ "source": {"name": "orders", "format": "CSV", "path": "drops/orders_{window_start:%Y-%m-%d}.csv"},
362
+ "target": {"name": "orders", "format": "ICEBERG", "write_mode": "overwrite",
363
+ "catalog": {"identifier": "shop", "warehouse_path": "warehouse"}}
364
+ }
365
+ }
366
+ ```
367
+
368
+ - `{window_start:FORMAT}` and `{window_end:FORMAT}` in `metadata.source.path` name the file of the window, with a Python
369
+ `strftime` FORMAT shown in the time zone of the window. A run without a window on such a path fails and names the
370
+ placeholder.
371
+ - `window.column` keeps only the rows at or after the start of the window and before its end, before the transforms and
372
+ the quality checks. `run.extracted` reports the rows read and the rows kept.
373
+ - With `window.column` and the write mode `overwrite`, a run replaces only the rows of its window, so one changed day can
374
+ be redone and the other days stay. A published window is skipped when it runs again, so to redo a day that changed at
375
+ the source, replace it with `ldp backfill daily_orders.json --from 2026-08-02 --to 2026-08-03 --replace`, or run it with a
376
+ key of your own: `ldp run daily_orders.json --window 2026-08-02/2026-08-03 --idempotency-key orders-2026-08-02-v2`.
377
+ - A `freshness` check in a windowed run measures the age of the data at the end of the window.
378
+ - A config with a schedule that appends, and that has neither a placeholder nor a `window.column`, would load the same
379
+ rows in every window. `ldp plan` reports that as an error and `ldp run --window` logs a warning.
380
+
381
+ The [contract](docs/design/v0_1_1.md#windows-and-schedules-013) has the full rules, including how each column type is
382
+ compared.
383
+
384
+ ### Workflow files
385
+
386
+ A workflow file lists the pipelines, dbt models and maintenance that run for each window, and in what order:
387
+
388
+ ```json
389
+ {
390
+ "apiVersion": "ldp/v1",
391
+ "kind": "Workflow",
392
+ "identifier": "daily_orders",
393
+ "metadata": {"schedule": "5 2 * * *", "window": {"size": "day"}, "retries": {"max_attempts": 3, "delay_s": 30}},
394
+ "tasks": [
395
+ {"id": "orders", "pipeline": "config/orders.json"},
396
+ {"id": "customers", "pipeline": "config/customers.json"},
397
+ {"id": "gold", "dbt": {"project": "dbt", "select": "+daily_revenue"}, "depends_on": ["orders", "customers"]},
398
+ {"id": "export", "pipeline": "config/export.json", "depends_on": ["gold"]},
399
+ {"id": "tidy", "maintain": {"config": "config/orders.json", "expire": 7}, "depends_on": ["export"]}
400
+ ]
401
+ }
402
+ ```
403
+
404
+ ```bash
405
+ ldp workflow plan daily_orders.json --window 2026-08-01/2026-08-02 # the tasks in run order, with each key
406
+ ldp workflow run daily_orders.json --window 2026-08-01/2026-08-02 # run them for that day
407
+ ldp runs .ldp/events.jsonl # the workflow run and the run of each task
408
+ ```
409
+
410
+ - Each task has an `id`, an optional `depends_on`, and exactly one kind: `pipeline` (the path of a dataset config,
411
+ relative to the workflow file), `dbt` (a dbt project, built for the window, see below), `maintain` (a config's Iceberg
412
+ table and the options of `ldp maintain`, a dry run unless `apply` is true) or `gold` (the gold tables of a folder, built
413
+ for the window, with a release cut at the end, see [Release the gold tables](#release-the-gold-tables)). Later
414
+ milestones add kinds without changing the file format.
415
+ - Tasks run one at a time, in dependency order. A pipeline task runs with the window of the workflow, so its idempotency key
416
+ is the one `ldp plan CONFIG --window` prints.
417
+ - A task that fails, or that a quality check blocks, makes the tasks that depend on it `upstream_failed`, and they do not run.
418
+ The tasks that do not depend on it still run. The exit status is 1 when a task failed and 3 when the only problem is a
419
+ data gate.
420
+ - Run the same window again after fixing the cause: the pipeline tasks that published are skipped by their keys, without
421
+ reading their sources, and only the tasks that failed or did not run do their work.
422
+ - A task that fails with an error that another attempt may fix runs again by itself, as `retries` says, in the workflow
423
+ or in the task, and a key of the task overrides the same key of the workflow. These are retried: a dropped connection,
424
+ a timeout, a file that has not landed yet (any `OSError`) and the commit errors of pyiceberg. A data gate, a wrong
425
+ config and a fenced attempt never are. A retry is the next attempt of the same run, with the same run id and the same
426
+ idempotency key, so the new attempt fences the old one and a load that already landed is found and not written
427
+ twice. The wait is a random time from 0 to `delay_s * backoff ** (n - 1)` seconds after attempt n. `ldp runs` lists the
428
+ last attempt of each run, and `ldp runs LEDGER --attempts` lists them all.
429
+ - A `dbt` task does what [`ldp dbt build`](#transform-with-dbt) does, for the window: dbt gets `window_start` and
430
+ `window_end` as variables (`'{{ var("window_start") }}'::timestamp` in a model), and the models marked `ldp_publish` are
431
+ written to Iceberg. Failing data tests are a data gate, so the task is `blocked_quality` and the exit status is 3. Run the
432
+ same window again and the task is skipped when its last build succeeded with the same project and no task before it may
433
+ have written data. List the pipelines that load the tables a dbt project reads in its `depends_on`, and name the dbt task
434
+ in the `depends_on` of every task that reads the tables it publishes.
435
+ - Every event of a run goes to `.ldp/events.jsonl` next to the workflow file, with the `workflow`, `workflow_run_id` and
436
+ `task_id` of the task in its payload. A dataset config that has a `metadata.schedule` runs as a workflow of one task.
437
+
438
+ The [contract](docs/design/v0_1_1.md#workflow-files-013) has the full file format. `ldp schema --kind Workflow` prints its JSON
439
+ Schema.
440
+
441
+ ### Backfill a range of windows
442
+
443
+ `ldp backfill` runs a workflow, or a config that has a window, for every window of a range. It is how a new pipeline loads
444
+ its history, and how a load that stopped halfway finishes:
445
+
446
+ ```bash
447
+ ldp backfill daily_orders.json --from 2026-06-01 --to 2026-09-01 --dry-run # every window, with its key and its state
448
+ ldp backfill daily_orders.json --from 2026-06-01 --to 2026-09-01 # load them, oldest first
449
+ ldp backfill daily_orders.json --from 2026-06-01 --to 2026-09-01 # again: it skips every window that is done
450
+ ldp backfill daily_orders.json --from 2026-08-02 --to 2026-08-03 --replace # redo a day that changed at the source
451
+ ```
452
+
453
+ - The windows are the windows of `metadata.window.size` (`day`, `hour` or `15m`), or of the size the schedule means, a day for
454
+ `@daily` and an hour for `@hourly`. `--from` and `--to` are dates or datetimes in the time zone of the workflow. Each of them
455
+ has to start a window, and the window that starts at `--to` is not part of the range, so `--from 2026-06-01 --to
456
+ 2026-06-04` is the three days of June 1st, 2nd and 3rd. Without `--to` the range ends with the last complete window. A
457
+ range of more than `--max-windows` windows, 1000 by default, is refused, to catch a typo.
458
+ - Each window runs as `ldp workflow run` runs it, so each pipeline publishes under the key that `ldp plan CONFIG --window W`
459
+ prints, and a task with `retries` is retried. A window whose keys are on the table is skipped without being read, planned
460
+ or written to the ledger, so a finished backfill that runs again does nothing, and one that stopped at window 40 starts
461
+ again at window 40. A workflow with a task that a key cannot vouch for, such as a dbt task or a pipeline that writes a file,
462
+ also needs the last run of the window in the ledger to have succeeded for every task.
463
+ - A window that fails, or that a quality check blocks, does not stop the others, and it is listed with what went wrong,
464
+ such as the check that failed. The summary counts the windows that published, were skipped, were blocked and failed.
465
+ The exit status is 0 when no window failed or was blocked, 3 when a data gate is the only problem, and 1 otherwise.
466
+ `--stop-on-failure` stops at the first window that failed instead, and a window that a quality check blocked does not
467
+ stop it.
468
+ - `--dry-run` prints every window with its key and its state, which is `published` (done, so it is skipped), `partial`,
469
+ `missing`, `failed` or `blocked` (the last run of it in the ledger failed or was held back by a data gate), and runs
470
+ nothing, not even the creation of a table. `--run-backwards` runs the newest window first, and `--json` prints the result.
471
+ - `--replace` writes each window again under a replace key, the sha256 of its key, the word `replace` and the spec hash of
472
+ its config, with a window overwrite. The rows that a window has are replaced by the rows the source has now, a window that
473
+ has none is loaded, a window that was replaced under the same spec is skipped, and one that was replaced under an older
474
+ spec is replaced again. It needs a `metadata.window.column` in every pipeline that writes to Iceberg, and without one it
475
+ exits with 1 and changes nothing. A window that is replaced has to have rows in the source. An empty one is not written,
476
+ because the run fails with "No data to write", or is blocked by a `row_count` check, and the table keeps its rows.
477
+ - From Python, `from local_data_platform.orchestration import backfill` runs the same, and `backfill(path, start, end,
478
+ dry_run=True)` returns the windows with their keys and states. The [contract](docs/design/v0_1_1.md#backfills-013) has the
479
+ rest.
480
+
481
+ ### Run on a schedule
482
+
483
+ `ldp schedule tick` is a scheduler with no server and no daemon. cron or launchd calls it every 5 minutes, it runs every
484
+ window that is due and not done, and it exits, so a laptop that was asleep catches up at the first tick after it wakes:
485
+
486
+ ```bash
487
+ ldp schedule list ldp_demo/restaurant # what is scheduled, when it fires, what it missed
488
+ ldp schedule tick ldp_demo/restaurant --now 2026-08-05T02:05:00Z # run what is due at that time, then exit
489
+ ldp schedule install --print launchd ldp_demo/restaurant > ~/Library/LaunchAgents/restaurant.plist # macOS
490
+ (crontab -l 2>/dev/null; ldp schedule install --print cron ldp_demo/restaurant) | crontab - # Linux
491
+ ```
492
+
493
+ - A tick looks at every `.json` file under the folder that is a workflow file, or a dataset config, and has a schedule in
494
+ `metadata.schedule` or in a `when` such as `daily`. A config with a schedule is a workflow of one task. A file that has a
495
+ schedule and is not valid is printed with its problems and makes the tick exit with 1, and the other files still run.
496
+ - The schedule is a cron expression read in `metadata.window.timezone`, UTC without one. The window a fire owns is the
497
+ interval between the previous fire and this fire. When the workflow has a window size, in `metadata.window.size` or implied
498
+ by `@daily` and `@hourly`, it is the last complete window of that size at or before the fire, so the fire of `5 2 * * *` on
499
+ 5 August owns 4 August. A fixed time that a clock change skips fires right after the gap, and one that happens twice fires
500
+ once.
501
+ - Without `catchup`, a tick runs the window of the latest fire if it is not done, and `ldp schedule list` prints the
502
+ `ldp backfill` command for any windows before it that were missed. With `"catchup": true` and a `"start"` in the metadata of
503
+ the workflow, a tick runs every window since `start` that is not done, oldest first, up to `--max-windows` (1000) a tick.
504
+ - A window that failed, was blocked by a data gate, or never finished is run again by the next tick, as the next attempt of
505
+ the same run. The workflow and each task keep their run ids and the attempts go up by one, so the first commit of the new
506
+ attempt fences every older one and the window is loaded once however many ticks it takes. `retries` are the retries within
507
+ one tick.
508
+ - A tick takes the lock `DIR/.ldp/locks/schedule.lock` without waiting for it. A tick that starts while another runs does
509
+ nothing and exits with 0, and a crash leaves no lock. The exit status is 0 when nothing failed or was blocked, 3 when a data
510
+ gate is the only problem, and 1 otherwise.
511
+ - The cron line and the plist call `ldp` by its absolute path and write a log, `DIR/.ldp/schedule.log` or a file in
512
+ `~/Library/Logs`. cron does not run while a Mac sleeps, and launchd runs an interval that passed during sleep once when the
513
+ Mac wakes, so launchd is the recipe for macOS and cron the recipe for Linux.
514
+ - From Python, `from local_data_platform.orchestration.schedule import tick, list_scheduled` does the same. The
515
+ [contract](docs/design/v0_1_1.md#schedules-013) has the rest.
516
+
517
+ ### Run from Airflow
518
+
519
+ A team that already runs Apache Airflow 3 can run a config as a task. `LdpRunOperator` takes the data interval of the DAG run
520
+ as the window, `ti.try_number` as the attempt, and a run id made of the DAG id, the run id of the DAG run and the task id. So
521
+ a retry by Airflow is the next attempt of the same run, a backfill over a day that is in the table finds the key and writes
522
+ nothing, and a day is loaded once however Airflow got to it. The [guide](docs/airflow.md) installs Airflow, runs the daily
523
+ drops of the restaurant example from `examples/restaurant/airflow_dag.py`, and retries a late drop. Its commands, in short, are these.
524
+
525
+ ```bash
526
+ export LDP_AIRFLOW_DEMO="$PWD/ldp_demo/airflow/data"
527
+ python examples/restaurant/airflow_dag.py --prepare "$LDP_AIRFLOW_DEMO" # the seven daily drops and a windowed orders config
528
+ airflow standalone # terminal 1, with the DAG file in its dags folder
529
+ airflow dags unpause restaurant_orders # terminal 2, catches up seven days one after the other
530
+ ldp commits "$LDP_AIRFLOW_DEMO/config/orders.json" # terminal 2, one publish for each day
531
+ ```
532
+
533
+ - The window is `(data_interval_start, data_interval_end)`, so the key of an interval is the one
534
+ `ldp plan CONFIG --window START/END` prints for it.
535
+ - A data gate that holds a run back, and a config that cannot run, fail the task with `AirflowFailException`, so Airflow does
536
+ not retry them. A skipped duplicate succeeds, and the `PipelineResult` as a dict is the XCom of the task. Any other error is
537
+ left to the `retries` of the task.
538
+ - Give the DAG a `CronDataIntervalTimetable("@daily", timezone="UTC")` as its schedule and not the string `"@daily"`. Airflow 3
539
+ makes a trigger timetable for a cron string, whose runs have an empty data interval, and the operator fails them and says why.
540
+ - The mapping is plain Python and Airflow is imported when the operator is used, so `local_data_platform.integrations.airflow`
541
+ imports without it. There is no extra for it, because Airflow has to be installed with its own constraints file. Airflow 3.1.7
542
+ and later install next to LDP, and a team that cannot do that runs `ldp run CONFIG --window ... --run-id ... --attempt ...`
543
+ from a BashOperator, which the guide shows. The [contract](docs/design/v0_1_1.md#airflow-013) has the rest.
544
+
545
+ ### A DuckDB file as a source
546
+
547
+ A `DUCKDB` source reads one table, or the result of one query, from a DuckDB database file and
548
+ publishes it to Iceberg like any other source, with the same quality checks, run events and
549
+ snapshots. It is how tables built with SQL (or with dbt, through `ldp dbt`) reach the lakehouse:
550
+
551
+ ```json
552
+ {
553
+ "identifier": "churn_by_month",
554
+ "metadata": {
555
+ "source": {"name": "churn_by_month", "format": "DUCKDB", "path": "target/build.duckdb",
556
+ "relation": "gold.churn_by_month"},
557
+ "target": {"name": "churn_by_month", "format": "ICEBERG", "write_mode": "overwrite",
558
+ "catalog": {"identifier": "gold", "warehouse_path": "warehouse"}}
559
+ }
560
+ }
561
+ ```
562
+
563
+ Use `relation` (`table`, `schema.table` or `catalog.schema.table`), `sql` or `query_file`, one of them. The file is
564
+ opened read-only and closed before the Iceberg write starts, and a query must be exactly one `SELECT` or `WITH` query.
565
+ A `query_file` is a file that holds the query, in place of `sql`. It needs the `duckdb` extra. `ldp plan` reports a
566
+ missing file or a source with more than one or none of the three.
567
+
568
+ A source can also have `inputs` in place of `path`, which is a list of dataset configs. It then reads no file. It
569
+ opens an in-memory DuckDB, registers the Iceberg table that each input config writes under that table's name, with the
570
+ session time zone set to UTC, and runs the query on them. This is how a gold table is one SQL file over silver tables,
571
+ and [`ldp gold build`](#build-the-gold-tables) builds them in order.
572
+
573
+ ```json
574
+ "source": {"name": "revenue_by_restaurant", "format": "DUCKDB", "query_file": "models/revenue_by_restaurant.sql",
575
+ "inputs": ["silver_orders.json"]}
576
+ ```
577
+
578
+ A query that reads a table that is not in `inputs` fails with DuckDB's own unknown table error, so `inputs` is the whole
579
+ lineage of the table, and every input is an input dataset in the run's OpenLineage events.
580
+
581
+ ### An Excel sheet as a source
582
+
583
+ An `XLSX` source reads one sheet of an `.xlsx` workbook and writes it to a CSV file (`ExcelToCSV`) or an
584
+ Iceberg table (`ExcelToIceberg`), so a spreadsheet that someone downloads from a website stops being converted
585
+ by hand:
586
+
587
+ ```json
588
+ {
589
+ "identifier": "sales",
590
+ "metadata": {
591
+ "source": {"name": "sales", "format": "XLSX", "path": "downloads/sales.xlsx", "sheet": "Sales",
592
+ "header_row": 1},
593
+ "target": {"name": "sales", "format": "CSV", "path": "drive/sales.csv"}
594
+ }
595
+ }
596
+ ```
597
+
598
+ `sheet` is the sheet's name or its position from 0, and the first sheet by default. `header_row` is the row
599
+ that holds the column names, counted from 1, and the rows above it are skipped. Dates and datetimes become
600
+ timestamps, whole numbers `int64`, other numbers `float64`, TRUE and FALSE `bool`, and text `string`. A column
601
+ that mixes kinds becomes `string`, and a warning names it. A formula cell gives the value Excel saved with it,
602
+ and empty rows at the end of the sheet and empty columns at its right edge are dropped. Formatting, charts and
603
+ macros are ignored, and a merged range gives its value to its first cell and nulls to the rest. A `.xls` file, a
604
+ sheet that does not exist, and an empty or repeated header cell fail with a message that names the file and the
605
+ sheet. It needs the `excel` extra, and `ldp plan` reports a missing file or a `.xls` path. The CSV is replaced
606
+ atomically, so a folder that Google Drive syncs never holds a half-written file; see the recipe
607
+ [Excel to CSV in a Google Drive folder](docs/recipes.md#excel-to-csv-in-a-google-drive-folder).
608
+
609
+ ### Table contracts
610
+
611
+ A table says what it means, who owns it, what one row is and what each column holds, in a `metadata.docs` block next to
612
+ the pipeline that builds it. After a run publishes, LDP writes the block into the Iceberg table, as the properties
613
+ `comment`, `ldp.owner`, `ldp.layer`, `ldp.grain` and `ldp.primary-key` and the doc of each column. Anyone who finds the
614
+ table reads it there, in `ldp mcp`, in DuckDB or in any other client.
615
+
616
+ ```json
617
+ "docs": {
618
+ "layer": "gold",
619
+ "description": "Orders and revenue for each restaurant on each day it had an order.",
620
+ "owner": "restaurant operations",
621
+ "grain": "one row for each restaurant on each day it had an order",
622
+ "primary_key": ["restaurant_id", "day"],
623
+ "columns": {"restaurant_id": "The restaurant.", "day": "The day.", "orders": "How many orders.",
624
+ "revenue": "What they cost."}
625
+ }
626
+ ```
627
+
628
+ - A `gold` table has to keep its promise. It needs a `description`, a `grain` and an owner (`owner`, or the `who` of the
629
+ config). `ldp run` stops it before anything is written when a column has no description, or when `columns` describes
630
+ a column that the table does not have, and the exit status is 3, as for a failed quality check. A table of another
631
+ layer may document as much or as little as it likes.
632
+ - The contract is one metadata-only commit, so it adds no snapshot, and a table that already has it gets no commit. It
633
+ only adds and changes text, and a run that wrote nothing does not touch the table.
634
+ - `ldp gold sync CONFIG` writes the contract of a config into a table that exists, with no new data. With
635
+ `--dbt-manifest target/manifest.json` it reads the contract from the manifest, for a table that dbt built.
636
+ - `ldp dbt build` fills the block from the dbt manifest. A model that is published to the `gold` namespace is a gold
637
+ table, so it needs `owner` and `grain` in its meta, a description, and a description of each column.
638
+ - Editing the block does not change the spec hash. The design is in [the v0.1.4 contract](docs/design/v0_1_4.md).
639
+
640
+ ### Build the gold tables
641
+
642
+ A gold table is one SQL file over other tables. Its config has a `DUCKDB` source with `inputs` and a `query_file`, and
643
+ a `metadata.docs` block with `"layer": "gold"`. `ldp gold build DIR` finds every gold config of a folder, orders them by
644
+ their `inputs`, since a gold table may read another, and builds them one at a time.
645
+
646
+ ```bash
647
+ ldp gold build examples/robot_episodes/data
648
+ ```
649
+
650
+ ```text
651
+ gold_episode_stats built 96 rows snapshot 3844274472678295321 4 of 4 checks passed
652
+ gold_robot_daily built 16 rows snapshot 1985638678580193697 2 of 2 checks passed
653
+ gold build examples/robot_episodes/data: 2 built, 0 checked, 0 blocked, 0 failed, 0 not run in 0.6s
654
+ metrics: 1 semantic model and 5 metrics fit the gold tables
655
+ release r1 cut with 2 tables and 112 rows in robots
656
+ ```
657
+
658
+ The last line is the release that the build cuts, see [Release the gold tables](#release-the-gold-tables).
659
+
660
+ - A cycle between configs is refused before anything runs, and the message names it.
661
+ - The build stops at the first table that a gate blocks or that fails, and the tables after it are `not run`, because
662
+ they could read what it did not write. The exit status is 3 when a gate held a table back, and 1 on any other
663
+ failure. A table whose contract does not hold is held back by a gate too.
664
+ - A gold config with no `inputs`, such as a dbt model that `ldp dbt build` published or a table that Spark or a script
665
+ wrote, is built elsewhere. The build does not run it. It checks that the table exists and runs its quality checks on
666
+ the current snapshot, and it writes nothing. A failing check blocks the build, unless the config says
667
+ `on_failure` `warn`.
668
+ - The runs are recorded in `_ldp.runs` and `_ldp.quality_results`, through the `iceberg` sink, unless the config names
669
+ sinks of its own, so the freshness and the gate status of a gold table show in `describe_table` of `ldp mcp`.
670
+ - A folder with a `metrics.json` has its metrics checked, see [Define metrics](#define-metrics). A file that is not
671
+ valid stops the build before anything is built. When every table was built or checked, the names and expressions are
672
+ checked against the tables, and the build says so with a `metrics:` line. Metrics that do not fit cut no release and
673
+ the exit status is 1, and the readers still see the last release.
674
+
675
+ ### Release the gold tables
676
+
677
+ A gold table is rebuilt on its own, so two tables that are read at the same moment can come from different builds. A
678
+ release fixes that. When every gold table of a build was built or checked, `ldp gold build` cuts release `r1`, and then
679
+ `r2` and so on. A release names the snapshot that each gold table ended the build with, so a reader that reads the
680
+ current release sees tables that agree. A build that a gate blocks cuts no release, and the last release stays current.
681
+ The build says which release readers still see.
682
+
683
+ ```bash
684
+ ldp gold build examples/robot_episodes/data # builds, and then cuts a release
685
+ ldp gold releases examples/robot_episodes/data # the releases, newest first
686
+ ldp gold releases examples/robot_episodes/data --show r1
687
+ ldp query examples/robot_episodes/data "SELECT count(*) FROM robots.gold_episode_stats" --release r1
688
+ ```
689
+
690
+ ```text
691
+ gold_episode_stats built 96 rows snapshot 7902942047508908069 4 of 4 checks passed
692
+ gold_robot_daily built 16 rows snapshot 7453607785734378588 2 of 2 checks passed
693
+ gold build examples/robot_episodes/data: 2 built, 0 checked, 0 blocked, 0 failed, 0 not run in 0.6s
694
+ metrics: 1 semantic model and 5 metrics fit the gold tables
695
+ release r1 cut with 2 tables and 112 rows in robots
696
+ ```
697
+
698
+ - A release is a file in the warehouse, `<warehouse>/.ldp/releases/<namespace>/r1.json`. It lists each table with its
699
+ snapshot, the dataset version that pins it, the run that built it and its rows. The file is written to a temporary
700
+ path and renamed, so a reader sees all of it or none of it.
701
+ - Each table is pinned with `datasets.pin` as the dataset `<namespace>.<table>`, with the property `release=r1`. A pin
702
+ tags its snapshot, and snapshot expiry never removes a tagged snapshot, so `ldp maintain` keeps every released
703
+ snapshot. That keeps its files too, so a table that is rebuilt in full for each release keeps a copy for each of them.
704
+ - A release covers one namespace of one warehouse, because that is where its file is. A folder whose gold tables are in
705
+ more than one is refused before anything is built. Give each its own folder, or build with `--no-release`.
706
+ - A build that finds the tables at the snapshots of the current release, as a workflow does when it runs a window that
707
+ it already ran, cuts no new release.
708
+ - `ldp query DIR SQL --release r1` reads the gold tables as r1 names them. A table that the release does not hold, and
709
+ a table in a namespace that has no releases, is read at its current snapshot. Without `--release`, `ldp query` reads the
710
+ tables as they are now.
711
+ - Code that reads the gold layer asks for the release with `local_data_platform.gold.release`. `current_release(dir)`
712
+ gives the release readers see, `snapshot_for(table, release)` gives the snapshot to read, and `register_release(engine,
713
+ dir)` registers every gold table in a `DuckDBEngine` at its snapshot in the release. Pass `release="r1"` for an older
714
+ one and `live=True` to read the tables as they are.
715
+ - `ldp runs` shows a release in the history of each table, as a run with the status `released`.
716
+
717
+ A workflow can build and release the gold layer as one of its tasks, after the pipelines and the dbt project that
718
+ feed it. The task is one line.
719
+
720
+ ```json
721
+ {"id": "gold", "depends_on": ["orders", "dbt"], "gold": {"dir": "config"}}
722
+ ```
723
+
724
+ The task builds the gold tables of `config` for the window of the run, as a pipeline task runs a config, so running the
725
+ same window again finds the tables already published and cuts no second release. It ends `blocked_quality` when a gate
726
+ holds a table back, and then no release is cut.
727
+
728
+ ### Browse the gold tables
729
+
730
+ A person who is not an engineer should find the right gold table without reading pipeline code, and so should an agent
731
+ that works for them. `ldp gold list` prints one row for each gold table of a folder, and `ldp gold show` describes one.
732
+
733
+ ```bash
734
+ ldp gold list examples/robot_episodes/data
735
+ ```
736
+
737
+ ```text
738
+ 2 gold tables in examples/robot_episodes/data
739
+ table description owner grain rows refreshed_utc run quality release
740
+ ------------------------- ------------------------------------------------ -------------------------- ------------------------- ---- ------------------- --------- ------------- -------
741
+ robots.gold_episode_stats One row for each promoted episode, with its o... robot learning researchers one row per episode 96 2026-10-16 09:17:03 published 4 of 4 passed r1
742
+ robots.gold_robot_daily Fleet health for each robot on each day: how ... fleet operations one row per robot per day 16 2026-10-16 09:17:03 published 2 of 2 passed r1
743
+ (2 rows)
744
+ ```
745
+
746
+ ```bash
747
+ ldp gold show examples/robot_episodes/data gold_robot_daily
748
+ ```
749
+
750
+ ```text
751
+ robots.gold_robot_daily (gold)
752
+ Fleet health for each robot on each day: how many episodes it recorded, how many succeeded, and how healthy its frames were.
753
+ Owner fleet operations
754
+ Grain one row per robot per day
755
+ Primary key robot_id, recorded_on
756
+ Refresh after each promotion
757
+ Tags robotics, fleet
758
+ Config gold_robot_daily.json
759
+ Rows 16
760
+ Refreshed 2026-10-16 09:17:03 UTC
761
+ Last run published, run 01a14400-2856-7074-a459-21ae4afcae71, 2026-10-16 09:17:03 UTC
762
+ Quality 2 of 2 passed
763
+ Release r1
764
+
765
+ Columns (9)
766
+ ...
767
+ Metrics
768
+ ...
769
+ Releases that include it (1)
770
+ ...
771
+ Read it
772
+ ldp query examples/robot_episodes/data "SELECT * FROM robots.gold_robot_daily LIMIT 10"
773
+ ```
774
+
775
+ `show` also prints every column with its type and description, the metrics defined on the table, the last 5 runs, the
776
+ releases that include the table and 5 sample rows. The last line is an `ldp query` line that reads the table, ready to copy. `TABLE` is a table with its
777
+ namespace, or its bare name when only one table has it, and it can be a table of any layer.
778
+
779
+ - The tables are the Iceberg tables that the dataset configs of the folder write. A gold table is one whose
780
+ `metadata.docs.layer` is `gold`, or whose Iceberg table says so, or that is in a namespace called `gold`.
781
+ `--all-layers` adds the bronze and silver tables and the tables that have no layer, and `--json` prints the same as
782
+ data.
783
+ - The contract is the one in the `metadata.docs` of the config and the one that the Iceberg table carries. Where both
784
+ have a value the config wins, because it is what the next run writes, and `show` says so when the table does not carry
785
+ it yet. A table with no contract shows `no description` and does not fail the listing, and its description is `null`
786
+ in JSON.
787
+ - `--strict` exits with 1 when a gold table has no description, no owner or no grain, or has a column without a
788
+ description, and it names the table and what it lacks. A table of another layer is never held to it. Run it in CI, so
789
+ that a gold table that nobody documented does not get in. A table that does not exist yet is listed as `not built`.
790
+ - The last run and the last quality check are what `describe_table` of `ldp mcp` reports, from the same code. A release is
791
+ not a run, so a table that was released still shows the status of its last build.
792
+ - `release` is the release that readers see now. `r1 (behind live)` says that the table was built again after r1, so
793
+ readers of the release do not see the newer data yet. The rows and the time are the table as it is now, and the time
794
+ leaves out the snapshots that `ldp maintain` makes.
795
+ - Nothing is written, not even a catalog or a namespace that does not exist yet.
796
+
797
+ `ldp gold docs` writes the same content as one page, or as Markdown for a MkDocs site.
798
+
799
+ ```bash
800
+ ldp gold docs examples/robot_episodes/data --out catalog.html
801
+ ldp gold docs examples/robot_episodes/data --out docs/catalog.md --format md
802
+ ```
803
+
804
+ ```text
805
+ wrote catalog.html, 2 tables, 18.8 KB. Open it as a page. It needs no network.
806
+ wrote docs/catalog.md, 2 tables, 7.4 KB. Open it as a Markdown file.
807
+ ```
808
+
809
+ - The page is one file with a search box, a light and a dark theme, and its CSS and its one script inline. It asks for
810
+ nothing from the network, and its Content-Security-Policy makes the browser refuse to try, so it opens from a folder or
811
+ from an e-mail. It is built with `local_data_platform.html`, which later pages share.
812
+ - Every value that a person wrote, such as the description of a table, is escaped. It can never become markup in the page,
813
+ and the Markdown escapes it too.
814
+ - The sample rows of each table are in the file. Leave them out with `--no-samples` when the file will be shared and the
815
+ rows must not be.
816
+ - In code, `local_data_platform.gold.catalog.collect(folder)` returns one entry for each table, and
817
+ `entry.document()` is the text of an entry for a search to index.
818
+
819
+ `ldp mcp` says the same things to an agent. `list_tables` returns the layer, description, owner and grain of each table, and
820
+ `describe_table` returns the contract. `ldp mcp --config DIR --layer gold` serves only the gold tables, so no tool and no
821
+ SQL can reach the tables upstream of them.
822
+
823
+ ### Define metrics
824
+
825
+ Revenue, orders and average ticket should mean one thing in every report, export and answer of an agent. A
826
+ `metrics.json` file next to the configs of the gold tables defines each number once, with what it means, who answers
827
+ for it and how a person reads it. The robot example has one, `examples/robot_episodes/configs/metrics.json`, and so has the
828
+ restaurant example, `examples/restaurant/config/metrics.json`, which defines `churn_rate` and `active_customers` on the
829
+ churn model of the warehouse demo.
830
+
831
+ ```json
832
+ {
833
+ "apiVersion": "ldp/v1", "kind": "Metrics", "identifier": "robot_fleet_metrics", "who": "fleet operations",
834
+ "semantic_models": [{
835
+ "name": "robot_daily", "table": "robots.gold_robot_daily",
836
+ "time_dimension": {"column": "recorded_on", "grains": ["day", "week", "month", "quarter", "year"]},
837
+ "dimensions": [{"name": "robot", "column": "robot_id"}],
838
+ "measures": [{"name": "episode_sum", "agg": "sum", "expr": "episodes"},
839
+ {"name": "success_sum", "agg": "sum", "expr": "successes"}]}],
840
+ "metrics": [
841
+ {"name": "episodes", "type": "simple", "measure": "episode_sum",
842
+ "description": "How many episodes the robots recorded."},
843
+ {"name": "success_rate", "type": "ratio", "numerator": "success_sum", "denominator": "episode_sum",
844
+ "format": "percent", "description": "The share of episodes that succeeded."}]
845
+ }
846
+ ```
847
+
848
+ A semantic model names one gold table, its `time_dimension` (a date column and the grains that its numbers may be
849
+ reported at), its `dimensions` and its `measures`. A measure is an aggregate, `sum`, `count`, `count_distinct`, `avg`,
850
+ `min` or `max`, over a SQL expression of the columns of the table. A metric is `simple`, which is one measure, or a
851
+ `ratio`, which is one measure over another. It has a `description`, an `owner` (its own, or the `who` of the file) and
852
+ a `format` of `number`, `percent` or `currency`. The terms follow MetricFlow, so an exporter can follow later.
853
+ `ldp plan DIR` validates the file as `Metrics`, and `ldp schema --kind Metrics` prints its schema.
854
+
855
+ ```bash
856
+ ldp metrics list examples/robot_episodes/data
857
+ ldp metrics query examples/robot_episodes/data episodes,success_rate,frame_drop_rate --by robot --grain week
858
+ ```
859
+
860
+ ```text
861
+ metric description owner format table dimensions grains
862
+ ----------------- ------------------------------------------------ -------------------------- ------- ----------------------- ---------- -------------------------------
863
+ episodes How many episodes the robots recorded. fleet operations number robots.gold_robot_daily robot day, week, month, quarter, year
864
+ success_rate The share of episodes that succeeded. fleet operations percent robots.gold_robot_daily robot day, week, month, quarter, year
865
+ frame_drop_rate The share of frames that were dropped. fleet operations percent robots.gold_robot_daily robot day, week, month, quarter, year
866
+ worst_skew_ms The largest RGB to depth gap of any frame, in... fleet operations number robots.gold_robot_daily robot day, week, month, quarter, year
867
+ active_robot_days Days with a recording, added up over robots. robot learning researchers number robots.gold_robot_daily robot day, week, month, quarter, year
868
+ (5 rows)
869
+
870
+ week robot episodes success_rate frame_drop_rate
871
+ ---------- ----- -------- ------------ ---------------
872
+ 2026-08-31 h01 12 83.3% 0.3%
873
+ 2026-08-31 h02 12 75.0% 0.5%
874
+ 2026-08-31 h03 12 83.3% 0.3%
875
+ 2026-08-31 h04 12 91.7% 0.4%
876
+ 2026-08-31 h05 12 41.7% 0.4%
877
+ 2026-08-31 h06 12 66.7% 0.3%
878
+ 2026-08-31 h07 12 75.0% 0.4%
879
+ 2026-08-31 h08 12 66.7% 0.0%
880
+ (8 rows)
881
+ read release r1, cut 2026-10-20 08:34:32 UTC
882
+ ```
883
+
884
+ `--filter dimension=value` keeps the rows of a dimension, `--from` and `--to` take days that are both included, and
885
+ `--sql` prints what would run, with the value of each parameter, and runs nothing.
886
+
887
+ ```bash
888
+ ldp metrics query examples/robot_episodes/data success_rate --filter robot=h05 --grain month --from 2026-09-01 --sql
889
+ ```
890
+
891
+ ```text
892
+ SELECT
893
+ CAST(date_trunc('month', "recorded_on") AS DATE) AS "month",
894
+ sum(successes) / NULLIF(sum(episodes), 0) AS "success_rate"
895
+ FROM "robots"."gold_robot_daily"
896
+ WHERE "robot_id" = $1 AND "recorded_on" >= $2
897
+ GROUP BY 1
898
+ ORDER BY 1
899
+ -- $1 = "h05"
900
+ -- $2 = DATE '2026-09-01'
901
+ ```
902
+
903
+ - **A ratio is a ratio of sums.** `success_rate` is the sum of the successes over the sum of the episodes of the group,
904
+ and never the average of the daily rates, which is a different number when the days have different counts. A group
905
+ with no episodes has no value and not infinity.
906
+ - **A value is never SQL.** A filter value, a date and every other value that a person gives is bound as a parameter of
907
+ the type of its column, so `--filter robot="h05' OR '1'='1"` matches no robot. Names go into the SQL only after they
908
+ were checked against the definitions, and an unknown metric, dimension or grain is refused with the valid names and the
909
+ closest one.
910
+ - **A query reads one table.** Metrics of two semantic models are refused with the table of each, because that would need
911
+ a join, which this version does not have.
912
+ - **`grains` is where the numbers are right.** A number that counts customers at the start of a month is wrong at any
913
+ other grain, so its model allows only `month`, and another grain is refused. A query with no grain adds the whole
914
+ period into one row, which is the same step as adding up years, so it needs `year` to be allowed. The churn of the
915
+ restaurant example is such a number. After `make demo-warehouse`,
916
+ `ldp metrics query ldp_demo/restaurant/config churn_rate,active_customers --grain month` prints it for every month,
917
+ and the [recipe](docs/recipes.md) for monthly churn from an orders table shows the definition in SQL.
918
+ - **The format is for people.** `percent` shows `0.5` as `50.0%` and `currency` has two decimals, on a terminal. `--raw`
919
+ shows what was computed, and `--out FILE`, `--json` and the Python function always give the values as computed.
920
+ `--out` writes CSV, Parquet or JSONL by the ending of the name.
921
+ - **The tables are read as the current release holds them.** `--release r1` reads an older one and `--live` reads the
922
+ tables as they are now, as `ldp query` does. The line under the table says which was read. A release pins the data and
923
+ not the definitions, so an old release that is read with a definition that needs a column that it lacks fails with a
924
+ message that says so.
925
+ - **The definitions are checked against the tables.** `ldp metrics validate DIR` checks that every table is a gold table
926
+ that exists, that every dimension and the time column are columns, that the time column is a `DATE` or a `TIMESTAMP`,
927
+ that each expression is one plain SQL expression that works on the table, and that every metric is a number. It lists
928
+ every problem and exits with 1. `ldp gold build` runs the same check and cuts no release when metrics do not fit.
929
+ - **In code**, `local_data_platform.metrics.query(folder, "episodes,success_rate", by="robot", grain="week")` returns a
930
+ `pyarrow.Table`, and `metrics.compile_for(...)` returns the SQL and its parameters without running anything.
931
+
932
+ `ldp mcp` has two more tools for the same numbers, `list_metrics` and `query_metric`, see
933
+ [MCP server for agents](#mcp-server-for-agents). The gold catalog shows the metrics of each table, in `ldp gold show`,
934
+ on the page of `ldp gold docs`, and in its search.
935
+
936
+ ### Open the gold tables in a spreadsheet or a BI tool
937
+
938
+ A person who works in a spreadsheet or a desktop SQL tool should be able to use a gold table without Python, Iceberg or an
939
+ engineer. `ldp gold export` writes one table as a file, and `ldp gold duckdb` writes the whole gold layer as one DuckDB file.
940
+ Both read the gold layer as every reader does, so by default it is the current release, `--release r1` reads an older one
941
+ and `--live` reads the tables as they are now.
942
+
943
+ ```bash
944
+ ldp gold export examples/robot_episodes/data gold_robot_daily daily.csv
945
+ ldp gold export examples/robot_episodes/data gold_robot_daily daily.xlsx
946
+ ```
947
+
948
+ ```text
949
+ wrote daily.csv, 16 rows of robots.gold_robot_daily, read from release r1, cut 2026-10-23 11:26:36 UTC, snapshot 1317331787740633770
950
+ wrote daily.xlsx, 16 rows of robots.gold_robot_daily, read from release r1, cut 2026-10-23 11:26:36 UTC, snapshot 1317331787740633770
951
+ ```
952
+
953
+ The format comes from the end of the name, or from `--format`.
954
+
955
+ - **CSV, Parquet and JSONL.** The three are written by `datasets.write_rows`, the same function as `ldp datasets export`,
956
+ which has `--format csv` too. A CSV has a header and a line for each row. A list, a struct or a map is written as JSON text,
957
+ bytes as hexadecimal and a number that is not finite as no value. A Parquet file has the table, release and snapshot in its
958
+ metadata, under `ldp.gold`. A name that starts with `s3://` or `gs://` is written there.
959
+ - **Excel.** The file needs the `excel` extra, `pip install "local-data-platform[excel]"`, and says which if it is missing.
960
+ The first sheet has the rows, with a bold header that stays in view and a filter. The second sheet is called `about` and
961
+ has the description, owner, grain and key of the table, its release, snapshot id and build time, its rows, and every
962
+ column with a word for its type and its description. A text is always a text and never a formula, even when it starts
963
+ with `=`. A time with a zone is written in UTC, because a sheet has no zones. An integer beyond 2^53 is written as text,
964
+ because a sheet would round it. A table with more than 1,048,575 rows does not fit a sheet, with its header row, and is
965
+ refused before it is read, with a hint to use CSV. A table with more than 16,384 columns and a value of more than 32,767
966
+ characters are refused too, with the same hint, when the file is written, and no file is left.
967
+ - **A CSV and a formula.** Excel runs a CSV value that starts with `=`, `+`, `-` or `@` as a formula when a person opens
968
+ the file. LDP writes the values as they are, so use Excel for text that people typed and that you do not trust.
969
+ - **A table that the release does not hold.** When the current release was cut before the table existed, the table is read
970
+ as it is, and the line says why. `--release r1` for a table that r1 does not hold is refused.
971
+
972
+ ```bash
973
+ ldp gold duckdb examples/robot_episodes/data --out gold.duckdb
974
+ ```
975
+
976
+ ```text
977
+ wrote gold.duckdb, 2 tables and 5 metrics, from release r1, cut 2026-10-23 11:26:36 UTC, 1292.0 KB
978
+ ```
979
+
980
+ The file has a copy of every gold table of the release, in the `main` schema under its own name, so `SELECT * FROM
981
+ gold_robot_daily` works. It has no schema called `gold`, because DuckDB names a file's database after the file, and
982
+ `gold.daily_sales` in a file called `gold.duckdb` would be ambiguous. Three more things are in it.
983
+
984
+ - **Comments.** `COMMENT ON TABLE` has the description of each table and `COMMENT ON COLUMN` the description of each column,
985
+ from the contract, so a tool that shows comments shows what the data means.
986
+ - **`_catalog`.** One row for each table with `table_name`, `description`, `owner`, `grain`, `release`, `snapshot_id` and
987
+ `built_at`, which is when that snapshot was written, in UTC.
988
+ - **`_metrics`.** One row for each metric of `metrics.json` with its `metric`, `description`, `owner`, `type`, `format`,
989
+ `table_name`, `definition`, `dimensions`, `time_column` and `grains`. The `definition` is the SQL that computes it from
990
+ its table, such as `sum(dropped_frames) / NULLIF(sum(frames), 0)`. The table is empty when the folder has no metrics file.
991
+
992
+ Open it in the DuckDB command line, with `duckdb gold.duckdb`, or in the DuckDB UI, with `duckdb -ui gold.duckdb`. The UI
993
+ fetches its web pages from ui.duckdb.org, so it needs the network.
994
+
995
+ ```sql
996
+ SELECT table_name, owner, grain, release FROM _catalog;
997
+ SELECT metric, definition FROM _metrics;
998
+ SELECT robot_id, round(avg(success_rate), 2) AS success_rate FROM gold_robot_daily GROUP BY 1 ORDER BY 1;
999
+ ```
1000
+
1001
+ - **It needs no extension.** The file has plain tables, so a DuckDB with no extension reads it, and no `iceberg_scan` is in
1002
+ it. It is written in the storage version of DuckDB 1.0, so a DuckDB 1.x opens it as well as a DuckDB 2.0. DuckDB 2.0
1003
+ writes a newer version by default, which a DuckDB 1.x cannot open.
1004
+ - **A reader never sees half a file.** It is written to a temporary path next to the file and renamed. A failure leaves the
1005
+ file that was there, and no temporary file.
1006
+ - **It holds one release.** A gold table that the release does not hold is left out, and the output names it, so that the
1007
+ file is as consistent as the release is. With `--live` it holds every table as it is now and `_catalog.release` has no
1008
+ value.
1009
+ - **`ldp gold build DIR --duckdb gold.duckdb`** writes the file after the build cuts a release. When the build cuts none,
1010
+ because the tables are at the snapshots of the current release, the file is written only if it is missing or holds
1011
+ another release. A build that a gate blocks or that fails leaves the file alone, so the file that readers have is the
1012
+ one of the last good release. A file that cannot be written after the release was cut is an error with exit status 1,
1013
+ and the release stays cut. `--duckdb` cannot be used with `--no-release`.
1014
+ - **Windows.** Replacing a file that a program has open fails on Windows. `{release}` in the name is replaced by the release,
1015
+ so `--out "gold_{release}.duckdb"` writes a new file for each release and never replaces one.
1016
+ - **Memory.** A table is read whole before it is written, as `ldp datasets export` does, so a gold table has to fit in memory.
1017
+
1018
+ ## Catalogs and object storage
1019
+
1020
+ The catalog block's `type` picks the catalog. It defaults to `local`, so a catalog block without
1021
+ `type` keeps working unchanged.
1022
+
1023
+ | `type` | Keys | What you get |
1024
+ |---|---|---|
1025
+ | `local` (alias `LocalIceberg`) | `identifier`, `warehouse_path` | A SQLite catalog file and a warehouse folder |
1026
+ | `sql` (alias `sqlite`) | `uri` (a SQLAlchemy URI such as `sqlite:///…` or `postgresql+psycopg://…`), `warehouse`, `name`, `password_env` | pyiceberg's `SqlCatalog`. Its tables are the ones Iceberg's Java `JdbcCatalog` uses, so Spark can share it |
1027
+ | `rest` | `uri`, `warehouse`, `name`, `token_env`, `credential_env`, and optional `properties` (such as `s3.endpoint`) | pyiceberg's `RestCatalog`, for any Iceberg REST catalog, such as Polaris, Nessie or Lakekeeper |
1028
+ | `glue` | `name`, `warehouse`, and optional `properties` | pyiceberg's `GlueCatalog` (the `glue` extra). AWS credentials come from boto3's default chain |
1029
+
1030
+ What has been tested: `local` and `sql` on SQLite; `rest` against Apache Iceberg's REST catalog
1031
+ test server (`tools/rest_fixture`, opt-in); `glue` against a local moto server. Postgres and
1032
+ hosted REST catalogs (Polaris, S3 Tables, Unity and so on) are not tested yet.
1033
+
1034
+ The namespace tables live in comes from `identifier` for `local` catalogs, and from `namespace`,
1035
+ falling back to `identifier`, for every other type.
1036
+ `properties` are passed through to pyiceberg, and `properties_env` maps a property to the
1037
+ environment variable holding its value. A REST target looks like this:
1038
+
1039
+ ```json
1040
+ "catalog": {
1041
+ "type": "rest",
1042
+ "name": "lake",
1043
+ "uri": "https://catalog.example.com/api/catalog",
1044
+ "warehouse": "analytics",
1045
+ "namespace": "rides",
1046
+ "token_env": "LAKE_TOKEN",
1047
+ "properties": {"s3.endpoint": "http://127.0.0.1:9000"}
1048
+ }
1049
+ ```
1050
+
1051
+ **Secrets never go in configs.** A config names the environment variable that holds a token,
1052
+ credential or password (`token_env`, `credential_env`, `password_env`, `properties_env`). A key or
1053
+ property whose name contains `token`, `secret`, `password` or `credential` and holds a literal value
1054
+ is rejected, and so is a password inside a `uri`; log lines mask them. A `postgresql+psycopg://`
1055
+ URI also needs a SQLAlchemy driver, such as `pip install "psycopg[binary]"`.
1056
+
1057
+ `create_catalog` builds the same catalog a config would. For a read-only check before running
1058
+ anything, use `ldp catalog test SPEC`; calling `create_catalog` on an `sql` spec creates the SQLite
1059
+ file if it does not exist yet:
1060
+
1061
+ ```python
1062
+ from local_data_platform.catalog.provider import catalog_namespace, create_catalog
1063
+
1064
+ spec = {"type": "sql", "name": "lake", "uri": "sqlite:////tmp/lake/catalog.db", "warehouse": "file:///tmp/lake",
1065
+ "namespace": "rides"}
1066
+ catalog = create_catalog(spec)
1067
+ print(catalog_namespace(spec), catalog.list_namespaces())
1068
+ ```
1069
+
1070
+ `register_catalog_type(name)` is a decorator that adds your own type. `Iceberg(...,
1071
+ catalog_obj=catalog)` writes through a catalog object you already have.
1072
+
1073
+ **Object storage.** CSV, Parquet and JSON read and write through `local_data_platform.fs`, so their
1074
+ paths can be local, `file://`, `s3://` or `gs://`, all through `pyarrow.fs`. Settings come from the
1075
+ environment, never from a config: S3 credentials from the AWS default chain, `AWS_ENDPOINT_URL_S3`
1076
+ or `AWS_ENDPOINT_URL` for MinIO, LocalStack or moto, and `AWS_REGION`; `STORAGE_EMULATOR_HOST` for
1077
+ a GCS emulator. A URI with credentials in it (`s3://key:secret@bucket/...`) is rejected. Writes are
1078
+ atomic: locally a temp file is renamed into place, and on an object store the data is uploaded only
1079
+ once it is complete, so readers see the old object or the new one. The S3 tests run against a local
1080
+ moto server; `gs://` reads and writes have no test against a GCS emulator yet.
1081
+
1082
+ ## Exactly-once writes
1083
+
1084
+ A plain `Iceberg.put(df, mode)` is **direct mode**. It keeps the semantics of the
1085
+ [hardening contract](docs/design/v0_1_1.md) (appending twice gives duplicates, overwriting twice
1086
+ doesn't) with three fixes:
1087
+
1088
+ - **One commit per write.** Adding new columns and writing the data happen in one transaction.
1089
+ - **Counts from metadata.** `rows_before` and `rows_after` come from the snapshots' `total-records`,
1090
+ not from a scan.
1091
+ - **A lock on local catalogs.** `overwrite` and `upsert` take an exclusive file lock,
1092
+ `<warehouse>/.ldp/locks/<namespace>.<table>.lock`, so two overwrites or upserts on one laptop
1093
+ can't interleave. Appends take no lock. An append that loses a commit race reloads the table
1094
+ and runs again with a backoff (`CommitPolicy` sets the limits), so several processes can append
1095
+ to one table at once.
1096
+
1097
+ Direct `overwrite` and `upsert` on a shared remote catalog are single-writer. For several
1098
+ writers, pass a `CommitContext` and `put` switches to the **staged publish protocol**: it writes
1099
+ to a private branch, checks the staged snapshot, then fast-forwards `main` in one catalog
1100
+ compare-and-swap that asserts both `main` and the branch. The idempotency key is stored on the
1101
+ snapshot as `ldp.idempotency-key` (with `ldp.run-id` and `ldp.attempt`), so re-running a key finds
1102
+ the published snapshot and writes nothing:
1103
+
1104
+ ```python
1105
+ import pyarrow as pa
1106
+
1107
+ from local_data_platform.format.iceberg import Iceberg
1108
+ from local_data_platform.format.iceberg.commit import CommitContext
1109
+
1110
+ table = Iceberg("fares", {"identifier": "nyc", "warehouse_path": "warehouse"}, write_mode="upsert",
1111
+ join_cols=["ride_id"])
1112
+ batch = pa.table({"ride_id": [1, 2, 3], "fare": [12.5, 8.0, 30.25]})
1113
+ context = CommitContext.create("fares/2024-01-01") # a new run id; looks for the key 7 days back
1114
+
1115
+ first = table.put(batch, commit=context)
1116
+ again = table.put(batch, commit=context) # the same key: finds the published snapshot
1117
+ print(first.attempts, again.skipped_duplicate) # 1 True
1118
+ ```
1119
+
1120
+ If another writer moves `main` first, the attempt rebases and retries within its `CommitPolicy`; an
1121
+ upsert is recomputed against the new rows, never blindly rebased. A superseded attempt is fenced by
1122
+ removing its branch, so it can't publish afterwards. `WriteResult` gains `branch`,
1123
+ `idempotency_key`, `attempts` and `skipped_duplicate`. What this guarantees is at most one effect
1124
+ on `main` per key, which with retries means effectively once. It doesn't remove duplicates a
1125
+ producer sent twice upstream. `ldp commits
1126
+ CONFIG` lists the publishes on a table with their keys.
1127
+ [Exactly-once writes](docs/exactly_once.md) has the protocol and its tests, including the
1128
+ multi-process upsert race.
1129
+
1130
+ ## Run events and the `_ldp` namespace
1131
+
1132
+ `Pipeline.run(mode=None, *, commit=None, sink=None)` emits a `RunEvent` at each step:
1133
+ `run.started`, `run.extracted`, `quality.evaluated`, then one of `run.published`,
1134
+ `run.skipped_duplicate`, `run.blocked_quality` or `run.failed`, and finally `run.finished`.
1135
+ `PipelineResult` gains `run_id` (a uuid7), `idempotency_key` and `published_snapshot_id`.
1136
+
1137
+ | Sink | Writes |
1138
+ |---|---|
1139
+ | `NullSink` | Nothing (the default, so a run emits no events unless you choose a sink) |
1140
+ | `JsonlSink(path)` | One JSON event per line |
1141
+ | `OpenLineageSink(path_or_url)` | OpenLineage 1.x `RunEvent`s with schema, data-quality and output-statistics facets |
1142
+ | `IcebergSink(catalog_spec, base_dir)` | Batched appends to `_ldp.runs` and `_ldp.quality_results`, both day-partitioned, in your own catalog (the MCP server also uses it for `_ldp.audit`) |
1143
+ | `MultiSink` | Several of the above |
1144
+
1145
+ ```python
1146
+ from local_data_platform import Config
1147
+ from local_data_platform.events import JsonlSink
1148
+ from local_data_platform.pipeline.registry import create_pipeline
1149
+
1150
+ result = create_pipeline(Config.from_json("rides.json")).run(sink=JsonlSink("runs.jsonl"))
1151
+ print(result.run_id, result.published_snapshot_id)
1152
+ ```
1153
+
1154
+ Or set the default sinks in the config with `metadata.observability`, for example
1155
+ `{"sinks": ["iceberg", "jsonl"]}`; by default the `jsonl` sink writes `.ldp/events.jsonl` in the
1156
+ config's folder, and `ldp run` uses these sinks too. Because `_ldp.runs` and
1157
+ `_ldp.quality_results` are ordinary Iceberg tables in your catalog, you can query your run history
1158
+ with any engine; `ldp runs CONFIG` lists it, and `events.read_iceberg_events` and `summarize_runs`
1159
+ read it from Python. `local_data_platform.spec` gives each config a stable `spec_hash` and an
1160
+ `idempotency_key(config, window)`, and validates configs against the `ldp/v1` JSON Schema
1161
+ (`ldp plan CONFIG` shows all three). See
1162
+ [Run events and the `_ldp` namespace](docs/observability.md).
1163
+
1164
+ ## SQL with DuckDB
1165
+
1166
+ `DuckDBEngine.register_iceberg(table, alias, snapshot_id=None, row_filter=None, native=None)`
1167
+ uses DuckDB's `iceberg` extension when it can: the table becomes a view over `iceberg_scan` at the
1168
+ chosen snapshot, which streams and pushes filters down instead of loading the table into memory.
1169
+ If the extension can't load, it falls back to the in-memory scan with a warning. It never
1170
+ downloads the extension on its own, so it works offline; `DuckDBEngine(install_extensions=True)` allows
1171
+ that. `native=True` or `native=False` forces one path.
1172
+
1173
+ ```python
1174
+ from local_data_platform.engine.duckdb import DuckDBEngine
1175
+ from local_data_platform.format.iceberg import Iceberg
1176
+
1177
+ table = Iceberg("rides", {"identifier": "nyc", "warehouse_path": "warehouse"})
1178
+ with DuckDBEngine() as duck:
1179
+ duck.register_iceberg(table, "rides") # native iceberg_scan when available
1180
+ duck.register_iceberg(table, "rides_in_memory", native=False)
1181
+ print(duck.query("SELECT count(*) FROM rides").to_pylist())
1182
+ print(duck.snapshots("rides")) # from iceberg_snapshots
1183
+ ```
1184
+
1185
+ `open_lake(path)` registers every Iceberg table that some dataset configs name, so SQL can join them. `path` is a
1186
+ config file, a folder of configs, or a list of them, and `ldp query` uses it when it is given a folder or more than
1187
+ one config. Each table is a view named `namespace.table` and, when no other table has the same name, a view with the
1188
+ bare name. A namespace of several parts, such as `a.b`, is one schema named `a.b`, so SQL quotes it: `"a.b".events`.
1189
+ `snapshot_id=` or `as_of=` read one table as it was, and refuse a lake of several. Nothing is written and
1190
+ nothing is downloaded, and the MCP SDK is not needed.
1191
+
1192
+ ```python
1193
+ from local_data_platform.lake import open_lake
1194
+
1195
+ with open_lake("config") as lake:
1196
+ lake.query("SELECT o.id, c.label FROM raw.orders o JOIN raw.customers c ON c.id = o.id")
1197
+ with open_lake("config/raw.orders.json", as_of="2026-07-31 23:59:59") as lake: # UTC unless it has an offset
1198
+ lake.query("SELECT count(*) FROM orders")
1199
+ ```
1200
+
1201
+ `attach_rest(name, uri, warehouse, token_env=None)` attaches a whole REST catalog to DuckDB.
1202
+ `engine.router` estimates how many bytes a scan reads from the manifests, after partition pruning,
1203
+ and `choose_engine` picks DuckDB below 50 GiB by default, and PySpark above that when it's
1204
+ installed. No pipeline calls the router yet.
1205
+
1206
+ ## Transform with dbt
1207
+
1208
+ `ldp dbt` runs a [dbt](https://www.getdbt.com/) project over your Iceberg tables and writes the models you mark back
1209
+ to Iceberg, so what dbt builds is versioned, can be read as it was, and is there for `ldp query`, `ldp runs` and
1210
+ `ldp mcp` like any other table. It needs the `dbt` extra, and the `excel` extra when a source is an Excel sheet.
1211
+
1212
+ ```bash
1213
+ pip install "local-data-platform[dbt,excel]"
1214
+ ldp dbt init my_project --sources config --warehouse warehouse # a project that builds as soon as its sources exist
1215
+ ldp dbt build my_project # register the sources, run dbt, then publish the models marked ldp_publish
1216
+ ldp dbt test my_project # register the sources and run the data tests, and publish nothing
1217
+ ldp dbt docs my_project --serve # generate the dbt docs and serve them on http://127.0.0.1:8080
1218
+ ldp dbt exec my_project compile # any other dbt command, with the sources registered
1219
+ ```
1220
+
1221
+ Before dbt runs, every Iceberg table that the configs in `--sources` describe becomes a source in a DuckDB file inside
1222
+ the project, so a model reads it with `{{ source('raw', 'orders') }}`. After a build in which every model and every
1223
+ data test passed, each model whose config has `meta={'ldp_publish': true}` is written to Iceberg through the
1224
+ [`DUCKDB` source](#a-duckdb-file-as-a-source), with the same quality checks, run events and snapshots as any other load.
1225
+ A table or view model overwrites its table, and an incremental model with a `unique_key` upserts on it. Nothing is
1226
+ published when dbt fails. Everything after the project folder goes to dbt unchanged, and the options of `ldp dbt`
1227
+ itself go before it.
1228
+
1229
+ A published model also carries what the project says about it. Its description, the description of each column, its
1230
+ tags, its `owner` and `grain` meta and its layer become the table's [contract](docs/design/v0_1_4.md) and go into the
1231
+ Iceberg table. A model that is published to the `gold` namespace is a gold table, so it needs an owner, a grain, a
1232
+ description and a description of every column, and `ldp dbt build` stops before it publishes anything if one is
1233
+ missing.
1234
+
1235
+ - The [warehousing guide](docs/warehousing.md) walks through it on a restaurant chain's Excel and CSV exports, from
1236
+ the raw tables to tested models, churn, an incremental load and time travel. `make demo-warehouse` runs the same
1237
+ example offline in about 25 seconds.
1238
+ - [How `ldp dbt` works](docs/design/warehousing.md) has the contract, including the meta keys and the schema rules.
1239
+
1240
+ ## Orchestration
1241
+
1242
+ A workflow file declares the pipelines, dbt models and maintenance of a night once. LDP runs it for the right window, backfills a range of windows, retries what failed, and runs it on a schedule from cron or launchd, with no service. Every run goes by the idempotency key of its window, so a window is never loaded twice, whoever starts it.
1243
+
1244
+ ```bash
1245
+ make demo-orchestration # the restaurant's nightly drops, a bad row, a backfill, a fix and a scheduler tick
1246
+ ```
1247
+
1248
+ - The [orchestration guide](docs/orchestration.md) walks through it on the restaurant example, command by command, from a window and its key to a workflow file, a backfill that stops at a bad row, a replaced day, a retried task, launchd and cron, and Airflow, Dagster and Prefect.
1249
+ - `make demo-orchestration` runs `examples/restaurant/run.py --layer orchestration`. It drops three nights of orders into a folder, with a bad row in the third. `ldp backfill` publishes two days and a data gate holds the third back, with exit status 3. After the fix, the same command runs only the third, and a third run skips every day. `ldp schedule tick` then loads the fourth night, and `ldp runs`, `ldp commits`, `ldp schedule list` and `ldp schedule install` show what happened. It needs no extra, runs the `models` and `gold` tasks when the `dbt` extra is installed, and prints a skip line when it is not.
1250
+ - The pieces are described above, under [Windows](#windows), [Workflow files](#workflow-files), [Backfill a range of windows](#backfill-a-range-of-windows), [Run on a schedule](#run-on-a-schedule) and [Run from Airflow](#run-from-airflow), and the exit statuses are in the [table](#use-it-from-the-command-line) of the command line section.
1251
+
1252
+ ## Gold layer
1253
+
1254
+ A gold table is a table that people read. It says who owns it, what it means, what one row is and what each column holds, it is checked before it is written, and all the gold tables are published together as one release, so every reader sees the same build. A metric such as the average ticket is defined once. People who do not write code get the tables as a catalog page, an Excel file and one DuckDB file, and an AI agent can be limited to the gold tables.
1255
+
1256
+ ```bash
1257
+ make demo-gold # the restaurant's data mart, a faulty batch that the gate stops, a release, a metric and files for people
1258
+ ```
1259
+
1260
+ - The [gold layer guide](docs/gold.md) walks through it on the restaurant example, command by command, from the three layers and a gate that stops a bad batch to the contract of a gold table, a release that a gate can hold back, the catalog, metrics, files for people and `ldp mcp --layer gold`. It has a section for people who are not engineers, and every command in it runs in the tests.
1261
+ - `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
1262
+ - The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
1263
+
1264
+ ## MCP server for agents
1265
+
1266
+ `local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
1267
+ on stdio and read-only. Install the `mcp` extra, then point your MCP client at `ldp mcp`
1268
+ (`python -m local_data_platform.mcp_server` takes the same options):
1269
+
1270
+ ```json
1271
+ {
1272
+ "mcpServers": {
1273
+ "ldp": {
1274
+ "command": "/abs/path/to/.venv/bin/ldp",
1275
+ "args": ["mcp", "--config", "/abs/path/to/rides.json", "--max-rows", "500"]
1276
+ }
1277
+ }
1278
+ }
1279
+ ```
1280
+
1281
+ | Tool | Returns |
1282
+ |---|---|
1283
+ | `list_tables()` | Every table |
1284
+ | `describe_table(table)` | Schema, partition spec, row count, snapshots, the latest `_ldp` run and quality status, and freshness |
1285
+ | `query(sql, max_rows)` | The result of one read-only SQL statement |
1286
+ | `sample_rows(table, n)` | A sample of rows |
1287
+ | `table_history(table)` | The table's snapshot history |
1288
+ | `get_dataset(name)` | A pinned dataset version |
1289
+ | `list_metrics()` | The metrics that the `metrics.json` files of the `--config` folders define, each with what it means, its owner, format, table, dimensions and grains |
1290
+ | `query_metric(metrics, by, grain, filters, start, end, max_rows)` | Some metrics of one table, computed from the current release of the gold tables as the definitions say, with every value bound as a parameter |
1291
+
1292
+ The guardrails are on by default. A query must be exactly one `SELECT` or `WITH` statement, as
1293
+ DuckDB's parser sees it, so `COPY`, `ATTACH`, `INSTALL`, `SET`, `read_csv('/etc/passwd')` and
1294
+ multi-statement SQL are rejected. DuckDB runs with external access off and file access limited to
1295
+ the served tables' folders, and its configuration is locked once the tables are registered.
1296
+ Results are capped in rows and time, `--allow` limits the tables, and every call is audited: the
1297
+ tool, the SQL, row counts, duration and any error, never the result data. Records go to a JSONL file
1298
+ as each call ends, and to the `_ldp.audit` Iceberg table in the first served catalog when the
1299
+ server stops (`--no-iceberg-audit` turns that off).
1300
+ `query_metric` runs in the same sandbox as `query`, with the same row cap, timeout and audit, and its audit record
1301
+ has the arguments and the compiled SQL. It reads the current release of the gold tables, and `query` reads the tables
1302
+ as they are now. `python examples/agent_client.py` builds a small lake with a gold table, three metrics and a release,
1303
+ and calls both tools. `make demo-agent` runs a scripted client against the demo table. See
1304
+ [MCP server for agents](docs/agents.md).
1305
+
1306
+ ## Reproducible datasets and robotics
1307
+
1308
+ A dataset version pins a table to one snapshot, with an optional row filter and column list, and
1309
+ is saved as a JSON manifest under `<warehouse>/.ldp/datasets/`. Loading it later returns exactly
1310
+ the same rows, however the table has changed since:
1311
+
1312
+ ```python
1313
+ from local_data_platform import datasets
1314
+ from local_data_platform.format.iceberg import Iceberg
1315
+
1316
+ episodes = Iceberg("gold_episode_stats", {"identifier": "robots", "warehouse_path": "warehouse"})
1317
+ train = datasets.pin(episodes, "humanoid_train", row_filter="success = true",
1318
+ selected_fields=["episode_id", "robot_id", "task", "frame_count"])
1319
+ rows = datasets.load(train) # the same rows at any later time
1320
+ datasets.export(train, "exports/humanoid_train_v1.parquet") # or format="jsonl" or "csv"
1321
+ ```
1322
+
1323
+ Each pin also tags the snapshot (`ldp_ds_<name>_v<N>`), so snapshot expiry keeps it.
1324
+ `datasets.list_versions(name, warehouse="warehouse")` and
1325
+ `datasets.get_version("name@vN", warehouse="warehouse")` find versions again. From the command
1326
+ line, `ldp datasets pin CONFIG NAME`, `ldp datasets list` and `ldp datasets export` do the same.
1327
+
1328
+ The temporal quality checks suit sensor and event data:
1329
+
1330
+ | `check` | Keys | Passes when |
1331
+ |---|---|---|
1332
+ | `monotonic` | `column`, `group_by`, `strict`, `order_by` | The column never decreases (or, with `strict`, always increases) within each group |
1333
+ | `max_skew` | `column_a`, `column_b`, `max_ms` | Two timestamps on each row are at most `max_ms` apart |
1334
+ | `rate_below` | `predicate_column`, `max_rate`, `group_by` | The share of rows where the boolean column is true stays below `max_rate` |
1335
+ | `max_gap` | `column`, `max_ms`, `group_by` | Consecutive timestamps within each group are at most `max_ms` apart |
1336
+
1337
+ [`examples/robot_episodes/`](examples/robot_episodes/) puts it together: deterministic
1338
+ humanoid-robot episodes and sensor frames, bronze, silver and gold tables (episodes partitioned by
1339
+ `day(start_ts)` and `bucket(8, robot_id)`), quality configs for the frame-drop rate, RGB/depth sync
1340
+ skew, monotonic frame timestamps and accepted tasks, a pinned training split, DuckDB research
1341
+ queries and an optional Scala Spark per-robot aggregate. It runs offline in about a second with
1342
+ `make demo-robotics`. See [Robot-episode datasets](docs/robotics.md).
1343
+
1344
+ ## Table maintenance
1345
+
1346
+ ```python
1347
+ import datetime as dt
1348
+
1349
+ from local_data_platform import maintenance
1350
+ from local_data_platform.format.iceberg import Iceberg
1351
+
1352
+ table = Iceberg("rides", {"identifier": "nyc", "warehouse_path": "warehouse"})
1353
+ plan = maintenance.expire_snapshots(table, older_than=dt.timedelta(days=7), dry_run=True)
1354
+ print(plan["expired_snapshot_ids"], plan["protected_snapshot_ids"]["idempotency_key"])
1355
+ print(maintenance.find_orphans(table)) # files no metadata references; deletes nothing
1356
+ report = maintenance.remove_orphans(table) # a dry run unless dry_run=False
1357
+ ```
1358
+
1359
+ Expiry always keeps the current snapshot, branch and tag heads and `main`'s last 20 snapshots
1360
+ (`retain_last`), and it never expires a snapshot that carries an idempotency key newer than the
1361
+ table's idempotency horizon (7 days by default), because re-runs rely on finding it. Orphans are
1362
+ files under the table's location that no metadata references and that are older than 72 hours
1363
+ (`older_than_hours`), so the files of a write still in progress are left alone. `ldp maintain
1364
+ CONFIG` runs both from the command line, as a dry run unless `--apply` is given. There is no
1365
+ compaction yet.
1366
+
1367
+ ## Spark (experimental)
1368
+
1369
+ `engine.spark` reads and writes the same tables from Spark, in two ways: `ScalaSparkJob`, which runs
1370
+ the Scala job in [`spark/`](spark/README.md) with scala-cli (it fetches its own JDK, Spark and
1371
+ Iceberg), and `SparkEngine`, a PySpark engine (the `spark` extra plus a JDK 17+). Both open the same
1372
+ SQLite catalog as pyiceberg, and pyiceberg reads back what Spark writes. `spark_catalog_conf_for`
1373
+ turns a catalog block into Spark settings for the `local`, `sql` and `rest` types:
1374
+
1375
+ ```python
1376
+ from local_data_platform.engine.spark import spark_catalog_conf_for
1377
+
1378
+ conf = spark_catalog_conf_for({"identifier": "nyc", "warehouse_path": "warehouse"})
1379
+ print(conf["spark.sql.catalog.nyc.catalog-impl"]) # org.apache.iceberg.jdbc.JdbcCatalog
1380
+ ```
1381
+
1382
+ `make demo-spark` runs the Scala job on the demo table, and `ldp spark CONFIG` on any config's
1383
+ table. The integration tests are opt-in
1384
+ (`make spark-test`, which sets `LDP_RUN_SPARK=1`). No pipeline uses Spark, and DuckDB is the
1385
+ supported SQL engine. [Spark](docs/spark.md) covers versions, the shared catalog and the limits.
1386
+
1387
+ ## Architecture
1388
+
1389
+ ```text
1390
+ rides.json ──► Config.from_json ──► registry.create_pipeline ──► Pipeline.run(mode, commit=, sink=)
1391
+ │
1392
+ ┌──────────────────────────────────────────────────────────┘
1393
+ ▼
1394
+ source.get() ──► transforms ──► quality checks ──► target.put(mode, commit=)
1395
+ CSV · Parquet · your pyarrow fail: raise direct: one transaction, file lock
1396
+ JSON (local, s3, callables DataQualityError, staged: branch ─► checks ─► publish (CAS)
1397
+ gs via fs.py) · write nothing CSV · Parquet targets
1398
+ Iceberg · BigQuery │
1399
+ ▼
1400
+ catalog provider: local (SQLite + folder) · sql · rest · glue ── warehouse: file · s3 · gs
1401
+ │
1402
+ RunEvents ──► JSONL · OpenLineage · Iceberg (_ldp.runs, _ldp.quality_results)
1403
+ │
1404
+ read by: DuckDB (iceberg_scan) · Spark (experimental) · MCP server (read-only, audited to _ldp.audit) · datasets.pin
1405
+ ```
1406
+
1407
+ Data moves between steps as `pyarrow.Table`s. The package lives in `src/local_data_platform/`:
1408
+
1409
+ | Module | Purpose |
1410
+ |---|---|
1411
+ | `config`, `paths` | `Config` and path resolution relative to the config file |
1412
+ | `spec` | The `ldp/v1` JSON Schema, config validation, `spec_hash` and `idempotency_key` |
1413
+ | `fs` | Local and object-storage IO (`file://`, `s3://`, `gs://`) with atomic writes |
1414
+ | `format` | `CSV`, `Parquet` and `Iceberg` tables (`get` / `put`), and `WriteResult` |
1415
+ | `format.iceberg.commit` | The staged publish protocol: `CommitContext`, `write_once`, `stage`, `publish`, `fence` |
1416
+ | `catalog` | `create_catalog` and the `local`, `sql`, `rest` and `glue` types; `LocalIcebergCatalog` |
1417
+ | `store` | Sources: BigQuery (`[bigquery]` extra), JSON files, and older import paths kept for compatibility |
1418
+ | `quality` | Checks, including the temporal ones in `quality.temporal`, `run_checks` and `QualityReport` |
1419
+ | `events` | `RunEvent` and the event sinks |
1420
+ | `engine.duckdb` | `DuckDBEngine` for SQL over Iceberg and Arrow tables (`[duckdb]` extra) |
1421
+ | `engine.router` | Scan-size estimates and engine choice |
1422
+ | `engine.spark` | Experimental: `SparkEngine`, `ScalaSparkJob` and `spark_catalog_conf_for`; see [Spark](docs/spark.md) |
1423
+ | `mcp_server` | The read-only MCP server (`[mcp]` extra) |
1424
+ | `datasets` | Snapshot-pinned dataset versions |
1425
+ | `maintenance` | Snapshot expiry with the idempotency floor, and orphan-file detection |
1426
+ | `pipeline` | `Pipeline`, the built-in pipelines and the registry (`create_pipeline`) |
1427
+ | `cli`, `demo` | The `ldp` command and the demo |
1428
+ | `github`, `issue` | GitHub REST helpers used by the project's docs tooling |
1429
+
1430
+ [Why a registry instead of if/else](docs/design/factory_registry.md) explains how pipelines are
1431
+ chosen. The [SaaS architecture](docs/design/saas_architecture.md) is a proposal for where this
1432
+ could go; none of its control plane exists.
1433
+
1434
+ ## Development
1435
+
1436
+ ```bash
1437
+ make install # create .venv and install the package with the dev and docs extras
1438
+ make lint # flake8
1439
+ make test # pytest, offline: the Spark and REST integration tests skip
1440
+ make demo # run the demo into ./ldp_demo
1441
+ make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
1442
+ make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
1443
+ make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
1444
+ make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
1445
+ make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
1446
+ make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
1447
+ make docs # mkdocs build --strict
1448
+ make build # build the sdist and wheel into dist/
1449
+ make smoke # install the wheel in a fresh venv and run ldp --version and ldp demo
1450
+ ```
1451
+
1452
+ The JVM demos and tests use [scala-cli](https://scala-cli.virtuslab.org), which provisions its own
1453
+ Temurin 17 JDK; the first run downloads it with Spark, Iceberg and their jars (about 500 MB).
1454
+ `tests/test_rest_catalog.py` and `make demo-rest` start the Iceberg REST fixture in
1455
+ `tools/rest_fixture` (`make demo-rest` on port 8181; `REST_PORT=` changes it) and stop it
1456
+ afterwards, also when a step fails.
1457
+
1458
+ CI runs lint, tests on Python 3.12 and 3.13 against both pyiceberg 0.11.0 and the newest supported
1459
+ release, the wheel smoke test and a strict docs build. The Spark and REST jobs are in a separate,
1460
+ opt-in workflow (`.github/workflows/jvm.yml`): they run when started by hand, weekly, or on a pull
1461
+ request labelled `jvm`, and never block a merge. See the [changelog](CHANGELOG.md) for what changed
1462
+ in each release, and the [contributing guide](docs/contributing.md) to get involved.
1463
+
1464
+ ## Known limitations
1465
+
1466
+ - **The engine router is unused.** Nothing calls `engine.router` yet.
1467
+ - **Remote catalogs are lightly tested.** `rest` is tested against Iceberg's REST test server and
1468
+ `glue` against moto. Postgres, hosted REST catalogs and GCS have no integration tests yet.
1469
+ - **Direct overwrite and upsert on a shared catalog are single-writer.** Several writers need the
1470
+ staged protocol (`commit=`). The local file lock only covers processes on one machine.
1471
+ - **Exactly once means at most one effect per key.** Duplicates a producer sent upstream are not
1472
+ removed.
1473
+ - **An upsert batch needs every column of the table.** An upsert replaces whole rows, so a batch
1474
+ that lacks one of the table's columns is rejected with a `ConfigError` that names them (appends
1475
+ accept it).
1476
+ - **Mostly in memory.** Reads return a whole `pyarrow.Table`. DuckDB's native `iceberg_scan`
1477
+ streams, but pipelines don't; there are no streaming or incremental reads.
1478
+ - **Spark is experimental.** It needs a JDK (or scala-cli) and downloads jars on first use, no
1479
+ pipeline uses it, and its tests are opt-in.
1480
+ - **The MCP server is read-only and local.** It speaks MCP over stdio only. Its `_ldp.audit` rows
1481
+ are appended when it stops, so a server that is killed leaves them only in the JSONL audit file.
1482
+ - **No compaction.** Maintenance expires snapshots and finds orphans; it doesn't rewrite data files.
1483
+ - **BigQuery is read-only.** It can be a source, not a target.
1484
+ - **`ldp dbt` publishes one model at a time.** Nothing is published when dbt fails, but the writes of a
1485
+ successful build are separate commits, so a write that fails, such as a blocked quality check or a model
1486
+ with no rows, leaves the models before it published. A model published with an upsert that gains or loses
1487
+ a column needs one `ldp run CONFIG --mode overwrite`.
1488
+ - **The restaurant example is in the repo, not in the package.** `pip install` does not include
1489
+ `examples/`, so the warehousing and orchestration guides and `make demo-warehouse`, `make demo-orchestration` and
1490
+ `make demo-gold` need a clone.
1491
+ - **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
1492
+ respects DuckDB's single writer. There are no parallel tasks.
1493
+ - **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
1494
+ foreground loop and no systemd or Windows Task Scheduler recipe. A window that failed is run again by every tick, every
1495
+ 5 minutes under the recipes, with no wait between ticks. `retries` are the retries inside one tick.
1496
+ - **A dbt task builds its whole selection for every window.** dbt gets `window_start` and `window_end`, but only a
1497
+ model that reads them is limited to the window, and the models of the restaurant example are not.
1498
+ - **Airflow runs one config for each task.** DAGs built from workflow files, an `LdpTaskOperator` and managed Airflow
1499
+ recipes are planned for 0.2.0.
1500
+ - **A release covers one namespace of one warehouse.** `ldp gold build` refuses a folder whose gold tables are in more
1501
+ than one place, and says to give each place its own folder of configs.
1502
+ - **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
1503
+ question, so ask for them in separate queries.
1504
+
1505
+ ## Roadmap
1506
+
1507
+ From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
1508
+ of a data platform, and every layer runs on a laptop first. 0.1.4 Self Serving Gold Layer is released, and these
1509
+ are the milestones after it:
1510
+
1511
+ | Version | Milestone | What it adds |
1512
+ |---|---|---|
1513
+ | 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
1514
+ | 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
1515
+ | 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |
1516
+ | 0.1.8 | LLM | Catalog search and `ldp ask`, text to SQL that corrects itself |
1517
+ | 0.1.9 | Launch Documentation | The Learn path, generated reference pages, onboarding and contributor setup |
1518
+ | 0.2.0 | Cloud Integration | Profiles, cloud catalogs and storage, Snowflake and one container image |
1519
+
1520
+ The [roadmap](docs/roadmap.md) has the goal, issues and demo of each milestone.
1521
+
1522
+ ## References
1523
+
1524
+ - [PyIceberg](https://py.iceberg.apache.org)
1525
+ - [DuckDB Iceberg extension](https://duckdb.org/docs/extensions/iceberg.html)
1526
+ - [Model Context Protocol](https://modelcontextprotocol.io)
1527
+ - [OpenLineage](https://openlineage.io)
1528
+ - [NEAR Lake Framework](https://docs.near.org/concepts/advanced/near-lake-framework)
1529
+ - [Reliable change data capture using Iceberg](https://medium.com/@tushar.choudhary.de/reliable-cdc-apache-spark-ingestion-pipeline-using-iceberg-5d8f0fee6fd6)
1530
+ - [Internals of Apache PyIceberg](https://medium.com/@tushar.choudhary.de/internals-of-apache-pyiceberg-10c2302a5c8b)
1531
+
1532
+ ## License
1533
+
1534
+ MIT. See [LICENSE](LICENSE).
1535
+