local-data-platform 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/PKG-INFO +45 -14
  2. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/README.md +44 -13
  3. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/pyproject.toml +1 -1
  4. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/__init__.py +1 -1
  5. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/cli.py +12 -5
  6. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/config.py +20 -3
  7. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/events.py +162 -28
  8. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/build.py +9 -3
  9. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/__init__.py +8 -4
  10. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/cli.py +145 -5
  11. local_data_platform-0.1.5/src/local_data_platform/maintenance/system.py +213 -0
  12. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/discovery.py +18 -7
  13. local_data_platform-0.1.5/src/local_data_platform/mcp_server/sla.py +146 -0
  14. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/tools.py +7 -3
  15. local_data_platform-0.1.5/src/local_data_platform/monitoring/__init__.py +18 -0
  16. local_data_platform-0.1.5/src/local_data_platform/monitoring/alerts.py +189 -0
  17. local_data_platform-0.1.5/src/local_data_platform/monitoring/cli.py +201 -0
  18. local_data_platform-0.1.5/src/local_data_platform/monitoring/configs.py +71 -0
  19. local_data_platform-0.1.5/src/local_data_platform/monitoring/evaluation.py +78 -0
  20. local_data_platform-0.1.5/src/local_data_platform/monitoring/freshness.py +136 -0
  21. local_data_platform-0.1.5/src/local_data_platform/monitoring/history.py +200 -0
  22. local_data_platform-0.1.5/src/local_data_platform/monitoring/ledger.py +197 -0
  23. local_data_platform-0.1.5/src/local_data_platform/monitoring/metric.py +430 -0
  24. local_data_platform-0.1.5/src/local_data_platform/monitoring/results.py +136 -0
  25. local_data_platform-0.1.5/src/local_data_platform/monitoring/schema.py +216 -0
  26. local_data_platform-0.1.5/src/local_data_platform/monitoring/sla.py +709 -0
  27. local_data_platform-0.1.5/src/local_data_platform/monitoring/status.py +333 -0
  28. local_data_platform-0.1.5/src/local_data_platform/monitoring/status_page.py +117 -0
  29. local_data_platform-0.1.5/src/local_data_platform/monitoring/target.py +70 -0
  30. local_data_platform-0.1.5/src/local_data_platform/monitoring/volume.py +204 -0
  31. local_data_platform-0.1.5/src/local_data_platform/notify.py +680 -0
  32. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/spec.py +57 -7
  33. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/LICENSE +0 -0
  34. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/__init__.py +0 -0
  35. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/local/__init__.py +0 -0
  36. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/local/iceberg/__init__.py +0 -0
  37. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/provider.py +0 -0
  38. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/contracts.py +0 -0
  39. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/datasets.py +0 -0
  40. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/__init__.py +0 -0
  41. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/build.py +0 -0
  42. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/cli.py +0 -0
  43. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/project.py +0 -0
  44. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/publish.py +0 -0
  45. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/sources.py +0 -0
  46. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/demo.py +0 -0
  47. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/__init__.py +0 -0
  48. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/duckdb/__init__.py +0 -0
  49. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/duckdb/row_filter.py +0 -0
  50. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/router.py +0 -0
  51. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/spark/__init__.py +0 -0
  52. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/etl.py +0 -0
  53. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/exceptions.py +0 -0
  54. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/filelock.py +0 -0
  55. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/__init__.py +0 -0
  56. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/csv/__init__.py +0 -0
  57. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/excel/__init__.py +0 -0
  58. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/iceberg/__init__.py +0 -0
  59. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/iceberg/commit.py +0 -0
  60. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/parquet/__init__.py +0 -0
  61. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/fs.py +0 -0
  62. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/github/__init__.py +0 -0
  63. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/__init__.py +0 -0
  64. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/catalog.py +0 -0
  65. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/cli.py +0 -0
  66. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/release.py +0 -0
  67. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/render.py +0 -0
  68. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/serve.py +0 -0
  69. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/status.py +0 -0
  70. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/sync.py +0 -0
  71. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/html.py +0 -0
  72. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/integrations/__init__.py +0 -0
  73. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/integrations/airflow.py +0 -0
  74. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/issue/__init__.py +0 -0
  75. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/lake.py +0 -0
  76. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/logger.py +0 -0
  77. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/_common.py +0 -0
  78. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/orphans.py +0 -0
  79. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/snapshots.py +0 -0
  80. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/__init__.py +0 -0
  81. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/__main__.py +0 -0
  82. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/audit.py +0 -0
  83. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/cli.py +0 -0
  84. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/errors.py +0 -0
  85. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/guard.py +0 -0
  86. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/sandbox.py +0 -0
  87. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/server.py +0 -0
  88. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/__init__.py +0 -0
  89. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/check.py +0 -0
  90. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/cli.py +0 -0
  91. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/compiler.py +0 -0
  92. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/load.py +0 -0
  93. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/model.py +0 -0
  94. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/render.py +0 -0
  95. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/runner.py +0 -0
  96. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/sql.py +0 -0
  97. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/__init__.py +0 -0
  98. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/backfills.py +0 -0
  99. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/cli.py +0 -0
  100. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/dbt_task.py +0 -0
  101. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/gold_task.py +0 -0
  102. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/ledger.py +0 -0
  103. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/recipes.py +0 -0
  104. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/retries.py +0 -0
  105. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/runner.py +0 -0
  106. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/schedule.py +0 -0
  107. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/schedule_cli.py +0 -0
  108. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/tasks.py +0 -0
  109. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/windows.py +0 -0
  110. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/workflow.py +0 -0
  111. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/paths.py +0 -0
  112. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/__init__.py +0 -0
  113. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/builders.py +0 -0
  114. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/builtin.py +0 -0
  115. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/__init__.py +0 -0
  116. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/csv_to_iceberg/__init__.py +0 -0
  117. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/iceberg_to_csv/__init__.py +0 -0
  118. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/iceberg_to_parquet/__init__.py +0 -0
  119. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/__init__.py +0 -0
  120. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/bigquery_to_csv/__init__.py +0 -0
  121. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/csv_to_iceberg/__init__.py +0 -0
  122. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/duckdb_to_iceberg/__init__.py +0 -0
  123. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/excel_to_csv/__init__.py +0 -0
  124. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/excel_to_iceberg/__init__.py +0 -0
  125. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/parquet_to_iceberg/__init__.py +0 -0
  126. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/pyarrow/__init__.py +0 -0
  127. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/registry.py +0 -0
  128. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/__init__.py +0 -0
  129. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/checks.py +0 -0
  130. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/report.py +0 -0
  131. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/runner.py +0 -0
  132. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/temporal.py +0 -0
  133. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/__init__.py +0 -0
  134. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/__init__.py +0 -0
  135. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/duckdb/__init__.py +0 -0
  136. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/gcp/__init__.py +0 -0
  137. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/gcp/bigquery/__init__.py +0 -0
  138. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/json/__init__.py +0 -0
  139. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/near/__init__.py +0 -0
  140. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/parquet/__init__.py +0 -0
  141. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/target/__init__.py +0 -0
  142. {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/target/iceberg/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: local-data-platform
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: A Python API layer for running a complete Iceberg lakehouse data platform on your laptop.
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -67,10 +67,14 @@ It's for learning how a lakehouse works, for building and testing pipelines loca
67
67
  for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
68
68
  tables you build.
69
69
 
70
- > **Status.** This is version 0.1.4, the Self Serving Gold Layer milestone. It adds gold tables that
71
- > carry an owner, a meaning and a description of every column, `ldp gold build` and one release of all
72
- > the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once in
73
- > `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
70
+ > **Status.** This is version 0.1.5, the Monitoring milestone. It adds an SLA in the config of a table,
71
+ > `ldp monitor` with freshness, volume, schema and metric monitors, alerts to Slack, a webhook or a file
72
+ > when a monitor changes state and when it recovers, `ldp status` as one screen for every pipeline, the
73
+ > SLA verdict of a table for agents in `ldp mcp`, and `ldp maintain --system` to keep the `_ldp` history
74
+ > small (see [Monitoring](#monitoring)), with a [guide](docs/monitoring.md). Version 0.1.4 added gold
75
+ > tables that carry an owner, a meaning and a description of every column, `ldp gold build` and one
76
+ > release of all the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once
77
+ > in `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
74
78
  > [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
75
79
  > with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
76
80
  > launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
@@ -90,7 +94,7 @@ tables you build.
90
94
  `ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
91
95
 
92
96
  ```bash
93
- pip install "local-data-platform[duckdb]>=0.1.4"
97
+ pip install "local-data-platform[duckdb]>=0.1.5"
94
98
 
95
99
  ldp demo --workdir ldp_demo
96
100
  ```
@@ -132,6 +136,7 @@ More demos, each one a `make` target (see [Development](#development)):
132
136
  | `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
133
137
  | `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
134
138
  | `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
139
+ | `make demo-monitor` | The restaurant's orders watched through five nights: a late table, a short batch, a new column and a revenue drop raise alerts, a fresh night resolves them, and `ldp status` shows every pipeline on one screen (the revenue step runs when the `dbt` extra is installed) |
135
140
  | `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
136
141
  | `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
137
142
  | `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
@@ -266,7 +271,7 @@ Every command follows one exit-code table.
266
271
  |---|---|
267
272
  | 0 | success, including a run that published and a run that skipped a duplicate |
268
273
  | 1 | an error: a bad config, a usage error, a failed write |
269
- | 2 | reserved for an SLA breach that `ldp monitor` reports in 0.1.5 |
274
+ | 2 | an SLA breach, which `ldp monitor` reports when a monitor is in error |
270
275
  | 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
271
276
 
272
277
  A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
@@ -296,7 +301,7 @@ call:
296
301
  | `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
297
302
  | `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
298
303
  | `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
299
- | `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
304
+ | `ldp maintain CONFIG [--system [--keep-days N]] [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply`. With `--system` it trims the `_ldp` history tables of the config's catalog instead, deleting the rows older than N days (90 by default) first | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans`, `delete_old_rows`, `maintain_system_tables` |
300
305
  | `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
301
306
  | `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
302
307
  | `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
@@ -310,6 +315,9 @@ call:
310
315
  | `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
311
316
  | `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
312
317
  | `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
318
+ | `ldp monitor CONFIG [--json] [--now TIME]` | Check the SLA of a config, or of every dataset config of a folder, and print one line for each monitor. The freshness monitor measures the age of a table from the metadata of its last commit, or from the newest value of `freshness.column`, and scans no rows. The volume monitor compares the rows of the latest run with `min_rows` and `max_rows`, and with the median of the runs before it, and the schema monitor compares the columns of the table, and of its source in the latest run, with the last evaluation. A metric monitor computes a metric of the metrics file, or one query, for the latest complete period of its grain, and compares it with `min` and `max` and with the median of the periods before it in the history of the monitors, see the [monitoring guide](docs/monitoring.md). The command writes to no table and creates no catalog. It records the results in `_ldp.monitor_results` when the config has an `iceberg` sink and its catalog exists, and in `.ldp/monitor.jsonl` next to the config otherwise. A config that lists channels in `metadata.observability.alerts` is told when a monitor changes state and when it recovers, see [Alerts](docs/alerts.md). The exit status is 0 when every monitor is `ok` or `warn`, 2 when one is `error` and 1 when `ldp` itself fails | `monitoring.evaluate(config, now=...)` returns a `MonitorResult` for each monitor, and `monitoring.history.record` and `read_results` write and read the history |
319
+ | `ldp status CONFIG [--json] [--html FILE] [--now TIME]` | Print one row for each dataset config of a file or a folder, the worst first, with the last run and how long ago it started, how long ago a run succeeded, the rows of the table, what the freshness, volume, schema and metrics monitors say now, the checks that failed in the last 7 days and the open alerts. It evaluates the monitors in memory and records and sends nothing, reads each `_ldp` table once for the configs that share a catalog, and says how far back the history goes. The exit status is 0 unless `ldp` itself fails. `--json` prints the rows, and `--html FILE` also writes one self-contained page with the table, the monitors and the last 20 runs of each pipeline | `monitoring.status.rows(path, now=None)` returns a `StatusRow` for each config, and `monitoring.ledger.Ledger` reads the history of many configs with one read of each table |
320
+ | `ldp alert test CONFIG` | Send one test message to each channel of `metadata.observability.alerts`, whatever its `on` says, and print which ones got it. It sends real messages. The exit status is 0 when every channel got it and 1 otherwise | `notify.notifier_for(config).test()` returns a `Delivery` for each channel, and `notify.NotifySink` is the event sink that alerts when a run fails or is blocked |
313
321
  | `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
314
322
 
315
323
  ## Config schema
@@ -323,12 +331,13 @@ and a `metadata` block:
323
331
  | `metadata.target` | The same, plus for Iceberg `write_mode` (`append`, `overwrite` or `upsert`), `join_cols` for upserts, and `partition_by` |
324
332
  | `metadata.target.catalog` | `type` (`local` by default, `sql`, `rest` or `glue`) and that type's keys; see [Catalogs](#catalogs-and-object-storage) |
325
333
  | `metadata.quality` | `on_failure` (`fail` or `warn`) and a list of `checks`: `row_count`, `not_null`, `unique`, `accepted_values`, `range`, `freshness`, `schema`, and the temporal checks `monotonic`, `max_skew`, `rate_below` and `max_gap` |
326
- | `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`. The default sends them nowhere |
334
+ | `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`, and `alerts`, the channels that are told when a run fails or a monitor breaks and recovers, see [Alerts](docs/alerts.md). The default sends events nowhere and alerts to nobody |
327
335
  | `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
328
336
  | `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
329
337
  | `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
330
338
  | `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
331
339
  | `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
340
+ | `metadata.sla` | How fresh, how big and how stable the table should be, and which business numbers to watch, in up to four sections. `freshness` has `warn_after` and `error_after`, each a `count` of `minute`, `hour` or `day` periods such as `{"count": 26, "period": "hour"}`, and an optional `column`. `volume` has `min_rows`, `max_rows`, `max_change_pct` and `lookback_runs` (7 by default). `schema` has `on_change` (`ignore`, `warn` or `error`, `warn` by default) and `allow`, a list of `added_column` and `widened_type`. `metrics` is a list of entries that each name a `metric` of the metrics file next to the config, or give a `sql` query and a `name`, with a `grain`, `min`, `max`, `max_change_pct`, an optional `filter` and a `lookback` (7 by default). A malformed value is an error that names its path, `ldp plan` prints the SLA and reports a metric that the folder does not define, and `ldp schema` describes it. `ldp monitor` checks all four sections. Editing it does not change the spec hash |
332
341
 
333
342
  A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
334
343
  is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
@@ -1261,6 +1270,18 @@ make demo-gold # the restaurant's data mart, a faulty batch that the gate st
1261
1270
  - `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
1262
1271
  - The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
1263
1272
 
1273
+ ## Monitoring
1274
+
1275
+ A table can be wrong for days before a person notices. A config says in `metadata.sla` how fresh, how big and how stable its table should be, and which business number must stay above a floor. `ldp monitor` checks the table against it between runs, an alert goes to Slack, a webhook or a file when a monitor changes state and again when it recovers, and `ldp status` shows every pipeline on one screen. Nothing needs a server, and a laptop can run it from cron or launchd.
1276
+
1277
+ ```bash
1278
+ make demo-monitor # a late table, a short batch, a new column and a revenue drop raise alerts, and a fresh night resolves them
1279
+ ```
1280
+
1281
+ - The [monitoring guide](docs/monitoring.md) walks through it on the restaurant example, command by command, from an SLA to a late table, a short batch, a new column, a day of low revenue, alerts that say each problem once, the status screen, and a cron or launchd job that looks every 15 minutes with a Healthchecks.io heartbeat. It says plainly what a monitor on a laptop cannot do, and every command in it runs in the tests.
1282
+ - `make demo-monitor` runs `examples/restaurant/run.py --layer monitoring`. It loads the orders of one night and runs `ldp monitor --now` 50 hours later, which reports a freshness error with exit status 2 and writes one alert to `.ldp/alerts.jsonl`. A short batch raises a volume error, a batch with a new column raises a schema warning, and a day with a tenth of the revenue raises an error on the `revenue` metric. A fresh night resolves them, a second look sends nothing, and `ldp status` shows the result. With `LDP_SLACK_WEBHOOK` set the same alerts reach Slack, and without it the demo is fully offline. It needs no extra, runs the revenue step when the `dbt` extra is installed, and prints a skip line when it is not. It takes about 15 seconds with dbt and about a second without it.
1283
+ - The commands are in the [platform commands](#platform-commands) table, and the [alerts page](docs/alerts.md) describes the channels and the events.
1284
+
1264
1285
  ## MCP server for agents
1265
1286
 
1266
1287
  `local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
@@ -1361,7 +1382,10 @@ Expiry always keeps the current snapshot, branch and tag heads and `main`'s last
1361
1382
  table's idempotency horizon (7 days by default), because re-runs rely on finding it. Orphans are
1362
1383
  files under the table's location that no metadata references and that are older than 72 hours
1363
1384
  (`older_than_hours`), so the files of a write still in progress are left alone. `ldp maintain
1364
- CONFIG` runs both from the command line, as a dry run unless `--apply` is given. There is no
1385
+ CONFIG` runs both from the command line, as a dry run unless `--apply` is given. `ldp maintain
1386
+ CONFIG --system` does the same for the `_ldp` history tables of the config's catalog, after it
1387
+ deletes their rows older than `--keep-days` days (90 by default), a whole UTC day at a time, which
1388
+ drops data files and rewrites none. Run it once a week where `ldp monitor` runs often. There is no
1365
1389
  compaction yet.
1366
1390
 
1367
1391
  ## Spark (experimental)
@@ -1441,6 +1465,7 @@ make demo # run the demo into ./ldp_demo
1441
1465
  make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
1442
1466
  make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
1443
1467
  make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
1468
+ make demo-monitor # the restaurant's orders watched through a late table, a short batch, a new column and a revenue drop
1444
1469
  make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
1445
1470
  make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
1446
1471
  make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
@@ -1486,8 +1511,8 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1486
1511
  with no rows, leaves the models before it published. A model published with an upsert that gains or loses
1487
1512
  a column needs one `ldp run CONFIG --mode overwrite`.
1488
1513
  - **The restaurant example is in the repo, not in the package.** `pip install` does not include
1489
- `examples/`, so the warehousing and orchestration guides and `make demo-warehouse`, `make demo-orchestration` and
1490
- `make demo-gold` need a clone.
1514
+ `examples/`, so the warehousing, orchestration and monitoring guides and `make demo-warehouse`, `make demo-orchestration`,
1515
+ `make demo-gold` and `make demo-monitor` need a clone.
1491
1516
  - **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
1492
1517
  respects DuckDB's single writer. There are no parallel tasks.
1493
1518
  - **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
@@ -1501,16 +1526,22 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1501
1526
  than one place, and says to give each place its own folder of configs.
1502
1527
  - **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
1503
1528
  question, so ask for them in separate queries.
1529
+ - **A monitor looks at one moment, on a machine that is running.** A table that was late and was fixed between two looks
1530
+ never alerts, and nothing alerts while the laptop is asleep or off. A heartbeat to an outside service such as
1531
+ Healthchecks.io is the only cover, and the [monitoring guide](docs/monitoring.md) says how to set it up.
1532
+ - **A change is measured against the median of the periods before it.** It needs 3 earlier values and has no sense of the
1533
+ weekday, so a business that is busier at weekends needs a wide `max_change_pct`, or a `min` and a `max`.
1534
+ - **A local catalog keeps its history on one machine.** Two machines that look at the same table each keep a history of
1535
+ their own, and each one alerts.
1504
1536
 
1505
1537
  ## Roadmap
1506
1538
 
1507
1539
  From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
1508
- of a data platform, and every layer runs on a laptop first. 0.1.4 Self Serving Gold Layer is released, and these
1540
+ of a data platform, and every layer runs on a laptop first. 0.1.5 Monitoring is released, and these
1509
1541
  are the milestones after it:
1510
1542
 
1511
1543
  | Version | Milestone | What it adds |
1512
1544
  |---|---|---|
1513
- | 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
1514
1545
  | 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
1515
1546
  | 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |
1516
1547
  | 0.1.8 | LLM | Catalog search and `ldp ask`, text to SQL that corrects itself |
@@ -10,10 +10,14 @@ It's for learning how a lakehouse works, for building and testing pipelines loca
10
10
  for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
11
11
  tables you build.
12
12
 
13
- > **Status.** This is version 0.1.4, the Self Serving Gold Layer milestone. It adds gold tables that
14
- > carry an owner, a meaning and a description of every column, `ldp gold build` and one release of all
15
- > the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once in
16
- > `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
13
+ > **Status.** This is version 0.1.5, the Monitoring milestone. It adds an SLA in the config of a table,
14
+ > `ldp monitor` with freshness, volume, schema and metric monitors, alerts to Slack, a webhook or a file
15
+ > when a monitor changes state and when it recovers, `ldp status` as one screen for every pipeline, the
16
+ > SLA verdict of a table for agents in `ldp mcp`, and `ldp maintain --system` to keep the `_ldp` history
17
+ > small (see [Monitoring](#monitoring)), with a [guide](docs/monitoring.md). Version 0.1.4 added gold
18
+ > tables that carry an owner, a meaning and a description of every column, `ldp gold build` and one
19
+ > release of all the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once
20
+ > in `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
17
21
  > [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
18
22
  > with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
19
23
  > launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
@@ -33,7 +37,7 @@ tables you build.
33
37
  `ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
34
38
 
35
39
  ```bash
36
- pip install "local-data-platform[duckdb]>=0.1.4"
40
+ pip install "local-data-platform[duckdb]>=0.1.5"
37
41
 
38
42
  ldp demo --workdir ldp_demo
39
43
  ```
@@ -75,6 +79,7 @@ More demos, each one a `make` target (see [Development](#development)):
75
79
  | `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
76
80
  | `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
77
81
  | `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
82
+ | `make demo-monitor` | The restaurant's orders watched through five nights: a late table, a short batch, a new column and a revenue drop raise alerts, a fresh night resolves them, and `ldp status` shows every pipeline on one screen (the revenue step runs when the `dbt` extra is installed) |
78
83
  | `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
79
84
  | `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
80
85
  | `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
@@ -209,7 +214,7 @@ Every command follows one exit-code table.
209
214
  |---|---|
210
215
  | 0 | success, including a run that published and a run that skipped a duplicate |
211
216
  | 1 | an error: a bad config, a usage error, a failed write |
212
- | 2 | reserved for an SLA breach that `ldp monitor` reports in 0.1.5 |
217
+ | 2 | an SLA breach, which `ldp monitor` reports when a monitor is in error |
213
218
  | 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
214
219
 
215
220
  A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
@@ -239,7 +244,7 @@ call:
239
244
  | `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
240
245
  | `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
241
246
  | `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
242
- | `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
247
+ | `ldp maintain CONFIG [--system [--keep-days N]] [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply`. With `--system` it trims the `_ldp` history tables of the config's catalog instead, deleting the rows older than N days (90 by default) first | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans`, `delete_old_rows`, `maintain_system_tables` |
243
248
  | `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
244
249
  | `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
245
250
  | `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
@@ -253,6 +258,9 @@ call:
253
258
  | `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
254
259
  | `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
255
260
  | `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
261
+ | `ldp monitor CONFIG [--json] [--now TIME]` | Check the SLA of a config, or of every dataset config of a folder, and print one line for each monitor. The freshness monitor measures the age of a table from the metadata of its last commit, or from the newest value of `freshness.column`, and scans no rows. The volume monitor compares the rows of the latest run with `min_rows` and `max_rows`, and with the median of the runs before it, and the schema monitor compares the columns of the table, and of its source in the latest run, with the last evaluation. A metric monitor computes a metric of the metrics file, or one query, for the latest complete period of its grain, and compares it with `min` and `max` and with the median of the periods before it in the history of the monitors, see the [monitoring guide](docs/monitoring.md). The command writes to no table and creates no catalog. It records the results in `_ldp.monitor_results` when the config has an `iceberg` sink and its catalog exists, and in `.ldp/monitor.jsonl` next to the config otherwise. A config that lists channels in `metadata.observability.alerts` is told when a monitor changes state and when it recovers, see [Alerts](docs/alerts.md). The exit status is 0 when every monitor is `ok` or `warn`, 2 when one is `error` and 1 when `ldp` itself fails | `monitoring.evaluate(config, now=...)` returns a `MonitorResult` for each monitor, and `monitoring.history.record` and `read_results` write and read the history |
262
+ | `ldp status CONFIG [--json] [--html FILE] [--now TIME]` | Print one row for each dataset config of a file or a folder, the worst first, with the last run and how long ago it started, how long ago a run succeeded, the rows of the table, what the freshness, volume, schema and metrics monitors say now, the checks that failed in the last 7 days and the open alerts. It evaluates the monitors in memory and records and sends nothing, reads each `_ldp` table once for the configs that share a catalog, and says how far back the history goes. The exit status is 0 unless `ldp` itself fails. `--json` prints the rows, and `--html FILE` also writes one self-contained page with the table, the monitors and the last 20 runs of each pipeline | `monitoring.status.rows(path, now=None)` returns a `StatusRow` for each config, and `monitoring.ledger.Ledger` reads the history of many configs with one read of each table |
263
+ | `ldp alert test CONFIG` | Send one test message to each channel of `metadata.observability.alerts`, whatever its `on` says, and print which ones got it. It sends real messages. The exit status is 0 when every channel got it and 1 otherwise | `notify.notifier_for(config).test()` returns a `Delivery` for each channel, and `notify.NotifySink` is the event sink that alerts when a run fails or is blocked |
256
264
  | `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
257
265
 
258
266
  ## Config schema
@@ -266,12 +274,13 @@ and a `metadata` block:
266
274
  | `metadata.target` | The same, plus for Iceberg `write_mode` (`append`, `overwrite` or `upsert`), `join_cols` for upserts, and `partition_by` |
267
275
  | `metadata.target.catalog` | `type` (`local` by default, `sql`, `rest` or `glue`) and that type's keys; see [Catalogs](#catalogs-and-object-storage) |
268
276
  | `metadata.quality` | `on_failure` (`fail` or `warn`) and a list of `checks`: `row_count`, `not_null`, `unique`, `accepted_values`, `range`, `freshness`, `schema`, and the temporal checks `monotonic`, `max_skew`, `rate_below` and `max_gap` |
269
- | `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`. The default sends them nowhere |
277
+ | `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`, and `alerts`, the channels that are told when a run fails or a monitor breaks and recovers, see [Alerts](docs/alerts.md). The default sends events nowhere and alerts to nobody |
270
278
  | `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
271
279
  | `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
272
280
  | `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
273
281
  | `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
274
282
  | `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
283
+ | `metadata.sla` | How fresh, how big and how stable the table should be, and which business numbers to watch, in up to four sections. `freshness` has `warn_after` and `error_after`, each a `count` of `minute`, `hour` or `day` periods such as `{"count": 26, "period": "hour"}`, and an optional `column`. `volume` has `min_rows`, `max_rows`, `max_change_pct` and `lookback_runs` (7 by default). `schema` has `on_change` (`ignore`, `warn` or `error`, `warn` by default) and `allow`, a list of `added_column` and `widened_type`. `metrics` is a list of entries that each name a `metric` of the metrics file next to the config, or give a `sql` query and a `name`, with a `grain`, `min`, `max`, `max_change_pct`, an optional `filter` and a `lookback` (7 by default). A malformed value is an error that names its path, `ldp plan` prints the SLA and reports a metric that the folder does not define, and `ldp schema` describes it. `ldp monitor` checks all four sections. Editing it does not change the spec hash |
275
284
 
276
285
  A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
277
286
  is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
@@ -1204,6 +1213,18 @@ make demo-gold # the restaurant's data mart, a faulty batch that the gate st
1204
1213
  - `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
1205
1214
  - The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
1206
1215
 
1216
+ ## Monitoring
1217
+
1218
+ A table can be wrong for days before a person notices. A config says in `metadata.sla` how fresh, how big and how stable its table should be, and which business number must stay above a floor. `ldp monitor` checks the table against it between runs, an alert goes to Slack, a webhook or a file when a monitor changes state and again when it recovers, and `ldp status` shows every pipeline on one screen. Nothing needs a server, and a laptop can run it from cron or launchd.
1219
+
1220
+ ```bash
1221
+ make demo-monitor # a late table, a short batch, a new column and a revenue drop raise alerts, and a fresh night resolves them
1222
+ ```
1223
+
1224
+ - The [monitoring guide](docs/monitoring.md) walks through it on the restaurant example, command by command, from an SLA to a late table, a short batch, a new column, a day of low revenue, alerts that say each problem once, the status screen, and a cron or launchd job that looks every 15 minutes with a Healthchecks.io heartbeat. It says plainly what a monitor on a laptop cannot do, and every command in it runs in the tests.
1225
+ - `make demo-monitor` runs `examples/restaurant/run.py --layer monitoring`. It loads the orders of one night and runs `ldp monitor --now` 50 hours later, which reports a freshness error with exit status 2 and writes one alert to `.ldp/alerts.jsonl`. A short batch raises a volume error, a batch with a new column raises a schema warning, and a day with a tenth of the revenue raises an error on the `revenue` metric. A fresh night resolves them, a second look sends nothing, and `ldp status` shows the result. With `LDP_SLACK_WEBHOOK` set the same alerts reach Slack, and without it the demo is fully offline. It needs no extra, runs the revenue step when the `dbt` extra is installed, and prints a skip line when it is not. It takes about 15 seconds with dbt and about a second without it.
1226
+ - The commands are in the [platform commands](#platform-commands) table, and the [alerts page](docs/alerts.md) describes the channels and the events.
1227
+
1207
1228
  ## MCP server for agents
1208
1229
 
1209
1230
  `local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
@@ -1304,7 +1325,10 @@ Expiry always keeps the current snapshot, branch and tag heads and `main`'s last
1304
1325
  table's idempotency horizon (7 days by default), because re-runs rely on finding it. Orphans are
1305
1326
  files under the table's location that no metadata references and that are older than 72 hours
1306
1327
  (`older_than_hours`), so the files of a write still in progress are left alone. `ldp maintain
1307
- CONFIG` runs both from the command line, as a dry run unless `--apply` is given. There is no
1328
+ CONFIG` runs both from the command line, as a dry run unless `--apply` is given. `ldp maintain
1329
+ CONFIG --system` does the same for the `_ldp` history tables of the config's catalog, after it
1330
+ deletes their rows older than `--keep-days` days (90 by default), a whole UTC day at a time, which
1331
+ drops data files and rewrites none. Run it once a week where `ldp monitor` runs often. There is no
1308
1332
  compaction yet.
1309
1333
 
1310
1334
  ## Spark (experimental)
@@ -1384,6 +1408,7 @@ make demo # run the demo into ./ldp_demo
1384
1408
  make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
1385
1409
  make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
1386
1410
  make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
1411
+ make demo-monitor # the restaurant's orders watched through a late table, a short batch, a new column and a revenue drop
1387
1412
  make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
1388
1413
  make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
1389
1414
  make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
@@ -1429,8 +1454,8 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1429
1454
  with no rows, leaves the models before it published. A model published with an upsert that gains or loses
1430
1455
  a column needs one `ldp run CONFIG --mode overwrite`.
1431
1456
  - **The restaurant example is in the repo, not in the package.** `pip install` does not include
1432
- `examples/`, so the warehousing and orchestration guides and `make demo-warehouse`, `make demo-orchestration` and
1433
- `make demo-gold` need a clone.
1457
+ `examples/`, so the warehousing, orchestration and monitoring guides and `make demo-warehouse`, `make demo-orchestration`,
1458
+ `make demo-gold` and `make demo-monitor` need a clone.
1434
1459
  - **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
1435
1460
  respects DuckDB's single writer. There are no parallel tasks.
1436
1461
  - **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
@@ -1444,16 +1469,22 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
1444
1469
  than one place, and says to give each place its own folder of configs.
1445
1470
  - **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
1446
1471
  question, so ask for them in separate queries.
1472
+ - **A monitor looks at one moment, on a machine that is running.** A table that was late and was fixed between two looks
1473
+ never alerts, and nothing alerts while the laptop is asleep or off. A heartbeat to an outside service such as
1474
+ Healthchecks.io is the only cover, and the [monitoring guide](docs/monitoring.md) says how to set it up.
1475
+ - **A change is measured against the median of the periods before it.** It needs 3 earlier values and has no sense of the
1476
+ weekday, so a business that is busier at weekends needs a wide `max_change_pct`, or a `min` and a `max`.
1477
+ - **A local catalog keeps its history on one machine.** Two machines that look at the same table each keep a history of
1478
+ their own, and each one alerts.
1447
1479
 
1448
1480
  ## Roadmap
1449
1481
 
1450
1482
  From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
1451
- of a data platform, and every layer runs on a laptop first. 0.1.4 Self Serving Gold Layer is released, and these
1483
+ of a data platform, and every layer runs on a laptop first. 0.1.5 Monitoring is released, and these
1452
1484
  are the milestones after it:
1453
1485
 
1454
1486
  | Version | Milestone | What it adds |
1455
1487
  |---|---|---|
1456
- | 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
1457
1488
  | 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
1458
1489
  | 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |
1459
1490
  | 0.1.8 | LLM | Catalog search and `ldp ask`, text to SQL that corrects itself |
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "local-data-platform"
3
- version = "0.1.4"
3
+ version = "0.1.5"
4
4
  description = "A Python API layer for running a complete Iceberg lakehouse data platform on your laptop."
5
5
  readme = "README.md"
6
6
  license = { text = "MIT" }
@@ -15,7 +15,7 @@ from .config import Config
15
15
  from .exceptions import EngineNotFound, PipelineNotFound, TableNotFound
16
16
  from .paths import resolve_path
17
17
 
18
- __version__ = "0.1.4"
18
+ __version__ = "0.1.5"
19
19
 
20
20
  Transaction = namedtuple("Transaction", ["query", "desc"])
21
21
 
@@ -25,6 +25,9 @@ Commands::
25
25
  ldp workflow plan|run FILE ... (orchestration.cli)
26
26
  ldp backfill FILE --from START [--to END] [--dry-run] [--replace] (orchestration.cli)
27
27
  ldp schedule list|tick|install DIR ... (orchestration.schedule_cli)
28
+ ldp monitor CONFIG_OR_FOLDER [--json] [--now ISO-8601] (monitoring.cli)
29
+ ldp status CONFIG_OR_FOLDER [--json] [--html FILE] [--now ISO-8601] (monitoring.cli)
30
+ ldp alert test CONFIG (notify)
28
31
 
29
32
  The commands in the second group are added by the ``add_cli`` function of the module named
30
33
  next to them (``docs/design/v0_1_1_platform.md`` C9).
@@ -33,7 +36,7 @@ Every command follows one exit-code table::
33
36
 
34
37
  0 success, including a run that published and a run that skipped a duplicate
35
38
  1 an error: a bad config, a usage error, a failed write
36
- 2 reserved for an SLA breach that ``ldp monitor`` reports (0.1.5)
39
+ 2 an SLA breach, which ``ldp monitor`` reports when a monitor is in error
37
40
  3 a data gate held the run back, such as a quality check that blocked a write, or a gold table whose
38
41
  contract does not hold
39
42
 
@@ -67,7 +70,8 @@ PROG = "ldp"
67
70
  EXIT_OK = 0
68
71
  EXIT_ERROR = 1
69
72
  EXIT_SLA = 2
70
- """Reserved for an SLA breach that ``ldp monitor`` reports in 0.1.5. No command returns it yet."""
73
+ """An SLA breach: ``ldp monitor`` has a monitor in error. Running it again finds the same table, so a person has to look
74
+ at the table."""
71
75
  EXIT_GATE = 3
72
76
  """A data gate, such as a quality check, held the run back. A scheduler must not retry it."""
73
77
  DEFAULT_WORKDIR = "ldp_demo"
@@ -91,6 +95,8 @@ MODULE_COMMANDS = (
91
95
  ("local_data_platform.orchestration.schedule_cli", ("schedule",)),
92
96
  ("local_data_platform.gold.cli", ("gold",)),
93
97
  ("local_data_platform.metrics.cli", ("metrics",)),
98
+ ("local_data_platform.monitoring.cli", ("monitor", "status")),
99
+ ("local_data_platform.notify", ("alert",)),
94
100
  )
95
101
  """``(module, commands)``: each module's ``add_cli(subparsers, parents=...)`` registers its commands."""
96
102
 
@@ -361,7 +367,8 @@ def build_parser(*, module_commands: bool = True) -> argparse.ArgumentParser:
361
367
  parser = argparse.ArgumentParser(
362
368
  prog=PROG, parents=[common],
363
369
  description="local-data-platform: a local Iceberg lakehouse driven by JSON dataset configs.",
364
- epilog="Exit status is 0 on success, 1 on error and 3 when a data gate held a run back (2 is reserved).",
370
+ epilog="Exit status is 0 on success, 1 on error, 2 when ldp monitor finds an SLA breach and 3 when a data gate "
371
+ "held a run back.",
365
372
  )
366
373
  parser.add_argument("--version", action="version", version=f"{PROG} {__version__}")
367
374
  commands = parser.add_subparsers(dest="command", metavar="COMMAND")
@@ -478,8 +485,8 @@ def main(argv: Sequence[str] | None = None) -> int:
478
485
  argv: The arguments, without the program name. Defaults to ``sys.argv[1:]``.
479
486
 
480
487
  Returns:
481
- The exit status: 0 on success, 1 on error (including usage errors), and 3 when a data gate held
482
- the run back. See the exit-code table at the top of this module.
488
+ The exit status: 0 on success, 1 on error (including usage errors), 2 for an SLA breach that ``ldp monitor``
489
+ reports, and 3 when a data gate held the run back. See the exit-code table at the top of this module.
483
490
  """
484
491
  argv = list(sys.argv[1:] if argv is None else argv)
485
492
  parser = build_parser(module_commands=_needs_module_commands(argv))
@@ -1,7 +1,7 @@
1
1
  """Dataset configuration.
2
2
 
3
3
  A config describes one dataset with the 4W1H questions (who, what, where, when, how)
4
- plus a ``metadata`` block holding ``source``, ``target`` and optional ``quality`` and ``docs``
4
+ plus a ``metadata`` block holding ``source``, ``target`` and optional ``quality``, ``docs`` and ``sla``
5
5
  sections. See ``docs/design/v0_1_1.md`` for the full schema.
6
6
  """
7
7
 
@@ -13,6 +13,7 @@ from typing import Any
13
13
 
14
14
  from . import contracts
15
15
  from .exceptions import ConfigError
16
+ from .monitoring import sla as sla_settings
16
17
  from .orchestration import retries, windows
17
18
  from .paths import resolve_path
18
19
  from .spec import API_VERSION, DEFAULT_KIND, kind_of
@@ -27,7 +28,9 @@ class Config:
27
28
  ``base_dir`` is the folder relative paths resolve against. ``Config.from_json``
28
29
  sets it to the config file's folder; when built by hand it defaults to the cwd.
29
30
  ``api_version`` is the config's ``apiVersion``, when it has one, and ``kind`` is
30
- ``Pipeline``, the only kind that is a dataset config.
31
+ ``Pipeline``, the only kind that is a dataset config. ``source_path`` is the file that ``Config.from_json`` read,
32
+ and ``None`` for a config that was built by hand. It is not part of what a config is, so two configs that differ
33
+ only in it are equal.
31
34
  """
32
35
 
33
36
  identifier: str
@@ -40,6 +43,7 @@ class Config:
40
43
  base_dir: Path | None = None
41
44
  api_version: str | None = None
42
45
  kind: str = DEFAULT_KIND
46
+ source_path: Path | None = field(default=None, compare=False, repr=False)
43
47
 
44
48
  @classmethod
45
49
  def from_dict(cls, data: dict[str, Any], base_dir: str | os.PathLike | None = None) -> "Config":
@@ -78,7 +82,9 @@ class Config:
78
82
  data = json.loads(resolved.read_text())
79
83
  except json.JSONDecodeError as exc:
80
84
  raise ConfigError(f"config file {resolved} is not valid JSON: {exc}") from exc
81
- return cls.from_dict(data, base_dir=resolved.parent)
85
+ config = cls.from_dict(data, base_dir=resolved.parent)
86
+ config.source_path = resolved
87
+ return config
82
88
 
83
89
  def validate(self) -> None:
84
90
  """Raise :class:`ConfigError` if the metadata block is incomplete, or its schedule or window is invalid."""
@@ -103,6 +109,8 @@ class Config:
103
109
  raise ConfigError(f"metadata.retries{path}: {message}")
104
110
  for path, message in contracts.docs_problems(self.metadata.get("docs"), who=self.who):
105
111
  raise ConfigError(f"metadata.docs{path}: {message}")
112
+ for path, message in sla_settings.sla_problems(self.metadata.get("sla")):
113
+ raise ConfigError(f"metadata.sla{path}: {message}")
106
114
 
107
115
  @property
108
116
  def source(self) -> dict[str, Any]:
@@ -139,6 +147,15 @@ class Config:
139
147
  """
140
148
  return contracts.parse_docs(self.metadata.get("docs"), who=self.who, when=self.when)
141
149
 
150
+ @property
151
+ def sla(self) -> sla_settings.Sla | None:
152
+ """How fresh, how big and how stable the table should be: its ``metadata.sla`` block, normalised.
153
+
154
+ The defaults are applied, so ``volume.lookback_runs`` is 7 and ``schema.on_change`` is ``warn`` when the block
155
+ sets none, and it is ``None`` when the config has no block. See :mod:`local_data_platform.monitoring.sla`.
156
+ """
157
+ return sla_settings.parse_sla(self.metadata.get("sla"))
158
+
142
159
  @property
143
160
  def quality(self) -> dict[str, Any]:
144
161
  """The quality block, normalised to ``{"on_failure": "fail" | "warn", "checks": [...]}``.