local-data-platform 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/PKG-INFO +45 -14
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/README.md +44 -13
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/pyproject.toml +1 -1
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/__init__.py +1 -1
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/cli.py +12 -5
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/config.py +20 -3
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/events.py +162 -28
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/build.py +9 -3
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/__init__.py +8 -4
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/cli.py +145 -5
- local_data_platform-0.1.5/src/local_data_platform/maintenance/system.py +213 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/discovery.py +18 -7
- local_data_platform-0.1.5/src/local_data_platform/mcp_server/sla.py +146 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/tools.py +7 -3
- local_data_platform-0.1.5/src/local_data_platform/monitoring/__init__.py +18 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/alerts.py +189 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/cli.py +201 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/configs.py +71 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/evaluation.py +78 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/freshness.py +136 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/history.py +200 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/ledger.py +197 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/metric.py +430 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/results.py +136 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/schema.py +216 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/sla.py +709 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/status.py +333 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/status_page.py +117 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/target.py +70 -0
- local_data_platform-0.1.5/src/local_data_platform/monitoring/volume.py +204 -0
- local_data_platform-0.1.5/src/local_data_platform/notify.py +680 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/spec.py +57 -7
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/LICENSE +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/local/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/local/iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/catalog/provider.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/contracts.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/datasets.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/build.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/project.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/publish.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/dbt_layer/sources.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/demo.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/duckdb/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/duckdb/row_filter.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/router.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/engine/spark/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/etl.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/exceptions.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/filelock.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/csv/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/excel/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/iceberg/commit.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/format/parquet/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/fs.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/github/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/catalog.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/release.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/render.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/serve.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/status.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/gold/sync.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/html.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/integrations/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/integrations/airflow.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/issue/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/lake.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/logger.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/_common.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/orphans.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/maintenance/snapshots.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/__main__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/audit.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/errors.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/guard.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/sandbox.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/mcp_server/server.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/check.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/compiler.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/load.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/model.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/render.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/runner.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/metrics/sql.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/backfills.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/dbt_task.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/gold_task.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/ledger.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/recipes.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/retries.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/runner.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/schedule.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/schedule_cli.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/tasks.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/windows.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/orchestration/workflow.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/paths.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/builders.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/builtin.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/csv_to_iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/iceberg_to_csv/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/egression/iceberg_to_parquet/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/bigquery_to_csv/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/csv_to_iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/duckdb_to_iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/excel_to_csv/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/excel_to_iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/parquet_to_iceberg/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/ingestion/pyarrow/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/pipeline/registry.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/checks.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/report.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/runner.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/quality/temporal.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/duckdb/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/gcp/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/gcp/bigquery/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/json/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/near/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/source/parquet/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/target/__init__.py +0 -0
- {local_data_platform-0.1.4 → local_data_platform-0.1.5}/src/local_data_platform/store/target/iceberg/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: local-data-platform
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: A Python API layer for running a complete Iceberg lakehouse data platform on your laptop.
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -67,10 +67,14 @@ It's for learning how a lakehouse works, for building and testing pipelines loca
|
|
|
67
67
|
for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
|
|
68
68
|
tables you build.
|
|
69
69
|
|
|
70
|
-
> **Status.** This is version 0.1.
|
|
71
|
-
>
|
|
72
|
-
>
|
|
73
|
-
>
|
|
70
|
+
> **Status.** This is version 0.1.5, the Monitoring milestone. It adds an SLA in the config of a table,
|
|
71
|
+
> `ldp monitor` with freshness, volume, schema and metric monitors, alerts to Slack, a webhook or a file
|
|
72
|
+
> when a monitor changes state and when it recovers, `ldp status` as one screen for every pipeline, the
|
|
73
|
+
> SLA verdict of a table for agents in `ldp mcp`, and `ldp maintain --system` to keep the `_ldp` history
|
|
74
|
+
> small (see [Monitoring](#monitoring)), with a [guide](docs/monitoring.md). Version 0.1.4 added gold
|
|
75
|
+
> tables that carry an owner, a meaning and a description of every column, `ldp gold build` and one
|
|
76
|
+
> release of all the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once
|
|
77
|
+
> in `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
|
|
74
78
|
> [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
|
|
75
79
|
> with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
|
|
76
80
|
> launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
|
|
@@ -90,7 +94,7 @@ tables you build.
|
|
|
90
94
|
`ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
|
|
91
95
|
|
|
92
96
|
```bash
|
|
93
|
-
pip install "local-data-platform[duckdb]>=0.1.
|
|
97
|
+
pip install "local-data-platform[duckdb]>=0.1.5"
|
|
94
98
|
|
|
95
99
|
ldp demo --workdir ldp_demo
|
|
96
100
|
```
|
|
@@ -132,6 +136,7 @@ More demos, each one a `make` target (see [Development](#development)):
|
|
|
132
136
|
| `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
|
|
133
137
|
| `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
|
|
134
138
|
| `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
|
|
139
|
+
| `make demo-monitor` | The restaurant's orders watched through five nights: a late table, a short batch, a new column and a revenue drop raise alerts, a fresh night resolves them, and `ldp status` shows every pipeline on one screen (the revenue step runs when the `dbt` extra is installed) |
|
|
135
140
|
| `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
|
|
136
141
|
| `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
|
|
137
142
|
| `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
|
|
@@ -266,7 +271,7 @@ Every command follows one exit-code table.
|
|
|
266
271
|
|---|---|
|
|
267
272
|
| 0 | success, including a run that published and a run that skipped a duplicate |
|
|
268
273
|
| 1 | an error: a bad config, a usage error, a failed write |
|
|
269
|
-
| 2 |
|
|
274
|
+
| 2 | an SLA breach, which `ldp monitor` reports when a monitor is in error |
|
|
270
275
|
| 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
|
|
271
276
|
|
|
272
277
|
A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
|
|
@@ -296,7 +301,7 @@ call:
|
|
|
296
301
|
| `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
|
|
297
302
|
| `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
|
|
298
303
|
| `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
|
|
299
|
-
| `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
|
|
304
|
+
| `ldp maintain CONFIG [--system [--keep-days N]] [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply`. With `--system` it trims the `_ldp` history tables of the config's catalog instead, deleting the rows older than N days (90 by default) first | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans`, `delete_old_rows`, `maintain_system_tables` |
|
|
300
305
|
| `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
|
|
301
306
|
| `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
|
|
302
307
|
| `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
|
|
@@ -310,6 +315,9 @@ call:
|
|
|
310
315
|
| `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
|
|
311
316
|
| `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
|
|
312
317
|
| `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
|
|
318
|
+
| `ldp monitor CONFIG [--json] [--now TIME]` | Check the SLA of a config, or of every dataset config of a folder, and print one line for each monitor. The freshness monitor measures the age of a table from the metadata of its last commit, or from the newest value of `freshness.column`, and scans no rows. The volume monitor compares the rows of the latest run with `min_rows` and `max_rows`, and with the median of the runs before it, and the schema monitor compares the columns of the table, and of its source in the latest run, with the last evaluation. A metric monitor computes a metric of the metrics file, or one query, for the latest complete period of its grain, and compares it with `min` and `max` and with the median of the periods before it in the history of the monitors, see the [monitoring guide](docs/monitoring.md). The command writes to no table and creates no catalog. It records the results in `_ldp.monitor_results` when the config has an `iceberg` sink and its catalog exists, and in `.ldp/monitor.jsonl` next to the config otherwise. A config that lists channels in `metadata.observability.alerts` is told when a monitor changes state and when it recovers, see [Alerts](docs/alerts.md). The exit status is 0 when every monitor is `ok` or `warn`, 2 when one is `error` and 1 when `ldp` itself fails | `monitoring.evaluate(config, now=...)` returns a `MonitorResult` for each monitor, and `monitoring.history.record` and `read_results` write and read the history |
|
|
319
|
+
| `ldp status CONFIG [--json] [--html FILE] [--now TIME]` | Print one row for each dataset config of a file or a folder, the worst first, with the last run and how long ago it started, how long ago a run succeeded, the rows of the table, what the freshness, volume, schema and metrics monitors say now, the checks that failed in the last 7 days and the open alerts. It evaluates the monitors in memory and records and sends nothing, reads each `_ldp` table once for the configs that share a catalog, and says how far back the history goes. The exit status is 0 unless `ldp` itself fails. `--json` prints the rows, and `--html FILE` also writes one self-contained page with the table, the monitors and the last 20 runs of each pipeline | `monitoring.status.rows(path, now=None)` returns a `StatusRow` for each config, and `monitoring.ledger.Ledger` reads the history of many configs with one read of each table |
|
|
320
|
+
| `ldp alert test CONFIG` | Send one test message to each channel of `metadata.observability.alerts`, whatever its `on` says, and print which ones got it. It sends real messages. The exit status is 0 when every channel got it and 1 otherwise | `notify.notifier_for(config).test()` returns a `Delivery` for each channel, and `notify.NotifySink` is the event sink that alerts when a run fails or is blocked |
|
|
313
321
|
| `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
|
|
314
322
|
|
|
315
323
|
## Config schema
|
|
@@ -323,12 +331,13 @@ and a `metadata` block:
|
|
|
323
331
|
| `metadata.target` | The same, plus for Iceberg `write_mode` (`append`, `overwrite` or `upsert`), `join_cols` for upserts, and `partition_by` |
|
|
324
332
|
| `metadata.target.catalog` | `type` (`local` by default, `sql`, `rest` or `glue`) and that type's keys; see [Catalogs](#catalogs-and-object-storage) |
|
|
325
333
|
| `metadata.quality` | `on_failure` (`fail` or `warn`) and a list of `checks`: `row_count`, `not_null`, `unique`, `accepted_values`, `range`, `freshness`, `schema`, and the temporal checks `monotonic`, `max_skew`, `rate_below` and `max_gap` |
|
|
326
|
-
| `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}
|
|
334
|
+
| `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`, and `alerts`, the channels that are told when a run fails or a monitor breaks and recovers, see [Alerts](docs/alerts.md). The default sends events nowhere and alerts to nobody |
|
|
327
335
|
| `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
|
|
328
336
|
| `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
|
|
329
337
|
| `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
|
|
330
338
|
| `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
|
|
331
339
|
| `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
|
|
340
|
+
| `metadata.sla` | How fresh, how big and how stable the table should be, and which business numbers to watch, in up to four sections. `freshness` has `warn_after` and `error_after`, each a `count` of `minute`, `hour` or `day` periods such as `{"count": 26, "period": "hour"}`, and an optional `column`. `volume` has `min_rows`, `max_rows`, `max_change_pct` and `lookback_runs` (7 by default). `schema` has `on_change` (`ignore`, `warn` or `error`, `warn` by default) and `allow`, a list of `added_column` and `widened_type`. `metrics` is a list of entries that each name a `metric` of the metrics file next to the config, or give a `sql` query and a `name`, with a `grain`, `min`, `max`, `max_change_pct`, an optional `filter` and a `lookback` (7 by default). A malformed value is an error that names its path, `ldp plan` prints the SLA and reports a metric that the folder does not define, and `ldp schema` describes it. `ldp monitor` checks all four sections. Editing it does not change the spec hash |
|
|
332
341
|
|
|
333
342
|
A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
|
|
334
343
|
is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
|
|
@@ -1261,6 +1270,18 @@ make demo-gold # the restaurant's data mart, a faulty batch that the gate st
|
|
|
1261
1270
|
- `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
|
|
1262
1271
|
- The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
|
|
1263
1272
|
|
|
1273
|
+
## Monitoring
|
|
1274
|
+
|
|
1275
|
+
A table can be wrong for days before a person notices. A config says in `metadata.sla` how fresh, how big and how stable its table should be, and which business number must stay above a floor. `ldp monitor` checks the table against it between runs, an alert goes to Slack, a webhook or a file when a monitor changes state and again when it recovers, and `ldp status` shows every pipeline on one screen. Nothing needs a server, and a laptop can run it from cron or launchd.
|
|
1276
|
+
|
|
1277
|
+
```bash
|
|
1278
|
+
make demo-monitor # a late table, a short batch, a new column and a revenue drop raise alerts, and a fresh night resolves them
|
|
1279
|
+
```
|
|
1280
|
+
|
|
1281
|
+
- The [monitoring guide](docs/monitoring.md) walks through it on the restaurant example, command by command, from an SLA to a late table, a short batch, a new column, a day of low revenue, alerts that say each problem once, the status screen, and a cron or launchd job that looks every 15 minutes with a Healthchecks.io heartbeat. It says plainly what a monitor on a laptop cannot do, and every command in it runs in the tests.
|
|
1282
|
+
- `make demo-monitor` runs `examples/restaurant/run.py --layer monitoring`. It loads the orders of one night and runs `ldp monitor --now` 50 hours later, which reports a freshness error with exit status 2 and writes one alert to `.ldp/alerts.jsonl`. A short batch raises a volume error, a batch with a new column raises a schema warning, and a day with a tenth of the revenue raises an error on the `revenue` metric. A fresh night resolves them, a second look sends nothing, and `ldp status` shows the result. With `LDP_SLACK_WEBHOOK` set the same alerts reach Slack, and without it the demo is fully offline. It needs no extra, runs the revenue step when the `dbt` extra is installed, and prints a skip line when it is not. It takes about 15 seconds with dbt and about a second without it.
|
|
1283
|
+
- The commands are in the [platform commands](#platform-commands) table, and the [alerts page](docs/alerts.md) describes the channels and the events.
|
|
1284
|
+
|
|
1264
1285
|
## MCP server for agents
|
|
1265
1286
|
|
|
1266
1287
|
`local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
|
|
@@ -1361,7 +1382,10 @@ Expiry always keeps the current snapshot, branch and tag heads and `main`'s last
|
|
|
1361
1382
|
table's idempotency horizon (7 days by default), because re-runs rely on finding it. Orphans are
|
|
1362
1383
|
files under the table's location that no metadata references and that are older than 72 hours
|
|
1363
1384
|
(`older_than_hours`), so the files of a write still in progress are left alone. `ldp maintain
|
|
1364
|
-
CONFIG` runs both from the command line, as a dry run unless `--apply` is given.
|
|
1385
|
+
CONFIG` runs both from the command line, as a dry run unless `--apply` is given. `ldp maintain
|
|
1386
|
+
CONFIG --system` does the same for the `_ldp` history tables of the config's catalog, after it
|
|
1387
|
+
deletes their rows older than `--keep-days` days (90 by default), a whole UTC day at a time, which
|
|
1388
|
+
drops data files and rewrites none. Run it once a week where `ldp monitor` runs often. There is no
|
|
1365
1389
|
compaction yet.
|
|
1366
1390
|
|
|
1367
1391
|
## Spark (experimental)
|
|
@@ -1441,6 +1465,7 @@ make demo # run the demo into ./ldp_demo
|
|
|
1441
1465
|
make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
|
|
1442
1466
|
make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
|
|
1443
1467
|
make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
|
|
1468
|
+
make demo-monitor # the restaurant's orders watched through a late table, a short batch, a new column and a revenue drop
|
|
1444
1469
|
make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
|
|
1445
1470
|
make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
|
|
1446
1471
|
make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
|
|
@@ -1486,8 +1511,8 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
|
|
|
1486
1511
|
with no rows, leaves the models before it published. A model published with an upsert that gains or loses
|
|
1487
1512
|
a column needs one `ldp run CONFIG --mode overwrite`.
|
|
1488
1513
|
- **The restaurant example is in the repo, not in the package.** `pip install` does not include
|
|
1489
|
-
`examples/`, so the warehousing and
|
|
1490
|
-
`make demo-gold` need a clone.
|
|
1514
|
+
`examples/`, so the warehousing, orchestration and monitoring guides and `make demo-warehouse`, `make demo-orchestration`,
|
|
1515
|
+
`make demo-gold` and `make demo-monitor` need a clone.
|
|
1491
1516
|
- **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
|
|
1492
1517
|
respects DuckDB's single writer. There are no parallel tasks.
|
|
1493
1518
|
- **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
|
|
@@ -1501,16 +1526,22 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
|
|
|
1501
1526
|
than one place, and says to give each place its own folder of configs.
|
|
1502
1527
|
- **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
|
|
1503
1528
|
question, so ask for them in separate queries.
|
|
1529
|
+
- **A monitor looks at one moment, on a machine that is running.** A table that was late and was fixed between two looks
|
|
1530
|
+
never alerts, and nothing alerts while the laptop is asleep or off. A heartbeat to an outside service such as
|
|
1531
|
+
Healthchecks.io is the only cover, and the [monitoring guide](docs/monitoring.md) says how to set it up.
|
|
1532
|
+
- **A change is measured against the median of the periods before it.** It needs 3 earlier values and has no sense of the
|
|
1533
|
+
weekday, so a business that is busier at weekends needs a wide `max_change_pct`, or a `min` and a `max`.
|
|
1534
|
+
- **A local catalog keeps its history on one machine.** Two machines that look at the same table each keep a history of
|
|
1535
|
+
their own, and each one alerts.
|
|
1504
1536
|
|
|
1505
1537
|
## Roadmap
|
|
1506
1538
|
|
|
1507
1539
|
From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
|
|
1508
|
-
of a data platform, and every layer runs on a laptop first. 0.1.
|
|
1540
|
+
of a data platform, and every layer runs on a laptop first. 0.1.5 Monitoring is released, and these
|
|
1509
1541
|
are the milestones after it:
|
|
1510
1542
|
|
|
1511
1543
|
| Version | Milestone | What it adds |
|
|
1512
1544
|
|---|---|---|
|
|
1513
|
-
| 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
|
|
1514
1545
|
| 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
|
|
1515
1546
|
| 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |
|
|
1516
1547
|
| 0.1.8 | LLM | Catalog search and `ldp ask`, text to SQL that corrects itself |
|
|
@@ -10,10 +10,14 @@ It's for learning how a lakehouse works, for building and testing pipelines loca
|
|
|
10
10
|
for cloud infrastructure, and for giving agents and notebooks safe, reproducible access to the
|
|
11
11
|
tables you build.
|
|
12
12
|
|
|
13
|
-
> **Status.** This is version 0.1.
|
|
14
|
-
>
|
|
15
|
-
>
|
|
16
|
-
>
|
|
13
|
+
> **Status.** This is version 0.1.5, the Monitoring milestone. It adds an SLA in the config of a table,
|
|
14
|
+
> `ldp monitor` with freshness, volume, schema and metric monitors, alerts to Slack, a webhook or a file
|
|
15
|
+
> when a monitor changes state and when it recovers, `ldp status` as one screen for every pipeline, the
|
|
16
|
+
> SLA verdict of a table for agents in `ldp mcp`, and `ldp maintain --system` to keep the `_ldp` history
|
|
17
|
+
> small (see [Monitoring](#monitoring)), with a [guide](docs/monitoring.md). Version 0.1.4 added gold
|
|
18
|
+
> tables that carry an owner, a meaning and a description of every column, `ldp gold build` and one
|
|
19
|
+
> release of all the gold tables, `ldp gold list`, `show` and `docs` as a catalog, metrics defined once
|
|
20
|
+
> in `metrics.json`, and Excel, CSV and DuckDB files for people (see [Gold layer](#gold-layer)), with a
|
|
17
21
|
> [guide](docs/gold.md). Version 0.1.3 added windows and idempotency keys to `ldp run`, workflow files
|
|
18
22
|
> with `ldp workflow plan` and `ldp workflow run`, `ldp backfill`, retries, `ldp schedule` for cron and
|
|
19
23
|
> launchd, and an Airflow 3 operator (see [Orchestration](#orchestration)), with a
|
|
@@ -33,7 +37,7 @@ tables you build.
|
|
|
33
37
|
`ldp` needs Python 3.12 or newer. Install it from PyPI, preferably in a virtual environment:
|
|
34
38
|
|
|
35
39
|
```bash
|
|
36
|
-
pip install "local-data-platform[duckdb]>=0.1.
|
|
40
|
+
pip install "local-data-platform[duckdb]>=0.1.5"
|
|
37
41
|
|
|
38
42
|
ldp demo --workdir ldp_demo
|
|
39
43
|
```
|
|
@@ -75,6 +79,7 @@ More demos, each one a `make` target (see [Development](#development)):
|
|
|
75
79
|
| `make demo-warehouse` | A restaurant chain from Excel and CSV exports to dbt-tested Iceberg tables with monthly customer churn, a late batch and time travel (needs the `dbt` and `excel` extras) |
|
|
76
80
|
| `make demo-orchestration` | The restaurant's nightly drops run as a workflow: a backfill that stops at a bad row and resumes after the fix, a third run that skips every day, and a scheduler tick (runs its dbt models task and its gold task, which cuts a release for each night, when the `dbt` extra is installed) |
|
|
77
81
|
| `make demo-gold` | The restaurant data mart: a faulty batch that the gate stops with each failed check named, dbt models whose gold tables have an owner, a meaning and a gate, a release of the gold tables, a metric, an Excel file, a DuckDB file and a catalog page (needs the `dbt` and `excel` extras) |
|
|
82
|
+
| `make demo-monitor` | The restaurant's orders watched through five nights: a late table, a short batch, a new column and a revenue drop raise alerts, a fresh night resolves them, and `ldp status` shows every pipeline on one screen (the revenue step runs when the `dbt` extra is installed) |
|
|
78
83
|
| `make demo-spark` | The Scala Spark job reading the demo table and writing an aggregate that pyiceberg reads back (needs scala-cli) |
|
|
79
84
|
| `make demo-rest` | The demo config against an Iceberg REST catalog started locally on scala-cli's JVM: `ldp catalog test`, two runs, the snapshots and a query (needs scala-cli) |
|
|
80
85
|
| `make demo-all` | All of them; `demo-warehouse` and `demo-gold` are skipped when dbt or the Excel reader isn't installed and the two JVM demos when scala-cli isn't |
|
|
@@ -209,7 +214,7 @@ Every command follows one exit-code table.
|
|
|
209
214
|
|---|---|
|
|
210
215
|
| 0 | success, including a run that published and a run that skipped a duplicate |
|
|
211
216
|
| 1 | an error: a bad config, a usage error, a failed write |
|
|
212
|
-
| 2 |
|
|
217
|
+
| 2 | an SLA breach, which `ldp monitor` reports when a monitor is in error |
|
|
213
218
|
| 3 | a data gate held the run back, such as a quality check that blocked a write, or a gold table whose contract does not hold |
|
|
214
219
|
|
|
215
220
|
A scheduler retries a run that exits with 1, because the cause may pass, and never retries one that exits with 3,
|
|
@@ -239,7 +244,7 @@ call:
|
|
|
239
244
|
| `ldp runs CONFIG [--run RUN_ID] [--events] [--attempts]` | List the runs a config's sinks recorded, the last attempt of a run that was retried, or every attempt with `--attempts` | `events.summarize_runs(events.read_jsonl(path))` or `read_iceberg_events(catalog)` |
|
|
240
245
|
| `ldp commits CONFIG [--key KEY] [--branches]` | List the staged publishes on a table, newest first | `format.iceberg.commit.commit_log(iceberg.table())` and `staged_branches(...)` |
|
|
241
246
|
| `ldp datasets pin\|list\|export` | Pin, list and export dataset versions, as Parquet, JSONL or CSV | `datasets.pin`, `list_versions`, `export` |
|
|
242
|
-
| `ldp maintain CONFIG [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply` | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans` |
|
|
247
|
+
| `ldp maintain CONFIG [--system [--keep-days N]] [--expire DAYS] [--orphans] [--apply]` | Expire snapshots and find orphan files; a dry run unless `--apply`. With `--system` it trims the `_ldp` history tables of the config's catalog instead, deleting the rows older than N days (90 by default) first | `maintenance.expire_snapshots`, `find_orphans`, `remove_orphans`, `delete_old_rows`, `maintain_system_tables` |
|
|
243
248
|
| `ldp mcp --config CONFIG ...` | Serve tables to an agent over MCP (stdio) | `python -m local_data_platform.mcp_server ...`, same options |
|
|
244
249
|
| `ldp dbt init\|build\|test\|docs\|exec PROJECT_DIR` | Run a dbt project over your Iceberg tables; `build` writes the models marked `ldp_publish` back to Iceberg (needs the `dbt` extra) | `dbt_layer.register_sources(...)`, `dbt_layer.publish.plan_publish(...)` and `publish(...)` |
|
|
245
250
|
| `ldp gold sync CONFIG [--dbt-manifest FILE] [--dry-run] [--json]` | Write the contract of a config (its `metadata.docs`) into its existing Iceberg table, with no new data. `--dbt-manifest` reads it from a dbt manifest, for a table that dbt built, and `--dry-run` only says what would change. See [Table contracts](#table-contracts) | `gold.sync.sync_config(path)`, `contracts.apply_contract(table, docs)` |
|
|
@@ -253,6 +258,9 @@ call:
|
|
|
253
258
|
| `ldp metrics list DIR [--json]` | One row for each metric of the `metrics.json` files of a folder, with its description, owner, format, table, dimensions and grains. See [Define metrics](#define-metrics) | `metrics.load_metrics(folder)` |
|
|
254
259
|
| `ldp metrics validate DIR [--json]` | Check every name and expression of the metrics against the schema of the gold tables, list every problem, and exit with 1 if there is one. No row is read, and `ldp gold build` runs the same check | `metrics.validate_folder(folder)` |
|
|
255
260
|
| `ldp metrics query DIR METRICS [--by DIMS] [--grain GRAIN] [--filter DIM=VALUE] [--from DAY] [--to DAY] [--release RN\|--live] [--sql] [--out FILE] [--raw] [--json]` | Compute metrics from the current release of the gold tables, grouped and filtered, and print them in their format. `--sql` prints the SQL and runs nothing, and `--out` writes CSV, Parquet or JSONL | `metrics.query(folder, metrics, by=..., grain=..., filters=[...])` returns a `pyarrow.Table` |
|
|
261
|
+
| `ldp monitor CONFIG [--json] [--now TIME]` | Check the SLA of a config, or of every dataset config of a folder, and print one line for each monitor. The freshness monitor measures the age of a table from the metadata of its last commit, or from the newest value of `freshness.column`, and scans no rows. The volume monitor compares the rows of the latest run with `min_rows` and `max_rows`, and with the median of the runs before it, and the schema monitor compares the columns of the table, and of its source in the latest run, with the last evaluation. A metric monitor computes a metric of the metrics file, or one query, for the latest complete period of its grain, and compares it with `min` and `max` and with the median of the periods before it in the history of the monitors, see the [monitoring guide](docs/monitoring.md). The command writes to no table and creates no catalog. It records the results in `_ldp.monitor_results` when the config has an `iceberg` sink and its catalog exists, and in `.ldp/monitor.jsonl` next to the config otherwise. A config that lists channels in `metadata.observability.alerts` is told when a monitor changes state and when it recovers, see [Alerts](docs/alerts.md). The exit status is 0 when every monitor is `ok` or `warn`, 2 when one is `error` and 1 when `ldp` itself fails | `monitoring.evaluate(config, now=...)` returns a `MonitorResult` for each monitor, and `monitoring.history.record` and `read_results` write and read the history |
|
|
262
|
+
| `ldp status CONFIG [--json] [--html FILE] [--now TIME]` | Print one row for each dataset config of a file or a folder, the worst first, with the last run and how long ago it started, how long ago a run succeeded, the rows of the table, what the freshness, volume, schema and metrics monitors say now, the checks that failed in the last 7 days and the open alerts. It evaluates the monitors in memory and records and sends nothing, reads each `_ldp` table once for the configs that share a catalog, and says how far back the history goes. The exit status is 0 unless `ldp` itself fails. `--json` prints the rows, and `--html FILE` also writes one self-contained page with the table, the monitors and the last 20 runs of each pipeline | `monitoring.status.rows(path, now=None)` returns a `StatusRow` for each config, and `monitoring.ledger.Ledger` reads the history of many configs with one read of each table |
|
|
263
|
+
| `ldp alert test CONFIG` | Send one test message to each channel of `metadata.observability.alerts`, whatever its `on` says, and print which ones got it. It sends real messages. The exit status is 0 when every channel got it and 1 otherwise | `notify.notifier_for(config).test()` returns a `Delivery` for each channel, and `notify.NotifySink` is the event sink that alerts when a run fails or is blocked |
|
|
256
264
|
| `ldp spark CONFIG` | Run the Scala Spark job on a config's table (needs scala-cli) | `engine.spark.ScalaSparkJob` |
|
|
257
265
|
|
|
258
266
|
## Config schema
|
|
@@ -266,12 +274,13 @@ and a `metadata` block:
|
|
|
266
274
|
| `metadata.target` | The same, plus for Iceberg `write_mode` (`append`, `overwrite` or `upsert`), `join_cols` for upserts, and `partition_by` |
|
|
267
275
|
| `metadata.target.catalog` | `type` (`local` by default, `sql`, `rest` or `glue`) and that type's keys; see [Catalogs](#catalogs-and-object-storage) |
|
|
268
276
|
| `metadata.quality` | `on_failure` (`fail` or `warn`) and a list of `checks`: `row_count`, `not_null`, `unique`, `accepted_values`, `range`, `freshness`, `schema`, and the temporal checks `monotonic`, `max_skew`, `rate_below` and `max_gap` |
|
|
269
|
-
| `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}
|
|
277
|
+
| `metadata.observability` | Where run events go, for example `{"sinks": ["iceberg", "jsonl"]}`, and `alerts`, the channels that are told when a run fails or a monitor breaks and recovers, see [Alerts](docs/alerts.md). The default sends events nowhere and alerts to nobody |
|
|
270
278
|
| `metadata.schedule` | When the config runs: a 5-field cron expression, or `@hourly`, `@daily`, `@weekly` or `@monthly`. Without it, a `when` of `hourly`, `daily`, `weekly` or `monthly` means the alias. Editing it does not change the spec hash |
|
|
271
279
|
| `metadata.window` | `"none"`, or `size` (`day`, `hour` or `15m`), an optional `column` that selects the rows of a window, and an optional `timezone` (UTC by default); see [Windows](#windows) |
|
|
272
280
|
| `metadata.retries` | How a workflow tries the run again after a failure: `max_attempts` (1 by default, which is no retry), `delay_s` (30) and `backoff` (2.0); see [Workflow files](#workflow-files). Editing it does not change the spec hash |
|
|
273
281
|
| `metadata.lineage` | The Iceberg tables that a source was made from, as `ICEBERG` blocks, for the run events only. `ldp dbt build` writes it for each model that it publishes, from the manifest. A `DUCKDB` source with `inputs` needs none. Editing it does not change the spec hash |
|
|
274
282
|
| `metadata.docs` | The contract of the table: `layer` (`bronze`, `silver` or `gold`), `description`, `owner` (the config's `who` by default), `grain`, `primary_key`, `columns` (column name to description), `tags` and `refresh` (the config's `when` by default); see [Table contracts](#table-contracts). Editing it does not change the spec hash |
|
|
283
|
+
| `metadata.sla` | How fresh, how big and how stable the table should be, and which business numbers to watch, in up to four sections. `freshness` has `warn_after` and `error_after`, each a `count` of `minute`, `hour` or `day` periods such as `{"count": 26, "period": "hour"}`, and an optional `column`. `volume` has `min_rows`, `max_rows`, `max_change_pct` and `lookback_runs` (7 by default). `schema` has `on_change` (`ignore`, `warn` or `error`, `warn` by default) and `allow`, a list of `added_column` and `widened_type`. `metrics` is a list of entries that each name a `metric` of the metrics file next to the config, or give a `sql` query and a `name`, with a `grain`, `min`, `max`, `max_change_pct`, an optional `filter` and a `lookback` (7 by default). A malformed value is an error that names its path, `ldp plan` prints the SLA and reports a metric that the folder does not define, and `ldp schema` describes it. `ldp monitor` checks all four sections. Editing it does not change the spec hash |
|
|
275
284
|
|
|
276
285
|
A config may also say `"apiVersion": "ldp/v1"` and `"kind": "Pipeline"`. `kind` says what a JSON file is. `Pipeline`
|
|
277
286
|
is the default, so every config here is one, and a config has the same spec hash with and without it. `Workflow` is
|
|
@@ -1204,6 +1213,18 @@ make demo-gold # the restaurant's data mart, a faulty batch that the gate st
|
|
|
1204
1213
|
- `make demo-gold` runs `examples/restaurant/run.py --layer gold`. It loads the raw tables, stops a faulty week of orders at the gate with each failed check named, loads the week without the faults, builds the gold tables with dbt, cuts a release with `ldp gold build`, asks for a metric with `ldp metrics query`, and writes an Excel file, a DuckDB file and a catalog page. It needs the `dbt` and `excel` extras, and takes about 10 seconds.
|
|
1205
1214
|
- The commands are described above, under [Table contracts](#table-contracts), [Build the gold tables](#build-the-gold-tables), [Release the gold tables](#release-the-gold-tables), [Browse the gold tables](#browse-the-gold-tables), [Define metrics](#define-metrics) and [Open the gold tables in a spreadsheet or a BI tool](#open-the-gold-tables-in-a-spreadsheet-or-a-bi-tool).
|
|
1206
1215
|
|
|
1216
|
+
## Monitoring
|
|
1217
|
+
|
|
1218
|
+
A table can be wrong for days before a person notices. A config says in `metadata.sla` how fresh, how big and how stable its table should be, and which business number must stay above a floor. `ldp monitor` checks the table against it between runs, an alert goes to Slack, a webhook or a file when a monitor changes state and again when it recovers, and `ldp status` shows every pipeline on one screen. Nothing needs a server, and a laptop can run it from cron or launchd.
|
|
1219
|
+
|
|
1220
|
+
```bash
|
|
1221
|
+
make demo-monitor # a late table, a short batch, a new column and a revenue drop raise alerts, and a fresh night resolves them
|
|
1222
|
+
```
|
|
1223
|
+
|
|
1224
|
+
- The [monitoring guide](docs/monitoring.md) walks through it on the restaurant example, command by command, from an SLA to a late table, a short batch, a new column, a day of low revenue, alerts that say each problem once, the status screen, and a cron or launchd job that looks every 15 minutes with a Healthchecks.io heartbeat. It says plainly what a monitor on a laptop cannot do, and every command in it runs in the tests.
|
|
1225
|
+
- `make demo-monitor` runs `examples/restaurant/run.py --layer monitoring`. It loads the orders of one night and runs `ldp monitor --now` 50 hours later, which reports a freshness error with exit status 2 and writes one alert to `.ldp/alerts.jsonl`. A short batch raises a volume error, a batch with a new column raises a schema warning, and a day with a tenth of the revenue raises an error on the `revenue` metric. A fresh night resolves them, a second look sends nothing, and `ldp status` shows the result. With `LDP_SLACK_WEBHOOK` set the same alerts reach Slack, and without it the demo is fully offline. It needs no extra, runs the revenue step when the `dbt` extra is installed, and prints a skip line when it is not. It takes about 15 seconds with dbt and about a second without it.
|
|
1226
|
+
- The commands are in the [platform commands](#platform-commands) table, and the [alerts page](docs/alerts.md) describes the channels and the events.
|
|
1227
|
+
|
|
1207
1228
|
## MCP server for agents
|
|
1208
1229
|
|
|
1209
1230
|
`local_data_platform.mcp_server` serves your tables to an AI agent over the Model Context Protocol,
|
|
@@ -1304,7 +1325,10 @@ Expiry always keeps the current snapshot, branch and tag heads and `main`'s last
|
|
|
1304
1325
|
table's idempotency horizon (7 days by default), because re-runs rely on finding it. Orphans are
|
|
1305
1326
|
files under the table's location that no metadata references and that are older than 72 hours
|
|
1306
1327
|
(`older_than_hours`), so the files of a write still in progress are left alone. `ldp maintain
|
|
1307
|
-
CONFIG` runs both from the command line, as a dry run unless `--apply` is given.
|
|
1328
|
+
CONFIG` runs both from the command line, as a dry run unless `--apply` is given. `ldp maintain
|
|
1329
|
+
CONFIG --system` does the same for the `_ldp` history tables of the config's catalog, after it
|
|
1330
|
+
deletes their rows older than `--keep-days` days (90 by default), a whole UTC day at a time, which
|
|
1331
|
+
drops data files and rewrites none. Run it once a week where `ldp monitor` runs often. There is no
|
|
1308
1332
|
compaction yet.
|
|
1309
1333
|
|
|
1310
1334
|
## Spark (experimental)
|
|
@@ -1384,6 +1408,7 @@ make demo # run the demo into ./ldp_demo
|
|
|
1384
1408
|
make demo-all # every demo; demo-warehouse and demo-gold need the dbt and excel extras, demo-spark and demo-rest need scala-cli
|
|
1385
1409
|
make demo-orchestration # the restaurant's nightly workflow, a backfill and a scheduler tick
|
|
1386
1410
|
make demo-gold # the restaurant data mart: a blocked batch, gold tables, a release, a metric and files for people
|
|
1411
|
+
make demo-monitor # the restaurant's orders watched through a late table, a short batch, a new column and a revenue drop
|
|
1387
1412
|
make spark-test # opt-in Spark integration tests (LDP_RUN_SPARK=1, scala-cli)
|
|
1388
1413
|
make rest-test # opt-in REST catalog integration tests (LDP_RUN_REST=1, scala-cli)
|
|
1389
1414
|
make dbt-test # the tests that need the dbt extra (pip install -e ".[dev,dbt]" first)
|
|
@@ -1429,8 +1454,8 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
|
|
|
1429
1454
|
with no rows, leaves the models before it published. A model published with an upsert that gains or loses
|
|
1430
1455
|
a column needs one `ldp run CONFIG --mode overwrite`.
|
|
1431
1456
|
- **The restaurant example is in the repo, not in the package.** `pip install` does not include
|
|
1432
|
-
`examples/`, so the warehousing and
|
|
1433
|
-
`make demo-gold` need a clone.
|
|
1457
|
+
`examples/`, so the warehousing, orchestration and monitoring guides and `make demo-warehouse`, `make demo-orchestration`,
|
|
1458
|
+
`make demo-gold` and `make demo-monitor` need a clone.
|
|
1434
1459
|
- **Tasks run one at a time.** A workflow runs its tasks in order and a backfill its windows in order, which also
|
|
1435
1460
|
respects DuckDB's single writer. There are no parallel tasks.
|
|
1436
1461
|
- **The scheduler is a tick, not a daemon.** `ldp schedule tick` has to be called by cron or launchd, and there is no
|
|
@@ -1444,16 +1469,22 @@ in each release, and the [contributing guide](docs/contributing.md) to get invol
|
|
|
1444
1469
|
than one place, and says to give each place its own folder of configs.
|
|
1445
1470
|
- **A metrics query reads one table.** `ldp metrics query` and the `query_metric` tool refuse metrics of two tables in one
|
|
1446
1471
|
question, so ask for them in separate queries.
|
|
1472
|
+
- **A monitor looks at one moment, on a machine that is running.** A table that was late and was fixed between two looks
|
|
1473
|
+
never alerts, and nothing alerts while the laptop is asleep or off. A heartbeat to an outside service such as
|
|
1474
|
+
Healthchecks.io is the only cover, and the [monitoring guide](docs/monitoring.md) says how to set it up.
|
|
1475
|
+
- **A change is measured against the median of the periods before it.** It needs 3 earlier values and has no sense of the
|
|
1476
|
+
weekday, so a business that is busier at weekends needs a wide `max_change_pct`, or a `min` and a `max`.
|
|
1477
|
+
- **A local catalog keeps its history on one machine.** Two machines that look at the same table each keep a history of
|
|
1478
|
+
their own, and each one alerts.
|
|
1447
1479
|
|
|
1448
1480
|
## Roadmap
|
|
1449
1481
|
|
|
1450
1482
|
From 0.1.1 on, the package version equals the milestone number. Each milestone adds the next layer
|
|
1451
|
-
of a data platform, and every layer runs on a laptop first. 0.1.
|
|
1483
|
+
of a data platform, and every layer runs on a laptop first. 0.1.5 Monitoring is released, and these
|
|
1452
1484
|
are the milestones after it:
|
|
1453
1485
|
|
|
1454
1486
|
| Version | Milestone | What it adds |
|
|
1455
1487
|
|---|---|---|
|
|
1456
|
-
| 0.1.5 | Monitoring | SLAs, freshness, volume, schema and metric monitors, alerts and `ldp status` |
|
|
1457
1488
|
| 0.1.6 | Business Intelligence Reporting Dashboarding | Offline HTML reports, Slack delivery, a live dashboard and BI tool recipes |
|
|
1458
1489
|
| 0.1.7 | Data Science Insights | Churn, retention, funnels, sessions, training sets and a notebook |
|
|
1459
1490
|
| 0.1.8 | LLM | Catalog search and `ldp ask`, text to SQL that corrects itself |
|
|
@@ -25,6 +25,9 @@ Commands::
|
|
|
25
25
|
ldp workflow plan|run FILE ... (orchestration.cli)
|
|
26
26
|
ldp backfill FILE --from START [--to END] [--dry-run] [--replace] (orchestration.cli)
|
|
27
27
|
ldp schedule list|tick|install DIR ... (orchestration.schedule_cli)
|
|
28
|
+
ldp monitor CONFIG_OR_FOLDER [--json] [--now ISO-8601] (monitoring.cli)
|
|
29
|
+
ldp status CONFIG_OR_FOLDER [--json] [--html FILE] [--now ISO-8601] (monitoring.cli)
|
|
30
|
+
ldp alert test CONFIG (notify)
|
|
28
31
|
|
|
29
32
|
The commands in the second group are added by the ``add_cli`` function of the module named
|
|
30
33
|
next to them (``docs/design/v0_1_1_platform.md`` C9).
|
|
@@ -33,7 +36,7 @@ Every command follows one exit-code table::
|
|
|
33
36
|
|
|
34
37
|
0 success, including a run that published and a run that skipped a duplicate
|
|
35
38
|
1 an error: a bad config, a usage error, a failed write
|
|
36
|
-
2
|
|
39
|
+
2 an SLA breach, which ``ldp monitor`` reports when a monitor is in error
|
|
37
40
|
3 a data gate held the run back, such as a quality check that blocked a write, or a gold table whose
|
|
38
41
|
contract does not hold
|
|
39
42
|
|
|
@@ -67,7 +70,8 @@ PROG = "ldp"
|
|
|
67
70
|
EXIT_OK = 0
|
|
68
71
|
EXIT_ERROR = 1
|
|
69
72
|
EXIT_SLA = 2
|
|
70
|
-
"""
|
|
73
|
+
"""An SLA breach: ``ldp monitor`` has a monitor in error. Running it again finds the same table, so a person has to look
|
|
74
|
+
at the table."""
|
|
71
75
|
EXIT_GATE = 3
|
|
72
76
|
"""A data gate, such as a quality check, held the run back. A scheduler must not retry it."""
|
|
73
77
|
DEFAULT_WORKDIR = "ldp_demo"
|
|
@@ -91,6 +95,8 @@ MODULE_COMMANDS = (
|
|
|
91
95
|
("local_data_platform.orchestration.schedule_cli", ("schedule",)),
|
|
92
96
|
("local_data_platform.gold.cli", ("gold",)),
|
|
93
97
|
("local_data_platform.metrics.cli", ("metrics",)),
|
|
98
|
+
("local_data_platform.monitoring.cli", ("monitor", "status")),
|
|
99
|
+
("local_data_platform.notify", ("alert",)),
|
|
94
100
|
)
|
|
95
101
|
"""``(module, commands)``: each module's ``add_cli(subparsers, parents=...)`` registers its commands."""
|
|
96
102
|
|
|
@@ -361,7 +367,8 @@ def build_parser(*, module_commands: bool = True) -> argparse.ArgumentParser:
|
|
|
361
367
|
parser = argparse.ArgumentParser(
|
|
362
368
|
prog=PROG, parents=[common],
|
|
363
369
|
description="local-data-platform: a local Iceberg lakehouse driven by JSON dataset configs.",
|
|
364
|
-
epilog="Exit status is 0 on success, 1 on error and 3 when a data gate
|
|
370
|
+
epilog="Exit status is 0 on success, 1 on error, 2 when ldp monitor finds an SLA breach and 3 when a data gate "
|
|
371
|
+
"held a run back.",
|
|
365
372
|
)
|
|
366
373
|
parser.add_argument("--version", action="version", version=f"{PROG} {__version__}")
|
|
367
374
|
commands = parser.add_subparsers(dest="command", metavar="COMMAND")
|
|
@@ -478,8 +485,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
478
485
|
argv: The arguments, without the program name. Defaults to ``sys.argv[1:]``.
|
|
479
486
|
|
|
480
487
|
Returns:
|
|
481
|
-
The exit status: 0 on success, 1 on error (including usage errors),
|
|
482
|
-
the run back. See the exit-code table at the top of this module.
|
|
488
|
+
The exit status: 0 on success, 1 on error (including usage errors), 2 for an SLA breach that ``ldp monitor``
|
|
489
|
+
reports, and 3 when a data gate held the run back. See the exit-code table at the top of this module.
|
|
483
490
|
"""
|
|
484
491
|
argv = list(sys.argv[1:] if argv is None else argv)
|
|
485
492
|
parser = build_parser(module_commands=_needs_module_commands(argv))
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Dataset configuration.
|
|
2
2
|
|
|
3
3
|
A config describes one dataset with the 4W1H questions (who, what, where, when, how)
|
|
4
|
-
plus a ``metadata`` block holding ``source``, ``target`` and optional ``quality`` and ``
|
|
4
|
+
plus a ``metadata`` block holding ``source``, ``target`` and optional ``quality``, ``docs`` and ``sla``
|
|
5
5
|
sections. See ``docs/design/v0_1_1.md`` for the full schema.
|
|
6
6
|
"""
|
|
7
7
|
|
|
@@ -13,6 +13,7 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
from . import contracts
|
|
15
15
|
from .exceptions import ConfigError
|
|
16
|
+
from .monitoring import sla as sla_settings
|
|
16
17
|
from .orchestration import retries, windows
|
|
17
18
|
from .paths import resolve_path
|
|
18
19
|
from .spec import API_VERSION, DEFAULT_KIND, kind_of
|
|
@@ -27,7 +28,9 @@ class Config:
|
|
|
27
28
|
``base_dir`` is the folder relative paths resolve against. ``Config.from_json``
|
|
28
29
|
sets it to the config file's folder; when built by hand it defaults to the cwd.
|
|
29
30
|
``api_version`` is the config's ``apiVersion``, when it has one, and ``kind`` is
|
|
30
|
-
``Pipeline``, the only kind that is a dataset config.
|
|
31
|
+
``Pipeline``, the only kind that is a dataset config. ``source_path`` is the file that ``Config.from_json`` read,
|
|
32
|
+
and ``None`` for a config that was built by hand. It is not part of what a config is, so two configs that differ
|
|
33
|
+
only in it are equal.
|
|
31
34
|
"""
|
|
32
35
|
|
|
33
36
|
identifier: str
|
|
@@ -40,6 +43,7 @@ class Config:
|
|
|
40
43
|
base_dir: Path | None = None
|
|
41
44
|
api_version: str | None = None
|
|
42
45
|
kind: str = DEFAULT_KIND
|
|
46
|
+
source_path: Path | None = field(default=None, compare=False, repr=False)
|
|
43
47
|
|
|
44
48
|
@classmethod
|
|
45
49
|
def from_dict(cls, data: dict[str, Any], base_dir: str | os.PathLike | None = None) -> "Config":
|
|
@@ -78,7 +82,9 @@ class Config:
|
|
|
78
82
|
data = json.loads(resolved.read_text())
|
|
79
83
|
except json.JSONDecodeError as exc:
|
|
80
84
|
raise ConfigError(f"config file {resolved} is not valid JSON: {exc}") from exc
|
|
81
|
-
|
|
85
|
+
config = cls.from_dict(data, base_dir=resolved.parent)
|
|
86
|
+
config.source_path = resolved
|
|
87
|
+
return config
|
|
82
88
|
|
|
83
89
|
def validate(self) -> None:
|
|
84
90
|
"""Raise :class:`ConfigError` if the metadata block is incomplete, or its schedule or window is invalid."""
|
|
@@ -103,6 +109,8 @@ class Config:
|
|
|
103
109
|
raise ConfigError(f"metadata.retries{path}: {message}")
|
|
104
110
|
for path, message in contracts.docs_problems(self.metadata.get("docs"), who=self.who):
|
|
105
111
|
raise ConfigError(f"metadata.docs{path}: {message}")
|
|
112
|
+
for path, message in sla_settings.sla_problems(self.metadata.get("sla")):
|
|
113
|
+
raise ConfigError(f"metadata.sla{path}: {message}")
|
|
106
114
|
|
|
107
115
|
@property
|
|
108
116
|
def source(self) -> dict[str, Any]:
|
|
@@ -139,6 +147,15 @@ class Config:
|
|
|
139
147
|
"""
|
|
140
148
|
return contracts.parse_docs(self.metadata.get("docs"), who=self.who, when=self.when)
|
|
141
149
|
|
|
150
|
+
@property
|
|
151
|
+
def sla(self) -> sla_settings.Sla | None:
|
|
152
|
+
"""How fresh, how big and how stable the table should be: its ``metadata.sla`` block, normalised.
|
|
153
|
+
|
|
154
|
+
The defaults are applied, so ``volume.lookback_runs`` is 7 and ``schema.on_change`` is ``warn`` when the block
|
|
155
|
+
sets none, and it is ``None`` when the config has no block. See :mod:`local_data_platform.monitoring.sla`.
|
|
156
|
+
"""
|
|
157
|
+
return sla_settings.parse_sla(self.metadata.get("sla"))
|
|
158
|
+
|
|
142
159
|
@property
|
|
143
160
|
def quality(self) -> dict[str, Any]:
|
|
144
161
|
"""The quality block, normalised to ``{"on_failure": "fail" | "warn", "checks": [...]}``.
|