database-tycoon 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/CHANGELOG.md +33 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/PKG-INFO +1 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/pyproject.toml +1 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/__init__.py +1 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/db.py +73 -15
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/history.py +4 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/init.py +3 -2
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/register.py +3 -2
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/run_all.py +9 -6
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/status.py +17 -7
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/transform.py +112 -29
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/config.py +15 -10
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/core/events.py +14 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/core/history.py +26 -28
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/dbt_profiles.py +4 -2
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/observability/dbt_capture.py +12 -2
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/project.py +17 -0
- database_tycoon-0.2.4/src/tycoon/utils/sql_guard.py +87 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/dbt_manifest.py +63 -3
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/loader.py +31 -6
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/models.py +5 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/demo/factory.py +2 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/blocks.py +6 -0
- database_tycoon-0.2.3/src/tycoon_city/web_dist/assets/index-BJC7D2Ta.js → database_tycoon-0.2.4/src/tycoon_city/web_dist/assets/index-B0UUzpxe.js +174 -174
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/web_dist/index.html +1 -1
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/.gitignore +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/LICENSE +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/README.md +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/THIRD-PARTY.md +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/__main__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/cli.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/city.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/data.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/docs.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/doctor.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/explore.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/fivetran.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/notify.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/observability.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/profiles.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/run.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/schedule.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/semantics.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/setup.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/sources.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/start.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/stop.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/commands/sync_cmd.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/constants.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/core/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/core/metadata.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/catalog.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/ducklake_config.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/fivetran_client.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/fivetran_sync.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/mta_bus_speeds_pipeline.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/mta_pipeline.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/nyc_dot_pipeline.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/runner.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/source_installer.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/source_manager.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/ingestion/sources/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/layers.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/metadata_backends/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/metadata_backends/duckdb_file.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/notify.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/observability/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/observability/dlt_capture.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/observability/store.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/quack.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/dbt_generator.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/observability_dbt.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/osi_generator.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/rill_generator.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/schemas/osi-v0.1.1.schema.json +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/scaffolding/templates.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/schedule.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/sync.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/README +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/data/input/widgets.csv +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/dbt_project.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/models/marts/fct_widget_summary.sql +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/models/marts/schema.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/models/staging/schema.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/models/staging/stg_widgets.sql +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/dbt_project/profiles.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/csv-import/tycoon.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/github-analytics/template.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/github-analytics/tycoon.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/nyc-transit/README +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/nyc-transit/tycoon.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/weather-station/template.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/templates/weather-station/tycoon.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/utils/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/utils/console.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/utils/duckdb_utils.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/utils/process.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/venv.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon/yaml_merge.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/column_lineage.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/errors.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/osi.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/retention.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/run_history.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/sql_lineage.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/catalog/tycoon_project.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/cli.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/demo/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/demo/cli.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/demo/project.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/demo/semantic.yml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/achievements.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/build.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/city_json.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/cli.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/measured.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/export/run_json.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/pricing.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/__init__.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/build_replay.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/channels.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/city.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/contracts.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/generator.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/layout.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/logistics.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/road_junctions.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/schemas/request_schema.json +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/schemas/shipment_schema.json +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/signals.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/tiles.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_arterials.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_blocks.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_endings.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_frontage.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_hierarchy.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_network.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_plan.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_precincts.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_rows.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_streets.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_texture.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/sim/town_zoning.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/theme_data.py +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/themes/default/spritesheet.png +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/themes/default/theme.toml +0 -0
- {database_tycoon-0.2.3 → database_tycoon-0.2.4}/src/tycoon_city/webserve.py +0 -0
|
@@ -1,5 +1,38 @@
|
|
|
1
1
|
All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
2
2
|
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## [0.2.4] - 2026-11-07
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- **`tycoon data run-all` runs tycoon's own dbt, not whichever is first on `PATH`** ([#439][]). `data transform` already preferred the dbt installed next to the Python running tycoon; `run-all` looked it up on `PATH` instead. A global dbt Fusion on `PATH` rewrote the warehouse file in a storage format the project's dbt-duckdb could not read, and a `uv tool install` with no dbt on `PATH` ingested and then failed with "`dbt` not found on PATH". `run-all` now resolves dbt the same way `data transform` does.
|
|
10
|
+
- **`tycoon data run-all` records its dbt build in run history** ([#326][]). After `run-all`, `.tycoon/metadata.duckdb` had no `dbt_runs` or `dbt_nodes` rows, so a project that only used `run-all` showed no dbt history and the city had no build traffic or test signal for it. Its dbt step now records the build the same way `transform build` does.
|
|
11
|
+
- **`tycoon data status` shows when each dbt layer was last built** ([#390][]). The staging, intermediate and marts panels always said "last build never", even when `tycoon data history` listed successful builds of those models. Status looked models up by bare name, but the run ledger stores each node under its dbt `unique_id` (`model.<project>.<name>`), so nothing ever matched. It now matches on `unique_id`. The ledger also stores build start times as local wall-clock time and status read them as UTC, which would have made a build two hours old in New York show as "6h ago"; status now reads them in local time. Ledgers written by earlier versions need no migration: their builds show up on the next `data status`.
|
|
12
|
+
- **City labels no longer draw on top of each other** ([#385][]). District and civic chips were placed independently, so on a real catalog `main_semantic` and `main_intermediate` merged into one unreadable line and the library and firehouse chips stacked on one spot. A colliding chip now moves up or down by a chip height or two; larger districts keep their spot, and a civic chip that still has no room is hidden until the view changes.
|
|
13
|
+
|
|
14
|
+
- **`tycoon city` names the dbt sources a staging model reads, even when they live outside the warehouse** ([#387][]). When a project's sources point at a separate database (for example `database: raw`, an attached DuckDB file or a DuckLake), none of them is an object in the warehouse the city reads, so lineage stopped at staging and every staging model listed its upstream as "none". The export now lists those sources on each model under `objects[].dbt.external_upstream`, with dbt's freshness verdict when `sources.json` has one; a source read through an ephemeral model is listed on the model that uses it. The city still draws no building for them, because nothing about them can be measured from the warehouse.
|
|
15
|
+
|
|
16
|
+
- **`data history` no longer marks a successful dbt build as failed** ([#341][]). dbt 1.10 and later report exposures and saved queries as `no-op`, and tycoon counted every status outside a fixed list of good ones as an error, so a clean build that touched an exposure showed `✗`. tycoon now counts only dbt's failure statuses (`error`, `fail`, `runtime error`, `partial success`), so a status dbt adds later doesn't turn into a false failure. `dbt_runs.success` in `.tycoon/metadata.duckdb` was NULL on every row, because tycoon read a top-level `success` key that `run_results.json` doesn't have. It now comes from dbt's exit code, or from the node statuses when the exit code isn't known.
|
|
17
|
+
- **Failed dbt runs show up in `data history`** ([#377][]). A run that failed before dbt wrote `run_results.json`, such as an unknown target or a DuckDB file locked by another run, was never recorded, so the history only listed successes. `tycoon data transform` now records it as failed, and `data history show` gives dbt's exit code and a one-line error summary: the first failed node's message, or dbt's own error text when no node ran. dbt's output still prints as it runs; tycoon keeps its last lines for the summary. A run whose selector matched nothing, which dbt treats as a success, is now flagged with `!` and "nothing ran" instead of a green tick.
|
|
18
|
+
- **A `motherduck:` warehouse or raw target is no longer treated as a local file** ([#308][]). DuckDB and dbt-duckdb accept `motherduck:<name>` as well as `md:<name>`, but tycoon only recognised `md:`, so `database.warehouse: motherduck:x` resolved to `<root>/motherduck:x` and failed with "Database not found". One `is_motherduck()` helper in `tycoon.project` now decides, and `tycoon register`, `tycoon init`, dbt profile detection and secret redaction use it. Projects on `md:` or a local file behave as before; only strings that start with `motherduck:` change meaning. The city renderer and `tycoon data sync` follow in a later PR.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
|
|
22
|
+
- **`tycoon data query` refuses writes on a MotherDuck database unless you pass `--allow-writes`** ([#309][]). DuckDB cannot open an `md:` database read-only, so `data query` ran whatever it was given with the permissions of your token: `tycoon data query "DROP TABLE ..."` ran against production. tycoon now classifies each statement with sqlglot and, on an `md:` warehouse or `--raw` database, runs only SELECT, WITH, `FROM tbl`-first queries, `TABLE tbl`, VALUES, set operations, DESCRIBE, SHOW, SUMMARIZE, PIVOT, UNPIVOT and EXPLAIN of those. Anything else exits 1 before connecting, with `Refusing to run DROP against MotherDuck (md:x). On MotherDuck, 'data query' only runs SELECT, WITH, FROM, TABLE, VALUES, DESCRIBE, SHOW, SUMMARIZE, PIVOT, UNPIVOT and EXPLAIN. Pass --allow-writes to run other statements.` Every statement in a multi-statement input is checked, and a statement sqlglot cannot parse, or a query nested too deeply to parse, is refused too. **This changes behaviour for MotherDuck users who wrote through `data query`: add `--allow-writes` to those commands and scripts.** The check reads statement types only. Function calls inside a SELECT are not inspected, so `SELECT nextval('s')`, `read_csv` on a path or URL, or `FROM checkpoint()` still run without the flag. It guards against accidents and is not a permission boundary; scope your token if that matters. Local DuckDB files already open read-only and are unaffected.
|
|
23
|
+
- **`tycoon data schema` estimates row counts on a MotherDuck database** ([#310][]). It ran an exact `count(*)` on every table, which is slow and spends compute on a large account. It now reads `estimated_size` from `duckdb_tables()` and prints each figure as `~1,000 rows (estimate)`. The estimate does not drop after deletes (100,000 rows less half of them still shows `~100,000`), so treat it as an upper bound on a table that has lost rows. A view has no catalog estimate and prints `view (use --exact to count)`; a table whose catalog reports none prints `no estimate (use --exact to count)`. Pass `--exact` to count every table and view as before. Local DuckDB files are unchanged and still show exact counts, with or without `--exact`.
|
|
24
|
+
|
|
25
|
+
[#308]: https://github.com/Database-Tycoon/tycoon-cli/issues/308
|
|
26
|
+
[#309]: https://github.com/Database-Tycoon/tycoon-cli/issues/309
|
|
27
|
+
[#310]: https://github.com/Database-Tycoon/tycoon-cli/issues/310
|
|
28
|
+
[#341]: https://github.com/Database-Tycoon/tycoon-cli/issues/341
|
|
29
|
+
[#377]: https://github.com/Database-Tycoon/tycoon-cli/issues/377
|
|
30
|
+
[#385]: https://github.com/Database-Tycoon/tycoon-cli/issues/385
|
|
31
|
+
[#387]: https://github.com/Database-Tycoon/tycoon-cli/issues/387
|
|
32
|
+
[#390]: https://github.com/Database-Tycoon/tycoon-cli/issues/390
|
|
33
|
+
[#439]: https://github.com/Database-Tycoon/tycoon-cli/issues/439
|
|
34
|
+
[#326]: https://github.com/Database-Tycoon/tycoon-cli/issues/326
|
|
35
|
+
|
|
3
36
|
## [0.2.3] - 2026-10-24
|
|
4
37
|
|
|
5
38
|
_Headline: **documentation site and a tested city**. The documentation site now publishes to GitHub Pages whenever a release reaches `main`, with internal pages withheld by a check that fails the build rather than leaking them. The city renderer is tested and gated in CI like the CLI, and its vehicles, guests and refresh now work. Also here: `tycoon-city` help that matches every other command, and tycoon's agent skills as three Claude Code plugins._
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: database-tycoon
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Tycoon CLI — local-first analytics CLI that adapts to your existing data stack
|
|
5
5
|
Project-URL: Homepage, https://github.com/Database-Tycoon/tycoon-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/Database-Tycoon/tycoon-cli
|
|
@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
|
|
|
8
8
|
|
|
9
9
|
[project]
|
|
10
10
|
name = "database-tycoon"
|
|
11
|
-
version = "0.2.
|
|
11
|
+
version = "0.2.4"
|
|
12
12
|
description = "Tycoon CLI — local-first analytics CLI that adapts to your existing data stack"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = {text = "MIT"}
|
|
@@ -12,6 +12,7 @@ from rich.table import Table
|
|
|
12
12
|
from tycoon.config import config, display_target, redact_secrets
|
|
13
13
|
from tycoon.utils.console import console, error, header, info, status_table, success, warn
|
|
14
14
|
from tycoon.utils.duckdb_utils import db_file_size_mb, get_row_count, get_tables, quote_identifier
|
|
15
|
+
from tycoon.utils.sql_guard import refused_statement
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def _resolve_source_db(source_name: str) -> Path | None:
|
|
@@ -69,8 +70,24 @@ def _has_schema(db_path: Path, schema_name: str) -> bool:
|
|
|
69
70
|
# ---------------------------------------------------------------------------
|
|
70
71
|
|
|
71
72
|
|
|
72
|
-
def schema(
|
|
73
|
-
|
|
73
|
+
def schema(
|
|
74
|
+
exact: Annotated[
|
|
75
|
+
bool,
|
|
76
|
+
typer.Option(
|
|
77
|
+
"--exact",
|
|
78
|
+
help=(
|
|
79
|
+
"Count rows exactly on a MotherDuck database instead of reading the catalog's estimate, "
|
|
80
|
+
"which does not drop after deletes."
|
|
81
|
+
),
|
|
82
|
+
),
|
|
83
|
+
] = False,
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Show database tables, row counts, and file sizes.
|
|
86
|
+
|
|
87
|
+
On a MotherDuck database, row counts are estimates read from the catalog
|
|
88
|
+
unless --exact is passed; the estimate does not drop after deletes. Local
|
|
89
|
+
DuckDB files always show exact counts.
|
|
90
|
+
"""
|
|
74
91
|
header("Database Schema")
|
|
75
92
|
|
|
76
93
|
rows: list[tuple[str, str, str]] = []
|
|
@@ -82,7 +99,7 @@ def schema() -> None:
|
|
|
82
99
|
(local_db, "Warehouse", config.warehouse_target),
|
|
83
100
|
]:
|
|
84
101
|
if db_path is None:
|
|
85
|
-
rows.extend(_motherduck_schema_rows(target, label))
|
|
102
|
+
rows.extend(_motherduck_schema_rows(target, label, exact))
|
|
86
103
|
continue
|
|
87
104
|
size = db_file_size_mb(db_path)
|
|
88
105
|
if size is not None:
|
|
@@ -127,26 +144,41 @@ def schema() -> None:
|
|
|
127
144
|
console.print(status_table(rows, title="Database Schema"))
|
|
128
145
|
|
|
129
146
|
|
|
130
|
-
def _motherduck_schema_rows(target: str, label: str) -> list[tuple[str, str, str]]:
|
|
147
|
+
def _motherduck_schema_rows(target: str, label: str, exact: bool = False) -> list[tuple[str, str, str]]:
|
|
131
148
|
"""Schema rows for a MotherDuck database, read over one connection."""
|
|
132
149
|
try:
|
|
133
150
|
con = duckdb.connect(target)
|
|
134
151
|
try:
|
|
135
152
|
# An md: connection attaches every database on the account, so
|
|
136
|
-
# scope the listing to the one the target names.
|
|
153
|
+
# scope the listing to the one the target names. duckdb_tables()
|
|
154
|
+
# has no row for a view, and an attached catalog may report no
|
|
155
|
+
# estimate for a table, so the estimate can be NULL for either.
|
|
137
156
|
tables = con.execute(
|
|
138
|
-
"SELECT table_schema, table_name
|
|
139
|
-
"
|
|
140
|
-
"
|
|
141
|
-
"
|
|
157
|
+
"SELECT t.table_schema, t.table_name, t.table_type, d.estimated_size "
|
|
158
|
+
"FROM information_schema.tables t "
|
|
159
|
+
"LEFT JOIN duckdb_tables() d ON d.database_name = t.table_catalog "
|
|
160
|
+
"AND d.schema_name = t.table_schema AND d.table_name = t.table_name "
|
|
161
|
+
"WHERE t.table_catalog = current_database() "
|
|
162
|
+
"AND t.table_schema NOT IN ('information_schema', 'pg_catalog') "
|
|
163
|
+
"ORDER BY t.table_schema, t.table_name"
|
|
142
164
|
).fetchall()
|
|
143
165
|
rows = [
|
|
144
166
|
(f"{label} database", "OK", f"MotherDuck {display_target(target)}"),
|
|
145
167
|
(" Tables", "", f"{len(tables)}"),
|
|
146
168
|
]
|
|
147
|
-
for s, table in tables:
|
|
148
|
-
|
|
149
|
-
|
|
169
|
+
for s, table, table_type, estimate in tables:
|
|
170
|
+
if exact:
|
|
171
|
+
count = con.execute(
|
|
172
|
+
f"SELECT count(*) FROM {quote_identifier(s)}.{quote_identifier(table)}"
|
|
173
|
+
).fetchone()
|
|
174
|
+
detail = f"{count[0]:,} rows" if count else "empty"
|
|
175
|
+
elif table_type == "VIEW":
|
|
176
|
+
detail = "view (use --exact to count)"
|
|
177
|
+
elif estimate is None:
|
|
178
|
+
detail = "no estimate (use --exact to count)"
|
|
179
|
+
else:
|
|
180
|
+
detail = f"~{estimate:,} {'row' if estimate == 1 else 'rows'} (estimate)"
|
|
181
|
+
rows.append((f" {s}.{table}", "", detail))
|
|
150
182
|
finally:
|
|
151
183
|
con.close()
|
|
152
184
|
except duckdb.Error as exc:
|
|
@@ -173,11 +205,27 @@ def query(
|
|
|
173
205
|
Path | None,
|
|
174
206
|
typer.Option("--db", help="Path to a DuckDB file to query directly."),
|
|
175
207
|
] = None,
|
|
208
|
+
allow_writes: Annotated[
|
|
209
|
+
bool,
|
|
210
|
+
typer.Option(
|
|
211
|
+
"--allow-writes",
|
|
212
|
+
help=(
|
|
213
|
+
"On MotherDuck, run statements other than read-only queries (SELECT, WITH, FROM, TABLE, VALUES, "
|
|
214
|
+
"DESCRIBE, SHOW, SUMMARIZE, PIVOT, UNPIVOT, EXPLAIN). Only statement types are checked, so a "
|
|
215
|
+
"function call inside a SELECT is not inspected: this guards against accidents and is not a "
|
|
216
|
+
"permission boundary."
|
|
217
|
+
),
|
|
218
|
+
),
|
|
219
|
+
] = False,
|
|
176
220
|
) -> None:
|
|
177
221
|
"""Run a SQL query against the warehouse, raw, or a source database.
|
|
178
222
|
|
|
179
|
-
Local DuckDB files open read-only. A MotherDuck database
|
|
180
|
-
|
|
223
|
+
Local DuckDB files open read-only. A MotherDuck database cannot open
|
|
224
|
+
read-only, so tycoon refuses any statement that is not a read-only query
|
|
225
|
+
unless --allow-writes is passed; with it, the statement runs with the
|
|
226
|
+
permissions of your MotherDuck token. The check reads statement types
|
|
227
|
+
only: function calls inside a SELECT are not inspected, so it is a guard
|
|
228
|
+
against accidents and not a permission boundary.
|
|
181
229
|
"""
|
|
182
230
|
is_warehouse = False
|
|
183
231
|
db_path: Path | None
|
|
@@ -222,13 +270,23 @@ def query(
|
|
|
222
270
|
elif db_path is None:
|
|
223
271
|
label = f"{label} (MotherDuck)"
|
|
224
272
|
|
|
273
|
+
if db_path is None and not via_quack and not allow_writes:
|
|
274
|
+
refused = refused_statement(sql)
|
|
275
|
+
if refused is not None:
|
|
276
|
+
error(
|
|
277
|
+
f"Refusing to run {refused} against MotherDuck ({display_target(target)}). "
|
|
278
|
+
"On MotherDuck, 'data query' only runs SELECT, WITH, FROM, TABLE, VALUES, DESCRIBE, SHOW, "
|
|
279
|
+
"SUMMARIZE, PIVOT, UNPIVOT and EXPLAIN. Pass --allow-writes to run other statements."
|
|
280
|
+
)
|
|
281
|
+
raise typer.Exit(1)
|
|
282
|
+
|
|
225
283
|
try:
|
|
226
284
|
if via_quack:
|
|
227
285
|
con = quack.connect(quack_token)
|
|
228
286
|
elif db_path is None:
|
|
229
287
|
# read_only=True is a local-file flag: an md: URL opens an
|
|
230
288
|
# in-memory local database, which DuckDB refuses to start
|
|
231
|
-
# read-only
|
|
289
|
+
# read-only, so the guard above stands in for it.
|
|
232
290
|
con = duckdb.connect(target)
|
|
233
291
|
else:
|
|
234
292
|
con = duckdb.connect(target, read_only=True)
|
|
@@ -106,6 +106,8 @@ def _render_history_table(runs: list[RunSummary]) -> Table:
|
|
|
106
106
|
detail = _fmt_duration(s.duration_seconds)
|
|
107
107
|
if s.rows_total:
|
|
108
108
|
detail += f" · {s.rows_total} models"
|
|
109
|
+
elif s.zero_rows:
|
|
110
|
+
detail += " · [yellow]nothing ran[/yellow]"
|
|
109
111
|
else:
|
|
110
112
|
ref = f"{s.source_id}/{_short(s.run_id)}"
|
|
111
113
|
if not s.zero_rows:
|
|
@@ -250,6 +252,8 @@ def _show_run(id_prefix: str) -> None:
|
|
|
250
252
|
|
|
251
253
|
if s.status != "success":
|
|
252
254
|
status_str = "[red]failed[/red]"
|
|
255
|
+
elif s.zero_rows and is_dbt:
|
|
256
|
+
status_str = "[yellow]success, but nothing ran[/yellow]"
|
|
253
257
|
elif s.zero_rows and s.rows_total:
|
|
254
258
|
status_str = "[yellow]success, but a table got zero rows[/yellow]"
|
|
255
259
|
elif s.zero_rows:
|
|
@@ -15,6 +15,7 @@ from tycoon.project import (
|
|
|
15
15
|
StackConfig,
|
|
16
16
|
TransformationTool,
|
|
17
17
|
WarehouseType,
|
|
18
|
+
is_motherduck,
|
|
18
19
|
migrate_project,
|
|
19
20
|
)
|
|
20
21
|
from tycoon.scaffolding.templates import (
|
|
@@ -336,7 +337,7 @@ def _normalize_warehouse_for_compare(value: str) -> str:
|
|
|
336
337
|
``md:*`` strings are returned verbatim; filesystem paths are resolved
|
|
337
338
|
against CWD if relative and returned as absolute.
|
|
338
339
|
"""
|
|
339
|
-
if value
|
|
340
|
+
if is_motherduck(value):
|
|
340
341
|
return value
|
|
341
342
|
abs_path = Path(value).expanduser()
|
|
342
343
|
if not abs_path.is_absolute():
|
|
@@ -403,7 +404,7 @@ def _run_wizard(target: Path, project_name: str) -> WizardResult:
|
|
|
403
404
|
aligned = _maybe_align_warehouse(warehouse_path, Path(dbt_path))
|
|
404
405
|
if aligned != warehouse_path:
|
|
405
406
|
warehouse_path = aligned
|
|
406
|
-
if aligned
|
|
407
|
+
if is_motherduck(aligned):
|
|
407
408
|
warehouse = WarehouseType.motherduck
|
|
408
409
|
else:
|
|
409
410
|
warehouse = WarehouseType.duckdb
|
|
@@ -19,6 +19,7 @@ from tycoon.project import (
|
|
|
19
19
|
BITool,
|
|
20
20
|
TransformationTool,
|
|
21
21
|
WarehouseType,
|
|
22
|
+
is_motherduck,
|
|
22
23
|
)
|
|
23
24
|
from tycoon.utils.console import console, error, header, info, success, warn
|
|
24
25
|
|
|
@@ -360,7 +361,7 @@ def register_dbt(
|
|
|
360
361
|
def _align_duckdb_warehouse(raw: dict, stack: dict, dbt_warehouse: str) -> None:
|
|
361
362
|
"""DuckDB / MotherDuck alignment (v0.1.1 + v0.1.2 behavior, unchanged)."""
|
|
362
363
|
current_warehouse = raw.get("database", {}).get("warehouse", "")
|
|
363
|
-
if current_warehouse
|
|
364
|
+
if is_motherduck(current_warehouse):
|
|
364
365
|
current_norm = current_warehouse
|
|
365
366
|
else:
|
|
366
367
|
abs_path = Path(current_warehouse).expanduser() if current_warehouse else Path()
|
|
@@ -380,7 +381,7 @@ def _align_duckdb_warehouse(raw: dict, stack: dict, dbt_warehouse: str) -> None:
|
|
|
380
381
|
if raw["database"].get("raw", "") == current_warehouse:
|
|
381
382
|
raw["database"]["raw"] = dbt_warehouse
|
|
382
383
|
stack["warehouse"] = (
|
|
383
|
-
WarehouseType.motherduck.value if dbt_warehouse
|
|
384
|
+
WarehouseType.motherduck.value if is_motherduck(dbt_warehouse) else WarehouseType.duckdb.value
|
|
384
385
|
)
|
|
385
386
|
|
|
386
387
|
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import shutil
|
|
6
5
|
import time
|
|
7
6
|
from typing import Annotated
|
|
8
7
|
|
|
@@ -121,11 +120,13 @@ def run_all_cmd(
|
|
|
121
120
|
|
|
122
121
|
# 2. Transform
|
|
123
122
|
if not skip_transform:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
123
|
+
from tycoon.commands.transform import _capture_dbt_and_refresh_safe, _dbt_executable
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
dbt = _dbt_executable()
|
|
127
|
+
except typer.Exit:
|
|
128
|
+
_emit("error", "run-all failed: dbt not found", stage="transform")
|
|
129
|
+
raise
|
|
129
130
|
|
|
130
131
|
project_dir = cfg.dbt_project_dir
|
|
131
132
|
if not project_dir.exists():
|
|
@@ -136,7 +137,9 @@ def run_all_cmd(
|
|
|
136
137
|
console.rule("[bold cyan]dbt build")
|
|
137
138
|
cmd = [dbt, "build", "--target", target, "--profiles-dir", str(project_dir)]
|
|
138
139
|
console.print(f"[dim]Running: {' '.join(cmd)}[/dim]")
|
|
140
|
+
started_at = time.time()
|
|
139
141
|
result = subprocess.run(cmd, cwd=project_dir)
|
|
142
|
+
_capture_dbt_and_refresh_safe("build", started_at=started_at)
|
|
140
143
|
if result.returncode != 0:
|
|
141
144
|
error(f"dbt build failed (exit {result.returncode}).")
|
|
142
145
|
_emit("error", "run-all failed during dbt build", stage="transform", exit_code=str(result.returncode))
|
|
@@ -130,24 +130,34 @@ def _freshness_label(last_sync: datetime.datetime | None) -> tuple[str, str]:
|
|
|
130
130
|
# -- Layer-build freshness ------------------------------------------------------
|
|
131
131
|
|
|
132
132
|
|
|
133
|
-
def _query_layer_last_build(metadata_db: Path,
|
|
134
|
-
"""Latest successful build start time across ``
|
|
133
|
+
def _query_layer_last_build(metadata_db: Path, unique_ids: list[str]) -> datetime.datetime | None:
|
|
134
|
+
"""Latest successful build start time across the dbt nodes in ``unique_ids``.
|
|
135
|
+
|
|
136
|
+
``dbt_nodes.node_name`` holds the dbt ``unique_id``
|
|
137
|
+
(``model.<project>.<name>``), not the bare model name.
|
|
138
|
+
|
|
139
|
+
``capture_dbt`` writes a UTC-aware datetime into the naive
|
|
140
|
+
``started_at`` TIMESTAMP column, which DuckDB stores as wall-clock time
|
|
141
|
+
in its session time zone. Casting back to TIMESTAMPTZ in the same
|
|
142
|
+
session time zone recovers the real instant; reading the naive value
|
|
143
|
+
as UTC would skew the age by the local UTC offset.
|
|
144
|
+
"""
|
|
135
145
|
import duckdb
|
|
136
146
|
|
|
137
|
-
if not
|
|
147
|
+
if not unique_ids or not metadata_db.exists():
|
|
138
148
|
return None
|
|
139
149
|
try:
|
|
140
150
|
with duckdb.connect(str(metadata_db), read_only=True) as con:
|
|
141
151
|
row = con.execute(
|
|
142
|
-
"SELECT MAX(r.started_at) "
|
|
152
|
+
"SELECT MAX(r.started_at)::TIMESTAMPTZ "
|
|
143
153
|
"FROM dbt_runs r "
|
|
144
154
|
"JOIN dbt_nodes n ON n.invocation_id = r.invocation_id "
|
|
145
155
|
"WHERE n.status = 'success' AND list_contains(?, n.node_name)",
|
|
146
|
-
[
|
|
156
|
+
[unique_ids],
|
|
147
157
|
).fetchone()
|
|
148
|
-
return row[0] if row and row[0] else None
|
|
149
158
|
except Exception:
|
|
150
159
|
return None
|
|
160
|
+
return row[0] if row else None
|
|
151
161
|
|
|
152
162
|
|
|
153
163
|
# -- Panel renderers -----------------------------------------------------------
|
|
@@ -331,7 +341,7 @@ def _render_layer_panel(
|
|
|
331
341
|
info(empty_hint)
|
|
332
342
|
return
|
|
333
343
|
|
|
334
|
-
last_build = _query_layer_last_build(metadata_db, [m.
|
|
344
|
+
last_build = _query_layer_last_build(metadata_db, [m.identifier.removeprefix("dbt:") for m in listed])
|
|
335
345
|
fresh_label, fresh_style = _freshness_label(last_build)
|
|
336
346
|
|
|
337
347
|
table = Table(show_header=True, header_style="bold cyan")
|
|
@@ -2,9 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import re
|
|
5
6
|
import shutil
|
|
6
7
|
import subprocess
|
|
8
|
+
import sys
|
|
7
9
|
import time
|
|
10
|
+
from collections import deque
|
|
11
|
+
from collections.abc import Iterable
|
|
8
12
|
from pathlib import Path
|
|
9
13
|
|
|
10
14
|
import typer
|
|
@@ -40,8 +44,6 @@ _FULL_REFRESH_FLAG = typer.Option(False, "--full-refresh", help="Drop and recrea
|
|
|
40
44
|
|
|
41
45
|
def _dbt_executable() -> str:
|
|
42
46
|
"""Find the dbt binary, preferring the one co-located with this Python."""
|
|
43
|
-
import sys
|
|
44
|
-
|
|
45
47
|
venv_dbt = Path(sys.executable).parent / "dbt"
|
|
46
48
|
if venv_dbt.exists():
|
|
47
49
|
return str(venv_dbt)
|
|
@@ -52,8 +54,81 @@ def _dbt_executable() -> str:
|
|
|
52
54
|
return dbt
|
|
53
55
|
|
|
54
56
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
+
_OUTPUT_TAIL_LINES = 20
|
|
58
|
+
_ERROR_SUMMARY_CHARS = 300
|
|
59
|
+
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
|
|
60
|
+
_DBT_LOG_TIMESTAMP = re.compile(r"^\d{2}:\d{2}:\d{2}\s+")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _stream_dbt(cmd: list[str], cwd: Path) -> tuple[int, list[str]]:
|
|
64
|
+
"""Run dbt, echoing its stdout as it arrives, and keep the last lines.
|
|
65
|
+
|
|
66
|
+
The kept lines are the only record of an error dbt hits before it writes
|
|
67
|
+
run_results.json, such as an unknown target or a locked DuckDB file.
|
|
68
|
+
"""
|
|
69
|
+
tail: deque[str] = deque(maxlen=_OUTPUT_TAIL_LINES)
|
|
70
|
+
with subprocess.Popen(
|
|
71
|
+
cmd, cwd=cwd, stdout=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", bufsize=1
|
|
72
|
+
) as proc:
|
|
73
|
+
assert proc.stdout is not None
|
|
74
|
+
for line in proc.stdout:
|
|
75
|
+
sys.stdout.write(line)
|
|
76
|
+
sys.stdout.flush()
|
|
77
|
+
tail.append(line)
|
|
78
|
+
return proc.returncode, list(tail)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _clip(text: str) -> str:
|
|
82
|
+
text = " ".join(text.split())
|
|
83
|
+
if len(text) <= _ERROR_SUMMARY_CHARS:
|
|
84
|
+
return text
|
|
85
|
+
return text[: _ERROR_SUMMARY_CHARS - 3] + "..."
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _dbt_error_summary(results: list[dict], output_tail: Iterable[str]) -> str | None:
|
|
89
|
+
"""One line on why a dbt run failed: the first failed node, else dbt's own error text."""
|
|
90
|
+
from tycoon.core.events import DBT_ERROR_STATUSES
|
|
91
|
+
|
|
92
|
+
failed = [r for r in results if (r.get("status") or "").lower() in DBT_ERROR_STATUSES]
|
|
93
|
+
if failed:
|
|
94
|
+
first = failed[0]
|
|
95
|
+
text = f"{first.get('unique_id')}: {first.get('message') or first.get('status')}"
|
|
96
|
+
if len(failed) > 1:
|
|
97
|
+
text += f" (and {len(failed) - 1} more)"
|
|
98
|
+
return _clip(text)
|
|
99
|
+
|
|
100
|
+
lines = [_DBT_LOG_TIMESTAMP.sub("", _ANSI_ESCAPE.sub("", line)).strip() for line in output_tail]
|
|
101
|
+
lines = [line for line in lines if line]
|
|
102
|
+
if not lines:
|
|
103
|
+
return None
|
|
104
|
+
marker = max((i for i, line in enumerate(lines) if "Encountered an error" in line), default=None)
|
|
105
|
+
if marker is not None and marker + 1 < len(lines):
|
|
106
|
+
kind, *detail = lines[marker + 1 :]
|
|
107
|
+
return _clip(f"{kind}: {' '.join(detail)}" if detail else kind)
|
|
108
|
+
return _clip(lines[-1])
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _capture_dbt_and_refresh_safe(
|
|
112
|
+
dbt_cmd: str,
|
|
113
|
+
*,
|
|
114
|
+
started_at: float,
|
|
115
|
+
exit_code: int | None = None,
|
|
116
|
+
target: str | None = None,
|
|
117
|
+
output_tail: Iterable[str] = (),
|
|
118
|
+
) -> None:
|
|
119
|
+
"""Best-effort dbt observability capture + Rill dashboard refresh.
|
|
120
|
+
|
|
121
|
+
``exit_code`` is dbt's own; ``None`` means the caller doesn't know it, and
|
|
122
|
+
the run's outcome is then read from the node statuses alone. A failed run
|
|
123
|
+
is recorded even when dbt stopped before writing run_results.json, with
|
|
124
|
+
``output_tail`` (dbt's last lines of output) as its error summary.
|
|
125
|
+
"""
|
|
126
|
+
run_results_path = config.dbt_project_dir / "target" / "run_results.json"
|
|
127
|
+
try:
|
|
128
|
+
fresh = run_results_path.stat().st_mtime >= started_at
|
|
129
|
+
except Exception:
|
|
130
|
+
fresh = False
|
|
131
|
+
|
|
57
132
|
try:
|
|
58
133
|
from tycoon.observability import (
|
|
59
134
|
capture_dbt_manifest_safe,
|
|
@@ -63,11 +138,11 @@ def _capture_dbt_and_refresh_safe(dbt_cmd: str, *, started_at: float) -> None:
|
|
|
63
138
|
from tycoon.scaffolding.rill_generator import refresh_usage_dashboards
|
|
64
139
|
|
|
65
140
|
meta = metadata_db_path(config.root)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
141
|
+
# A stale run_results.json belongs to an earlier invocation, which may
|
|
142
|
+
# never have gone through tycoon; recording it here would file it as
|
|
143
|
+
# this command's run, with this run's outcome lost.
|
|
144
|
+
if fresh:
|
|
145
|
+
capture_dbt_safe(meta, config.dbt_project_dir, command=dbt_cmd, exit_code=exit_code)
|
|
71
146
|
capture_dbt_manifest_safe(meta, config.dbt_project_dir)
|
|
72
147
|
refresh_usage_dashboards(project_root=config.root, rill_dir=config.rill_dir)
|
|
73
148
|
except Exception:
|
|
@@ -76,36 +151,38 @@ def _capture_dbt_and_refresh_safe(dbt_cmd: str, *, started_at: float) -> None:
|
|
|
76
151
|
try:
|
|
77
152
|
import json
|
|
78
153
|
|
|
79
|
-
from tycoon.core.events import DbtRunCompleted
|
|
154
|
+
from tycoon.core.events import DBT_ERROR_STATUSES, DbtRunCompleted
|
|
80
155
|
from tycoon.metadata_backends.duckdb_file import DuckDBFileBackend
|
|
81
156
|
from tycoon.observability import metadata_db_path
|
|
82
157
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
158
|
+
if fresh:
|
|
159
|
+
with open(run_results_path, encoding="utf-8") as f:
|
|
160
|
+
run_results = json.load(f)
|
|
161
|
+
elapsed = float(run_results.get("elapsed_time") or 0.0)
|
|
162
|
+
args = run_results.get("args") or {}
|
|
163
|
+
target = args.get("target") or target
|
|
164
|
+
results = run_results.get("results") or []
|
|
165
|
+
elif exit_code not in (None, 0):
|
|
166
|
+
elapsed = time.time() - started_at
|
|
167
|
+
results = []
|
|
168
|
+
else:
|
|
87
169
|
return
|
|
88
170
|
|
|
89
|
-
with open(run_results_path, encoding="utf-8") as f:
|
|
90
|
-
run_results = json.load(f)
|
|
91
|
-
|
|
92
|
-
elapsed = float(run_results.get("elapsed_time") or 0.0)
|
|
93
|
-
args = run_results.get("args") or {}
|
|
94
|
-
target = args.get("target", "dev") or "dev"
|
|
95
|
-
results = run_results.get("results") or []
|
|
96
171
|
models_run = len(results)
|
|
97
|
-
models_errored = sum(1 for r in results if r.get("status")
|
|
98
|
-
|
|
172
|
+
models_errored = sum(1 for r in results if (r.get("status") or "").lower() in DBT_ERROR_STATUSES)
|
|
173
|
+
failed = models_errored > 0 or exit_code not in (None, 0)
|
|
99
174
|
|
|
100
175
|
event = DbtRunCompleted(
|
|
101
176
|
source_id="dbt",
|
|
102
177
|
runtime_id="dbt",
|
|
103
178
|
command=dbt_cmd,
|
|
104
|
-
target=target,
|
|
179
|
+
target=target or "dev",
|
|
105
180
|
models_run=models_run,
|
|
106
|
-
models_passed=
|
|
181
|
+
models_passed=models_run - models_errored,
|
|
107
182
|
models_errored=models_errored,
|
|
108
183
|
duration_seconds=elapsed,
|
|
184
|
+
exit_code=exit_code,
|
|
185
|
+
error=_dbt_error_summary(results, output_tail) if failed else None,
|
|
109
186
|
)
|
|
110
187
|
with DuckDBFileBackend(metadata_db_path(config.root)) as b:
|
|
111
188
|
b.append_event(event)
|
|
@@ -210,11 +287,17 @@ def _run_dbt(
|
|
|
210
287
|
console.print(f"[dim]Running: {' '.join(cmd)}[/dim]")
|
|
211
288
|
|
|
212
289
|
started_at = time.time()
|
|
213
|
-
|
|
214
|
-
_capture_dbt_and_refresh_safe(
|
|
215
|
-
|
|
290
|
+
returncode, output_tail = _stream_dbt(cmd, project_dir)
|
|
291
|
+
_capture_dbt_and_refresh_safe(
|
|
292
|
+
dbt_cmd,
|
|
293
|
+
started_at=started_at,
|
|
294
|
+
exit_code=returncode,
|
|
295
|
+
target=resolved_target,
|
|
296
|
+
output_tail=output_tail,
|
|
297
|
+
)
|
|
298
|
+
if returncode == 0 and dbt_cmd in {"run", "build"}:
|
|
216
299
|
_auto_osi_scaffold_safe()
|
|
217
|
-
return
|
|
300
|
+
return returncode
|
|
218
301
|
|
|
219
302
|
|
|
220
303
|
@app.command()
|
|
@@ -9,7 +9,14 @@ from __future__ import annotations
|
|
|
9
9
|
import re
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
|
|
12
|
-
from tycoon.project import
|
|
12
|
+
from tycoon.project import (
|
|
13
|
+
MOTHERDUCK_PREFIX as MOTHERDUCK_PREFIX,
|
|
14
|
+
PROJECT_FILENAME,
|
|
15
|
+
SCHEMA_VERSION,
|
|
16
|
+
TycoonProject,
|
|
17
|
+
is_motherduck,
|
|
18
|
+
load_project,
|
|
19
|
+
)
|
|
13
20
|
from tycoon.utils.console import error as _error_console, warn as _warn_console
|
|
14
21
|
|
|
15
22
|
# v0.1 defaults (used when no tycoon.yml exists)
|
|
@@ -18,10 +25,8 @@ _DEFAULT_LOCAL_DB = "data/warehouse.duckdb"
|
|
|
18
25
|
_DEFAULT_DBT_DIR = "dbt_project"
|
|
19
26
|
_DEFAULT_RILL_DIR = "rill"
|
|
20
27
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
# The query part of an md: string can carry motherduck_token=<secret>.
|
|
24
|
-
_MD_QUERY_RE = re.compile(r"(md:[^\s'\"?]*)\?[^\s'\"]*")
|
|
28
|
+
# The query part of an md:/motherduck: string can carry motherduck_token=<secret>.
|
|
29
|
+
_MD_QUERY_RE = re.compile(r"((?:md|motherduck):[^\s'\"?]*)\?[^\s'\"]*", re.IGNORECASE)
|
|
25
30
|
# The key, any spaces around "=", and an opening quote stay; the value is masked
|
|
26
31
|
# and stops at a quote, so a closing quote survives too (SET motherduck_token = '...').
|
|
27
32
|
_MD_TOKEN_RE = re.compile(r"(motherduck_token\s*=\s*['\"]?)[^\s&'\"]+", re.IGNORECASE)
|
|
@@ -30,10 +35,10 @@ _MD_TOKEN_RE = re.compile(r"(motherduck_token\s*=\s*['\"]?)[^\s&'\"]+", re.IGNOR
|
|
|
30
35
|
def display_target(target: str) -> str:
|
|
31
36
|
"""A connection target that is safe to print.
|
|
32
37
|
|
|
33
|
-
An ``md:`` string loses everything from ``?`` onwards, where a
|
|
38
|
+
An ``md:`` or ``motherduck:`` string loses everything from ``?`` onwards, where a
|
|
34
39
|
``motherduck_token`` can sit; a local path is returned unchanged.
|
|
35
40
|
"""
|
|
36
|
-
if target
|
|
41
|
+
if is_motherduck(target):
|
|
37
42
|
return target.split("?", 1)[0]
|
|
38
43
|
return target
|
|
39
44
|
|
|
@@ -93,7 +98,7 @@ class TycoonConfig:
|
|
|
93
98
|
|
|
94
99
|
@property
|
|
95
100
|
def raw_is_motherduck(self) -> bool:
|
|
96
|
-
return self._raw_setting
|
|
101
|
+
return is_motherduck(self._raw_setting)
|
|
97
102
|
|
|
98
103
|
@property
|
|
99
104
|
def raw_target(self) -> str:
|
|
@@ -128,13 +133,13 @@ class TycoonConfig:
|
|
|
128
133
|
|
|
129
134
|
@property
|
|
130
135
|
def warehouse_is_motherduck(self) -> bool:
|
|
131
|
-
return self._warehouse_setting
|
|
136
|
+
return is_motherduck(self._warehouse_setting)
|
|
132
137
|
|
|
133
138
|
@property
|
|
134
139
|
def warehouse_target(self) -> str:
|
|
135
140
|
"""What ``duckdb.connect`` opens for the warehouse.
|
|
136
141
|
|
|
137
|
-
A MotherDuck ``md:`` connection string passes through verbatim; a
|
|
142
|
+
A MotherDuck ``md:`` or ``motherduck:`` connection string passes through verbatim; a
|
|
138
143
|
local path resolves under the project root.
|
|
139
144
|
"""
|
|
140
145
|
if self.warehouse_is_motherduck:
|