database-tycoon 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/CHANGELOG.md +51 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/PKG-INFO +1 -1
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/pyproject.toml +14 -18
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/__init__.py +1 -1
- database_tycoon-0.2.3/src/tycoon/observability/__init__.py +85 -0
- database_tycoon-0.2.3/src/tycoon/observability/dbt_capture.py +444 -0
- database_tycoon-0.2.3/src/tycoon/observability/dlt_capture.py +415 -0
- database_tycoon-0.2.3/src/tycoon/observability/store.py +219 -0
- database_tycoon-0.2.3/src/tycoon_city/cli.py +154 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/demo/cli.py +20 -12
- database_tycoon-0.2.2/src/tycoon_city/web_dist/assets/index-CosJ8RPj.js → database_tycoon-0.2.3/src/tycoon_city/web_dist/assets/index-BJC7D2Ta.js +214 -213
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/web_dist/index.html +1 -1
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/webserve.py +40 -14
- database_tycoon-0.2.2/src/tycoon/observability.py +0 -1092
- database_tycoon-0.2.2/src/tycoon_city/sim/town_v5_plan.py +0 -333
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/.gitignore +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/LICENSE +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/README.md +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/THIRD-PARTY.md +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/__main__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/cli.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/city.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/data.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/db.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/docs.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/doctor.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/explore.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/fivetran.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/history.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/init.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/notify.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/observability.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/profiles.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/register.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/run.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/run_all.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/schedule.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/semantics.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/setup.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/sources.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/start.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/status.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/stop.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/sync_cmd.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/commands/transform.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/config.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/constants.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/core/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/core/events.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/core/history.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/core/metadata.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/dbt_profiles.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/catalog.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/ducklake_config.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/fivetran_client.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/fivetran_sync.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/mta_bus_speeds_pipeline.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/mta_pipeline.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/nyc_dot_pipeline.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/runner.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/source_installer.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/source_manager.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/ingestion/sources/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/layers.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/metadata_backends/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/metadata_backends/duckdb_file.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/notify.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/project.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/quack.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/dbt_generator.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/observability_dbt.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/osi_generator.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/rill_generator.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/schemas/osi-v0.1.1.schema.json +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/scaffolding/templates.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/schedule.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/sync.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/README +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/data/input/widgets.csv +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/dbt_project.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/models/marts/fct_widget_summary.sql +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/models/marts/schema.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/models/staging/schema.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/models/staging/stg_widgets.sql +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/dbt_project/profiles.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/csv-import/tycoon.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/github-analytics/template.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/github-analytics/tycoon.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/nyc-transit/README +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/nyc-transit/tycoon.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/weather-station/template.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/templates/weather-station/tycoon.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/utils/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/utils/console.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/utils/duckdb_utils.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/utils/process.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/venv.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon/yaml_merge.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/column_lineage.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/dbt_manifest.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/errors.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/loader.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/models.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/osi.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/retention.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/run_history.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/sql_lineage.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/catalog/tycoon_project.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/demo/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/demo/factory.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/demo/project.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/demo/semantic.yml +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/achievements.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/blocks.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/build.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/city_json.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/cli.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/measured.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/export/run_json.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/pricing.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/__init__.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/build_replay.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/channels.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/city.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/contracts.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/generator.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/layout.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/logistics.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/road_junctions.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/schemas/request_schema.json +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/schemas/shipment_schema.json +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/signals.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/tiles.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_arterials.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_blocks.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_endings.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_frontage.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_hierarchy.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_network.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_plan.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_precincts.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_rows.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_streets.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_texture.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/sim/town_zoning.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/theme_data.py +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/themes/default/spritesheet.png +0 -0
- {database_tycoon-0.2.2 → database_tycoon-0.2.3}/src/tycoon_city/themes/default/theme.toml +0 -0
|
@@ -1,5 +1,56 @@
|
|
|
1
1
|
All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
2
2
|
|
|
3
|
+
## [0.2.3] - 2026-10-24
|
|
4
|
+
|
|
5
|
+
_Headline: **documentation site and a tested city**. The documentation site now publishes to GitHub Pages whenever a release reaches `main`, with internal pages withheld by a check that fails the build rather than leaking them. The city renderer is tested and gated in CI like the CLI, and its vehicles, guests and refresh now work. Also here: `tycoon-city` help that matches every other command, and tycoon's agent skills as three Claude Code plugins._
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **The documentation site is published** ([#348][], PR [#415][]). `mkdocs build --strict` already ran on every pull request, but the built site was discarded: GitHub Pages had never been enabled, `https://database-tycoon.github.io/tycoon-cli/` returned 404, and three places in the repository linked to that dead URL (`site_url` in `mkdocs.yml`, the `tycoon docs` module docstring, which calls it "the public docs", and `docs/reference/dependencies.md`, which calls it "the docs site you're reading right now"). A `docs.yml` workflow builds the site on every push to `main` and deploys it to Pages, so those three references resolve. Since `main` only advances when a release branch merges, the published site tracks the released version. Both the pull-request job and the deploy run `scripts/check_docs_publishable.py` after the build: it fails if a page `draft_docs` or `exclude_docs` withholds reached the build, or if a `nav` entry points at a page that was not built, which `mkdocs build --strict` reports only at INFO and would otherwise ship as a dead link on every page. `CONTRIBUTING.md` gains a Documentation section recording where the line between published and internal pages is drawn, and which of its two rules the docs job can enforce.
|
|
10
|
+
- **The docs site has a releases index and a regrouped navigation** (PRs [#416][], [#417][]). A releases page links the current release and the 0.2 series, the home page's Releases card points at it instead of the 0.1.5 notes, a Recipes card is added, the sidebar groups releases into 0.2.x and 0.1.x sections that fold away, and every page gets its own table of contents.
|
|
11
|
+
- **Agent skills install as Claude Code plugins** ([#307][], PRs [#374][], [#375][]). Three plugins ship from this repository's marketplace, each in its own plugin root under `plugins/`: `tycoon`, five skills for working in a tycoon project (adding data sources, diagnosing projects, reading pipeline state, building models and dashboards, operating pipelines), and `tycoon-contributors` and `tycoon-maintainers`, for working on tycoon itself. Install with `/plugin marketplace add Database-Tycoon/tycoon-cli`, then `/plugin install tycoon@tycoon-skills`. The skills are not part of the Python package. `tests/test_agent_skills.py` checks each plugin's manifest and skill list, frontmatter, install names and `CONTRIBUTING.md` citations, and fails on stale text such as `uvx ruff`.
|
|
12
|
+
- **A MotherDuck roadmap for contributors** ([#429][], PR [#432][]). `docs/proposals/motherduck-support.md` orders the local-to-MotherDuck work into four phases across 0.2.4 to 0.3.1. It is a contributor page and is not published.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- **The city's traffic actually drives** (PR [#250][]). The animation loop ticked the vehicle and guest simulations but never called the layers' `update()`, the only writers of the instance matrices, so both meshes sat at count 0 over a fully simulated street. Listed as a known limitation in 0.2.0 and 0.2.1. The verification hooks counted the simulation arrays and hid it; they now answer from the drawn mesh.
|
|
17
|
+
- **`R` refresh repaints the document-derived chrome** (PR [#250][]). The footer status line, the degradation-notes popover and the legend were painted once at boot, so a refresh showed a fresh "exported N ago" over the previous city's name, counts, notes and legend.
|
|
18
|
+
- **Coverage renders unknown, not zero, when the evidence was never read** (PR [#251][]). The renderer's schema was missing the `achievements` block the exporter has emitted since 2026-08-06, so it was silently stripped and the problems gauges, library panel and library tour recomputed coverage from raw fields, reporting "columns documented 0%" for a catalog with no dbt manifest. All three now honour the unknown state.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
|
|
22
|
+
- **The city renderer is CI-gated** (PRs [#248][], [#249][], [#256][], [#252][]). `tests/tycoon_city`, including the `city.json` contract golden, joins the default `pytest` run and `src/tycoon_city` counts toward coverage (floor re-baselined to 73). A new `web` job runs `tsc --noEmit`, the production build, a check that the shipped `web_dist/` is exactly the build of `web/`, and the Playwright suite. Through 0.2.1 the CLI was gated and the renderer was not.
|
|
23
|
+
- **`tycoon-city` renders its help like every other tycoon command** ([#282][], PR [#287][]). The renderer's own console script is now a Typer app with `serve` and `demo` subcommands, and `tycoon-city PATH` stays legal as shorthand for `serve`, so `tycoon-city demo --help` and `tycoon city --help` no longer look like two different tools. The argparse `python -m tycoon_city.webserve [demo]` path remains for Docker and CI; both surfaces share one implementation.
|
|
24
|
+
- **Internal pages are kept out of the docs build** ([#348][], PR [#421][]). `docs_dir` carries 29 city-renderer pages (handover notes, agent task contracts, design log) and 4 draft proposals alongside the PyPI publishing runbook, an OpenMetadata note and an install script. They were out of `nav`, but MkDocs still built them, served them at guessable URLs and indexed them for search, so hosting the site at any point would have published all of them. `draft_docs` keeps the 33 city and proposal pages out of the build while `tycoon docs serve` still renders them locally, and `exclude_docs` drops the other three in both modes. The build carries no internal entries in its search index. Four links that pointed into the withheld directories now point at the repository instead.
|
|
25
|
+
- **The observability module is a package** ([#380][], PR [#381][]). `src/tycoon/observability.py`, at 1,092 lines, is split into `src/tycoon/observability/` (`store.py`, `dlt_capture.py`, `dbt_capture.py`) with the public names re-exported, so imports and behaviour are unchanged. For contributors.
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
### Removed
|
|
29
|
+
|
|
30
|
+
- **The retired v5 city planner and its tests** ([#281][], PR [#256][]). The ring planner replaced it in 0.2.1, and the city contract doc and design log now describe the ring planner instead (PR [#254][]). For contributors; nothing user-facing used it.
|
|
31
|
+
[#250]: https://github.com/Database-Tycoon/tycoon-cli/pull/250
|
|
32
|
+
[#251]: https://github.com/Database-Tycoon/tycoon-cli/pull/251
|
|
33
|
+
[#252]: https://github.com/Database-Tycoon/tycoon-cli/pull/252
|
|
34
|
+
[#256]: https://github.com/Database-Tycoon/tycoon-cli/pull/256
|
|
35
|
+
[#282]: https://github.com/Database-Tycoon/tycoon-cli/issues/282
|
|
36
|
+
[#348]: https://github.com/Database-Tycoon/tycoon-cli/issues/348
|
|
37
|
+
[#421]: https://github.com/Database-Tycoon/tycoon-cli/pull/421
|
|
38
|
+
[#417]: https://github.com/Database-Tycoon/tycoon-cli/pull/417
|
|
39
|
+
[#416]: https://github.com/Database-Tycoon/tycoon-cli/pull/416
|
|
40
|
+
[#415]: https://github.com/Database-Tycoon/tycoon-cli/pull/415
|
|
41
|
+
[#248]: https://github.com/Database-Tycoon/tycoon-cli/pull/248
|
|
42
|
+
[#249]: https://github.com/Database-Tycoon/tycoon-cli/pull/249
|
|
43
|
+
[#254]: https://github.com/Database-Tycoon/tycoon-cli/pull/254
|
|
44
|
+
[#281]: https://github.com/Database-Tycoon/tycoon-cli/issues/281
|
|
45
|
+
[#287]: https://github.com/Database-Tycoon/tycoon-cli/pull/287
|
|
46
|
+
[#307]: https://github.com/Database-Tycoon/tycoon-cli/issues/307
|
|
47
|
+
[#374]: https://github.com/Database-Tycoon/tycoon-cli/pull/374
|
|
48
|
+
[#375]: https://github.com/Database-Tycoon/tycoon-cli/pull/375
|
|
49
|
+
[#380]: https://github.com/Database-Tycoon/tycoon-cli/issues/380
|
|
50
|
+
[#381]: https://github.com/Database-Tycoon/tycoon-cli/pull/381
|
|
51
|
+
[#429]: https://github.com/Database-Tycoon/tycoon-cli/issues/429
|
|
52
|
+
[#432]: https://github.com/Database-Tycoon/tycoon-cli/pull/432
|
|
53
|
+
|
|
3
54
|
## [0.2.2] - 2026-10-09
|
|
4
55
|
|
|
5
56
|
_Headline: **project-local environments**. Every tycoon project now gets its own uv-managed `.venv`, and a catalog source's code and dependencies live inside the project, so deleting `.venv` and running `uv sync` rebuilds the same environment. `--no-prompt` installs what it registers instead of skipping it, and `tycoon data sources migrate` moves an existing project over. Also here: a filesystem load that finds nothing no longer empties a table that already held rows, and a zero-row run is flagged instead of reading as a healthy sync._
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: database-tycoon
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Tycoon CLI — local-first analytics CLI that adapts to your existing data stack
|
|
5
5
|
Project-URL: Homepage, https://github.com/Database-Tycoon/tycoon-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/Database-Tycoon/tycoon-cli
|
|
@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
|
|
|
8
8
|
|
|
9
9
|
[project]
|
|
10
10
|
name = "database-tycoon"
|
|
11
|
-
version = "0.2.
|
|
11
|
+
version = "0.2.3"
|
|
12
12
|
description = "Tycoon CLI — local-first analytics CLI that adapts to your existing data stack"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = {text = "MIT"}
|
|
@@ -74,9 +74,11 @@ dev = [
|
|
|
74
74
|
tycoon = "tycoon.cli:app"
|
|
75
75
|
# The city addon's own entry points, carried over from pipeline-city. The
|
|
76
76
|
# contract docs' quickstart and web/package.json's demo-data script invoke
|
|
77
|
-
# tycoon-city-export; `tycoon city` wraps
|
|
78
|
-
#
|
|
79
|
-
tycoon-city
|
|
77
|
+
# tycoon-city-export; `tycoon city` wraps the server but the standalone
|
|
78
|
+
# `tycoon-city` stays invocable for renderer development without a project.
|
|
79
|
+
# tycoon-city is a Typer app (tycoon_city.cli) so its --help renders like
|
|
80
|
+
# every other tycoon command; the argparse webserve.main stays for python -m.
|
|
81
|
+
tycoon-city = "tycoon_city.cli:app"
|
|
80
82
|
tycoon-city-export = "tycoon_city.export.cli:main"
|
|
81
83
|
|
|
82
84
|
[tool.hatch.build.targets.sdist]
|
|
@@ -106,13 +108,7 @@ markers = [
|
|
|
106
108
|
"recipe: marker for recipe doctest blocks parsed from README / docs/recipes (`tests/test_recipe_doctests.py`). `mode=offline` blocks run by default; `mode=online` blocks gate on `--run-online`.",
|
|
107
109
|
"subprocess_e2e: subprocess-driven end-to-end tests that shell out to the installed `tycoon` binary (catches PATH / Rich rendering / packaging bugs that in-process tests can't see). Included in the default run.",
|
|
108
110
|
]
|
|
109
|
-
|
|
110
|
-
# pipeline-city repo before that repo is deleted. It is not collected yet: its
|
|
111
|
-
# paths are anchored with `Path(__file__).parents[N]` against the old repo root,
|
|
112
|
-
# so nesting it one level deeper here shifts every one of them. Carrying it
|
|
113
|
-
# un-run beats losing it. Wiring it up is tracked separately; until then CI
|
|
114
|
-
# gates the CLI, not the renderer.
|
|
115
|
-
addopts = "-m 'not e2e' --ignore=tests/tycoon_city"
|
|
111
|
+
addopts = "-m 'not e2e'"
|
|
116
112
|
|
|
117
113
|
[tool.ruff]
|
|
118
114
|
line-length = 120
|
|
@@ -130,16 +126,16 @@ target-version = "py312"
|
|
|
130
126
|
"tests/test_ingestion.py" = ["F821"]
|
|
131
127
|
|
|
132
128
|
[tool.coverage.run]
|
|
133
|
-
source = ["src/tycoon"]
|
|
129
|
+
source = ["src/tycoon", "src/tycoon_city"]
|
|
134
130
|
branch = false
|
|
135
131
|
|
|
136
132
|
[tool.coverage.report]
|
|
137
|
-
# Floor re-baselined
|
|
138
|
-
#
|
|
139
|
-
#
|
|
140
|
-
#
|
|
141
|
-
#
|
|
142
|
-
fail_under =
|
|
133
|
+
# Floor re-baselined for v0.2.0 when tests/tycoon_city (563 tests) and the
|
|
134
|
+
# src/tycoon_city package joined the gated run: combined baseline is ~78%.
|
|
135
|
+
# Floor sits 5 points under baseline for routine-drift headroom (same
|
|
136
|
+
# margin as previous cycles); raise it 1-2 points per release alongside
|
|
137
|
+
# genuine test additions.
|
|
138
|
+
fail_under = 73
|
|
143
139
|
show_missing = true
|
|
144
140
|
skip_covered = false
|
|
145
141
|
exclude_lines = [
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Observability — capture dlt + dbt run history in a dedicated metadata DuckDB.
|
|
2
|
+
|
|
3
|
+
Architecture
|
|
4
|
+
============
|
|
5
|
+
|
|
6
|
+
A separate, disposable DuckDB file at ``.tycoon/metadata.duckdb`` holds
|
|
7
|
+
tycoon's observability state: dlt load history (mirrored from each raw
|
|
8
|
+
database) and dbt invocation history (parsed from
|
|
9
|
+
``target/run_results.json``). This file is decoupled from the
|
|
10
|
+
user-facing raw + warehouse databases so it survives ``tycoon data
|
|
11
|
+
clean`` and can be queried directly::
|
|
12
|
+
|
|
13
|
+
tycoon data query --db .tycoon/metadata.duckdb "SELECT * FROM dbt_runs ORDER BY started_at DESC"
|
|
14
|
+
|
|
15
|
+
Capture points
|
|
16
|
+
--------------
|
|
17
|
+
|
|
18
|
+
* ``capture_dlt``: called from the ingestion runner after each
|
|
19
|
+
successful dlt load. Idempotently mirrors ``<schema>._dlt_loads`` and
|
|
20
|
+
per-table ``_dlt_load_id`` row counts into the metadata DB.
|
|
21
|
+
* ``capture_dbt``: called from the ``tycoon data transform`` command
|
|
22
|
+
after each ``run`` / ``test`` / ``build``. Parses
|
|
23
|
+
``target/run_results.json`` and inserts one row per invocation plus
|
|
24
|
+
one row per node.
|
|
25
|
+
|
|
26
|
+
Display
|
|
27
|
+
-------
|
|
28
|
+
|
|
29
|
+
``export_to_parquet`` re-writes four Parquet files under
|
|
30
|
+
``data/parquet/_tycoon/``. The Rill dashboard YAMLs emitted by
|
|
31
|
+
``rill_generator.refresh_usage_dashboards`` point at those Parquets via
|
|
32
|
+
Rill's ``local_file`` connector.
|
|
33
|
+
|
|
34
|
+
All operations are best-effort: capture failures never propagate to
|
|
35
|
+
the caller. Ingestion and dbt invocations must not fail because of
|
|
36
|
+
observability bookkeeping.
|
|
37
|
+
|
|
38
|
+
Layout
|
|
39
|
+
------
|
|
40
|
+
|
|
41
|
+
* ``store``: the metadata file's path, schema, Parquet export and
|
|
42
|
+
``has_any_observability_data``.
|
|
43
|
+
* ``dlt_capture``: dlt load history and ``trace.pickle`` runs.
|
|
44
|
+
* ``dbt_capture``: dbt ``run_results.json`` and ``manifest.json``.
|
|
45
|
+
|
|
46
|
+
Callers import from this package; the submodules are an internal split.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
from tycoon.observability.dbt_capture import (
|
|
52
|
+
capture_dbt,
|
|
53
|
+
capture_dbt_manifest,
|
|
54
|
+
capture_dbt_manifest_safe,
|
|
55
|
+
capture_dbt_safe,
|
|
56
|
+
)
|
|
57
|
+
from tycoon.observability.dlt_capture import (
|
|
58
|
+
capture_dlt,
|
|
59
|
+
capture_dlt_safe,
|
|
60
|
+
capture_dlt_trace,
|
|
61
|
+
capture_dlt_trace_from_dict,
|
|
62
|
+
capture_dlt_trace_safe,
|
|
63
|
+
)
|
|
64
|
+
from tycoon.observability.store import (
|
|
65
|
+
ensure_schema,
|
|
66
|
+
export_to_parquet,
|
|
67
|
+
has_any_observability_data,
|
|
68
|
+
metadata_db_path,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
__all__ = [
|
|
72
|
+
"capture_dbt",
|
|
73
|
+
"capture_dbt_manifest",
|
|
74
|
+
"capture_dbt_manifest_safe",
|
|
75
|
+
"capture_dbt_safe",
|
|
76
|
+
"capture_dlt",
|
|
77
|
+
"capture_dlt_safe",
|
|
78
|
+
"capture_dlt_trace",
|
|
79
|
+
"capture_dlt_trace_from_dict",
|
|
80
|
+
"capture_dlt_trace_safe",
|
|
81
|
+
"ensure_schema",
|
|
82
|
+
"export_to_parquet",
|
|
83
|
+
"has_any_observability_data",
|
|
84
|
+
"metadata_db_path",
|
|
85
|
+
]
|
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
"""Record dbt invocations and manifest fingerprints in the metadata DuckDB."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import duckdb
|
|
10
|
+
|
|
11
|
+
from tycoon.observability.store import ensure_schema
|
|
12
|
+
|
|
13
|
+
# ---------------------------------------------------------------------------
|
|
14
|
+
# dbt capture
|
|
15
|
+
# ---------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parse_run_results_timestamp(ts: str | None) -> datetime | None:
|
|
19
|
+
"""Parse an ISO-8601 timestamp from dbt's run_results.json."""
|
|
20
|
+
if not ts:
|
|
21
|
+
return None
|
|
22
|
+
try:
|
|
23
|
+
cleaned = ts.replace("Z", "+00:00")
|
|
24
|
+
return datetime.fromisoformat(cleaned)
|
|
25
|
+
except ValueError:
|
|
26
|
+
return None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _resource_type_from_unique_id(unique_id: str) -> str:
|
|
30
|
+
"""dbt unique_id is '<resource_type>.<project>.<name>'. Return resource_type."""
|
|
31
|
+
return unique_id.split(".", 1)[0] if "." in unique_id else ""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _earliest_started_at(results: list[dict]) -> datetime | None:
|
|
35
|
+
"""Find the earliest 'started_at' across every node's timing list."""
|
|
36
|
+
earliest: datetime | None = None
|
|
37
|
+
for res in results:
|
|
38
|
+
for timing in res.get("timing", []) or []:
|
|
39
|
+
ts = _parse_run_results_timestamp(timing.get("started_at"))
|
|
40
|
+
if ts is None:
|
|
41
|
+
continue
|
|
42
|
+
if earliest is None or ts < earliest:
|
|
43
|
+
earliest = ts
|
|
44
|
+
return earliest
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _timing_duration(result: dict, name: str) -> float | None:
|
|
48
|
+
"""Return the duration in seconds of a named timing ('compile' / 'execute')."""
|
|
49
|
+
for t in result.get("timing", []) or []:
|
|
50
|
+
if t.get("name") != name:
|
|
51
|
+
continue
|
|
52
|
+
started = _parse_run_results_timestamp(t.get("started_at"))
|
|
53
|
+
completed = _parse_run_results_timestamp(t.get("completed_at"))
|
|
54
|
+
if started and completed:
|
|
55
|
+
return (completed - started).total_seconds()
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def capture_dbt(
|
|
60
|
+
metadata_db: Path,
|
|
61
|
+
dbt_project_dir: Path,
|
|
62
|
+
command: str | None = None,
|
|
63
|
+
) -> str | None:
|
|
64
|
+
"""Parse run_results.json and insert one invocation + one row per node.
|
|
65
|
+
|
|
66
|
+
Returns the invocation_id on successful capture, or None if
|
|
67
|
+
run_results.json is missing / malformed / duplicate.
|
|
68
|
+
"""
|
|
69
|
+
results_path = dbt_project_dir / "target" / "run_results.json"
|
|
70
|
+
if not results_path.exists():
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
data = json.loads(results_path.read_text())
|
|
75
|
+
except (OSError, json.JSONDecodeError):
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
metadata = data.get("metadata", {}) or {}
|
|
79
|
+
args = data.get("args", {}) or {}
|
|
80
|
+
results = data.get("results", []) or []
|
|
81
|
+
|
|
82
|
+
invocation_id = metadata.get("invocation_id")
|
|
83
|
+
if not invocation_id:
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
dbt_version = metadata.get("dbt_version")
|
|
87
|
+
target_name = args.get("target")
|
|
88
|
+
cmd = command or args.get("which") or args.get("rpc_method") or "unknown"
|
|
89
|
+
success = data.get("success")
|
|
90
|
+
elapsed = data.get("elapsed_time")
|
|
91
|
+
started_at = _earliest_started_at(results) or _parse_run_results_timestamp(metadata.get("generated_at"))
|
|
92
|
+
|
|
93
|
+
models_ok = models_error = tests_passed = tests_failed = 0
|
|
94
|
+
for res in results:
|
|
95
|
+
status = (res.get("status") or "").lower()
|
|
96
|
+
resource_type = _resource_type_from_unique_id(res.get("unique_id", ""))
|
|
97
|
+
if resource_type == "model":
|
|
98
|
+
if status == "success":
|
|
99
|
+
models_ok += 1
|
|
100
|
+
elif status == "error":
|
|
101
|
+
models_error += 1
|
|
102
|
+
elif resource_type == "test":
|
|
103
|
+
if status == "pass":
|
|
104
|
+
tests_passed += 1
|
|
105
|
+
elif status in ("fail", "error", "warn"):
|
|
106
|
+
tests_failed += 1
|
|
107
|
+
|
|
108
|
+
ensure_schema(metadata_db)
|
|
109
|
+
captured_at = datetime.now(tz=UTC)
|
|
110
|
+
|
|
111
|
+
con = duckdb.connect(str(metadata_db))
|
|
112
|
+
try:
|
|
113
|
+
pre = con.execute("SELECT 1 FROM dbt_runs WHERE invocation_id = ?", [invocation_id]).fetchone()
|
|
114
|
+
if pre is not None:
|
|
115
|
+
return None # already captured
|
|
116
|
+
|
|
117
|
+
con.execute(
|
|
118
|
+
"""
|
|
119
|
+
INSERT INTO dbt_runs
|
|
120
|
+
(invocation_id, command, started_at, elapsed_s, success,
|
|
121
|
+
models_ok, models_error, tests_passed, tests_failed,
|
|
122
|
+
dbt_version, target_name, captured_at)
|
|
123
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
124
|
+
""",
|
|
125
|
+
[
|
|
126
|
+
invocation_id,
|
|
127
|
+
cmd,
|
|
128
|
+
started_at,
|
|
129
|
+
elapsed,
|
|
130
|
+
success,
|
|
131
|
+
models_ok,
|
|
132
|
+
models_error,
|
|
133
|
+
tests_passed,
|
|
134
|
+
tests_failed,
|
|
135
|
+
dbt_version,
|
|
136
|
+
target_name,
|
|
137
|
+
captured_at,
|
|
138
|
+
],
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
for res in results:
|
|
142
|
+
unique_id = res.get("unique_id", "")
|
|
143
|
+
adapter = res.get("adapter_response") or {}
|
|
144
|
+
rows_affected = adapter.get("rows_affected")
|
|
145
|
+
con.execute(
|
|
146
|
+
"""
|
|
147
|
+
INSERT INTO dbt_nodes
|
|
148
|
+
(invocation_id, node_name, resource_type, status,
|
|
149
|
+
execution_time_s, rows_affected, compile_time_s, message)
|
|
150
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
151
|
+
ON CONFLICT DO NOTHING
|
|
152
|
+
""",
|
|
153
|
+
[
|
|
154
|
+
invocation_id,
|
|
155
|
+
unique_id,
|
|
156
|
+
_resource_type_from_unique_id(unique_id),
|
|
157
|
+
res.get("status"),
|
|
158
|
+
res.get("execution_time"),
|
|
159
|
+
rows_affected,
|
|
160
|
+
_timing_duration(res, "compile"),
|
|
161
|
+
res.get("message"),
|
|
162
|
+
],
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
return invocation_id
|
|
166
|
+
finally:
|
|
167
|
+
con.close()
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
# dbt manifest capture (manifest.json → fingerprint + schema diff)
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
_FINGERPRINTED_RESOURCE_TYPES = {"model", "seed", "snapshot"}
|
|
175
|
+
|
|
176
|
+
# DuckDB PRIMARY KEY columns must be NOT NULL. Use an empty string as a
|
|
177
|
+
# sentinel for change rows whose natural column_name is N/A (model-level
|
|
178
|
+
# changes like model_added / model_removed / sql_changed).
|
|
179
|
+
_NO_COLUMN_SENTINEL = ""
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _extract_manifest_fingerprint(manifest: dict) -> dict[str, dict]:
|
|
183
|
+
"""Reduce a dbt manifest to the minimum needed for a schema diff.
|
|
184
|
+
|
|
185
|
+
Returns ``{unique_id: {checksum, resource_type, columns: {name: type}}}``.
|
|
186
|
+
Non-model/seed/snapshot nodes are dropped to keep the fingerprint small.
|
|
187
|
+
"""
|
|
188
|
+
nodes = manifest.get("nodes") or {}
|
|
189
|
+
out: dict[str, dict] = {}
|
|
190
|
+
for unique_id, node in nodes.items():
|
|
191
|
+
if not isinstance(node, dict):
|
|
192
|
+
continue
|
|
193
|
+
resource_type = node.get("resource_type")
|
|
194
|
+
if resource_type not in _FINGERPRINTED_RESOURCE_TYPES:
|
|
195
|
+
continue
|
|
196
|
+
checksum_val = ""
|
|
197
|
+
checksum_obj = node.get("checksum") or {}
|
|
198
|
+
if isinstance(checksum_obj, dict):
|
|
199
|
+
checksum_val = checksum_obj.get("checksum") or ""
|
|
200
|
+
columns_raw = node.get("columns") or {}
|
|
201
|
+
columns: dict[str, str] = {}
|
|
202
|
+
if isinstance(columns_raw, dict):
|
|
203
|
+
for col_name, col in columns_raw.items():
|
|
204
|
+
if isinstance(col, dict):
|
|
205
|
+
columns[col_name] = col.get("data_type") or ""
|
|
206
|
+
else:
|
|
207
|
+
columns[col_name] = ""
|
|
208
|
+
out[unique_id] = {
|
|
209
|
+
"resource_type": resource_type,
|
|
210
|
+
"checksum": checksum_val,
|
|
211
|
+
"columns": columns,
|
|
212
|
+
}
|
|
213
|
+
return out
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _diff_fingerprints(
|
|
217
|
+
prev: dict[str, dict] | None,
|
|
218
|
+
curr: dict[str, dict],
|
|
219
|
+
) -> list[dict]:
|
|
220
|
+
"""Return a list of change records between two fingerprints.
|
|
221
|
+
|
|
222
|
+
Each record is a flat dict with keys ``change_type``, ``unique_id``,
|
|
223
|
+
``column_name`` (or ''), ``old_value`` / ``new_value`` (or None).
|
|
224
|
+
Returns an empty list when ``prev`` is None (first capture has nothing
|
|
225
|
+
to diff against).
|
|
226
|
+
"""
|
|
227
|
+
if prev is None:
|
|
228
|
+
return []
|
|
229
|
+
|
|
230
|
+
changes: list[dict] = []
|
|
231
|
+
prev_ids = set(prev)
|
|
232
|
+
curr_ids = set(curr)
|
|
233
|
+
|
|
234
|
+
for uid in sorted(curr_ids - prev_ids):
|
|
235
|
+
changes.append(
|
|
236
|
+
{
|
|
237
|
+
"change_type": "model_added",
|
|
238
|
+
"unique_id": uid,
|
|
239
|
+
"column_name": _NO_COLUMN_SENTINEL,
|
|
240
|
+
"old_value": None,
|
|
241
|
+
"new_value": curr[uid].get("resource_type"),
|
|
242
|
+
}
|
|
243
|
+
)
|
|
244
|
+
for uid in sorted(prev_ids - curr_ids):
|
|
245
|
+
changes.append(
|
|
246
|
+
{
|
|
247
|
+
"change_type": "model_removed",
|
|
248
|
+
"unique_id": uid,
|
|
249
|
+
"column_name": _NO_COLUMN_SENTINEL,
|
|
250
|
+
"old_value": prev[uid].get("resource_type"),
|
|
251
|
+
"new_value": None,
|
|
252
|
+
}
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
for uid in sorted(prev_ids & curr_ids):
|
|
256
|
+
p = prev[uid]
|
|
257
|
+
c = curr[uid]
|
|
258
|
+
p_sum = p.get("checksum") or ""
|
|
259
|
+
c_sum = c.get("checksum") or ""
|
|
260
|
+
if p_sum != c_sum:
|
|
261
|
+
changes.append(
|
|
262
|
+
{
|
|
263
|
+
"change_type": "sql_changed",
|
|
264
|
+
"unique_id": uid,
|
|
265
|
+
"column_name": _NO_COLUMN_SENTINEL,
|
|
266
|
+
"old_value": p_sum[:64] or None,
|
|
267
|
+
"new_value": c_sum[:64] or None,
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
p_cols = p.get("columns") or {}
|
|
272
|
+
c_cols = c.get("columns") or {}
|
|
273
|
+
p_col_set = set(p_cols)
|
|
274
|
+
c_col_set = set(c_cols)
|
|
275
|
+
for col in sorted(c_col_set - p_col_set):
|
|
276
|
+
changes.append(
|
|
277
|
+
{
|
|
278
|
+
"change_type": "column_added",
|
|
279
|
+
"unique_id": uid,
|
|
280
|
+
"column_name": col,
|
|
281
|
+
"old_value": None,
|
|
282
|
+
"new_value": c_cols[col] or None,
|
|
283
|
+
}
|
|
284
|
+
)
|
|
285
|
+
for col in sorted(p_col_set - c_col_set):
|
|
286
|
+
changes.append(
|
|
287
|
+
{
|
|
288
|
+
"change_type": "column_removed",
|
|
289
|
+
"unique_id": uid,
|
|
290
|
+
"column_name": col,
|
|
291
|
+
"old_value": p_cols[col] or None,
|
|
292
|
+
"new_value": None,
|
|
293
|
+
}
|
|
294
|
+
)
|
|
295
|
+
for col in sorted(p_col_set & c_col_set):
|
|
296
|
+
if p_cols[col] != c_cols[col]:
|
|
297
|
+
changes.append(
|
|
298
|
+
{
|
|
299
|
+
"change_type": "column_type_changed",
|
|
300
|
+
"unique_id": uid,
|
|
301
|
+
"column_name": col,
|
|
302
|
+
"old_value": p_cols[col] or None,
|
|
303
|
+
"new_value": c_cols[col] or None,
|
|
304
|
+
}
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
return changes
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _load_previous_fingerprint(
|
|
311
|
+
con: duckdb.DuckDBPyConnection,
|
|
312
|
+
exclude_invocation_id: str,
|
|
313
|
+
) -> tuple[str | None, dict[str, dict] | None]:
|
|
314
|
+
"""Return (prev_invocation_id, fingerprint) of the most recent snapshot."""
|
|
315
|
+
row = con.execute(
|
|
316
|
+
"""
|
|
317
|
+
SELECT invocation_id, fingerprint_json
|
|
318
|
+
FROM dbt_manifest_snapshots
|
|
319
|
+
WHERE invocation_id <> ?
|
|
320
|
+
ORDER BY COALESCE(generated_at, captured_at) DESC
|
|
321
|
+
LIMIT 1
|
|
322
|
+
""",
|
|
323
|
+
[exclude_invocation_id],
|
|
324
|
+
).fetchone()
|
|
325
|
+
if row is None:
|
|
326
|
+
return None, None
|
|
327
|
+
prev_id, fp_json = row
|
|
328
|
+
if not fp_json:
|
|
329
|
+
return prev_id, None
|
|
330
|
+
try:
|
|
331
|
+
return prev_id, json.loads(fp_json)
|
|
332
|
+
except (json.JSONDecodeError, TypeError):
|
|
333
|
+
return prev_id, None
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def capture_dbt_manifest(
|
|
337
|
+
metadata_db: Path,
|
|
338
|
+
dbt_project_dir: Path,
|
|
339
|
+
) -> tuple[str, int] | None:
|
|
340
|
+
"""Snapshot ``target/manifest.json`` and write its diff vs. the previous
|
|
341
|
+
snapshot into ``dbt_schema_changes``.
|
|
342
|
+
|
|
343
|
+
Returns ``(invocation_id, changes_recorded)`` on capture, or None when
|
|
344
|
+
the manifest is missing / malformed / already snapshotted.
|
|
345
|
+
"""
|
|
346
|
+
manifest_path = dbt_project_dir / "target" / "manifest.json"
|
|
347
|
+
if not manifest_path.exists():
|
|
348
|
+
return None
|
|
349
|
+
try:
|
|
350
|
+
manifest = json.loads(manifest_path.read_text())
|
|
351
|
+
except (OSError, json.JSONDecodeError):
|
|
352
|
+
return None
|
|
353
|
+
|
|
354
|
+
metadata = manifest.get("metadata") or {}
|
|
355
|
+
invocation_id = metadata.get("invocation_id")
|
|
356
|
+
if not invocation_id:
|
|
357
|
+
return None
|
|
358
|
+
|
|
359
|
+
fingerprint = _extract_manifest_fingerprint(manifest)
|
|
360
|
+
ensure_schema(metadata_db)
|
|
361
|
+
captured_at = datetime.now(tz=UTC)
|
|
362
|
+
generated_at = _parse_run_results_timestamp(metadata.get("generated_at"))
|
|
363
|
+
dbt_schema_version = metadata.get("dbt_schema_version")
|
|
364
|
+
|
|
365
|
+
con = duckdb.connect(str(metadata_db))
|
|
366
|
+
try:
|
|
367
|
+
pre = con.execute(
|
|
368
|
+
"SELECT 1 FROM dbt_manifest_snapshots WHERE invocation_id = ?",
|
|
369
|
+
[invocation_id],
|
|
370
|
+
).fetchone()
|
|
371
|
+
if pre is not None:
|
|
372
|
+
return None
|
|
373
|
+
|
|
374
|
+
prev_invocation_id, prev_fingerprint = _load_previous_fingerprint(con, exclude_invocation_id=invocation_id)
|
|
375
|
+
changes = _diff_fingerprints(prev_fingerprint, fingerprint)
|
|
376
|
+
|
|
377
|
+
con.execute(
|
|
378
|
+
"""
|
|
379
|
+
INSERT INTO dbt_manifest_snapshots
|
|
380
|
+
(invocation_id, generated_at, dbt_schema_version,
|
|
381
|
+
fingerprint_json, captured_at)
|
|
382
|
+
VALUES (?, ?, ?, ?, ?)
|
|
383
|
+
""",
|
|
384
|
+
[
|
|
385
|
+
invocation_id,
|
|
386
|
+
generated_at,
|
|
387
|
+
dbt_schema_version,
|
|
388
|
+
json.dumps(fingerprint, sort_keys=True),
|
|
389
|
+
captured_at,
|
|
390
|
+
],
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
for change in changes:
|
|
394
|
+
con.execute(
|
|
395
|
+
"""
|
|
396
|
+
INSERT INTO dbt_schema_changes
|
|
397
|
+
(invocation_id, prev_invocation_id, change_type, unique_id,
|
|
398
|
+
column_name, old_value, new_value, captured_at)
|
|
399
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
400
|
+
ON CONFLICT DO NOTHING
|
|
401
|
+
""",
|
|
402
|
+
[
|
|
403
|
+
invocation_id,
|
|
404
|
+
prev_invocation_id,
|
|
405
|
+
change["change_type"],
|
|
406
|
+
change["unique_id"],
|
|
407
|
+
change["column_name"],
|
|
408
|
+
change["old_value"],
|
|
409
|
+
change["new_value"],
|
|
410
|
+
captured_at,
|
|
411
|
+
],
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
return invocation_id, len(changes)
|
|
415
|
+
finally:
|
|
416
|
+
con.close()
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
# ---------------------------------------------------------------------------
|
|
420
|
+
# Best-effort wrappers for call sites that must never raise
|
|
421
|
+
# ---------------------------------------------------------------------------
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def capture_dbt_safe(
|
|
425
|
+
metadata_db: Path,
|
|
426
|
+
dbt_project_dir: Path,
|
|
427
|
+
command: str | None = None,
|
|
428
|
+
) -> None:
|
|
429
|
+
"""Best-effort wrapper around capture_dbt; swallows all exceptions."""
|
|
430
|
+
try:
|
|
431
|
+
capture_dbt(metadata_db, dbt_project_dir, command)
|
|
432
|
+
except Exception:
|
|
433
|
+
pass
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def capture_dbt_manifest_safe(
|
|
437
|
+
metadata_db: Path,
|
|
438
|
+
dbt_project_dir: Path,
|
|
439
|
+
) -> None:
|
|
440
|
+
"""Best-effort wrapper around capture_dbt_manifest; swallows all exceptions."""
|
|
441
|
+
try:
|
|
442
|
+
capture_dbt_manifest(metadata_db, dbt_project_dir)
|
|
443
|
+
except Exception:
|
|
444
|
+
pass
|