detectkit 0.64.0__tar.gz → 0.66.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectkit-0.64.0/detectkit.egg-info → detectkit-0.66.0}/PKG-INFO +8 -3
- {detectkit-0.64.0 → detectkit-0.66.0}/README.md +3 -2
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/__init__.py +1 -1
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/CLAUDE.section.md +1 -1
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/metrics.md +2 -2
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/overview.md +5 -3
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/project.md +54 -13
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/init.py +6 -3
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/profile.py +107 -26
- detectkit-0.66.0/detectkit/database/bigquery_manager.py +229 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/duckdb_manager.py +61 -18
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/context.py +3 -2
- {detectkit-0.64.0 → detectkit-0.66.0/detectkit.egg-info}/PKG-INFO +8 -3
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit.egg-info/SOURCES.txt +1 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit.egg-info/requires.txt +5 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/pyproject.toml +7 -1
- {detectkit-0.64.0 → detectkit-0.66.0}/LICENSE +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/MANIFEST.in +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/branding.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/discord.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/email.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/factory.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/googlechat.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/mattermost.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/ntfy.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/slack.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/teams.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/telegram.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/channels/webhook.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_decision.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_recovery.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_replay.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/_types.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/_base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/_types.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/autotuner.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/axis_spec.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/config_emitter.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/crossval.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/detector_select.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/distribution.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/grid_search.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/labels.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/result.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/runner.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/scoring.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/seasonality_search.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/settings.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/autotune/window_select.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/_output.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/alerting.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/autotune.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/cli.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/rules/detectors.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/skills/dtk-autotune/SKILL.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/skills/dtk-feedback/SKILL.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/skills/dtk-new-metric/SKILL.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/assets/claude/skills/dtk-tune/SKILL.md +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/autotune.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/clean.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/init_claude.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/mcp.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/osi.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/run.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/test_alert.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/tune.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/ui.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/commands/unlock.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/cli/main.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/metric_config.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/metric_io.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/project_config.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/config/validator.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/core/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/core/interval.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/core/models.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/_sql_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/clickhouse_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_alert_states.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_autotune_runs.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_datapoints.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_detections.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_maintenance.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_metrics.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_schema.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/_tasks.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/internal_tables/manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/mysql_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/postgres_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/snowflake_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/source_manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/database/tables.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/factory.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/seasonality.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/_windowed.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/autoreg.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/iqr.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/mad.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/manual_bounds.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/detectors/statistical/zscore.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/loaders/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/loaders/errors.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/loaders/metric_loader.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/loaders/query_template.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/errors.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/serialize.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/server.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/mcp/tools.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/error_dispatch.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/_base.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/_load_step.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/_types.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/orchestration/task_manager/manager.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/reporting/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/reporting/assets/report.js +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/reporting/builder.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/reporting/html_report.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/errors.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/exporter.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/importer.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/osi_model.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/semantic/query_gen.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/assets/tune.js +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/config_writer.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/html.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/payload.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/tuning/server.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/assets/ui.js +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/html.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/jobs.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/metric_files.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/overview.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/ui/server.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/utils/__init__.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/utils/datetime_utils.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/utils/env_interpolation.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/utils/json_utils.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit/utils/stats.py +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit.egg-info/dependency_links.txt +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit.egg-info/entry_points.txt +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/detectkit.egg-info/top_level.txt +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/requirements.txt +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/setup.cfg +0 -0
- {detectkit-0.64.0 → detectkit-0.66.0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: detectkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.66.0
|
|
4
4
|
Summary: Metric monitoring with automatic anomaly detection
|
|
5
5
|
Author: detectkit team
|
|
6
6
|
License: MIT
|
|
@@ -42,12 +42,15 @@ Provides-Extra: duckdb
|
|
|
42
42
|
Requires-Dist: duckdb>=1.1; extra == "duckdb"
|
|
43
43
|
Provides-Extra: snowflake
|
|
44
44
|
Requires-Dist: snowflake-connector-python>=3.12; extra == "snowflake"
|
|
45
|
+
Provides-Extra: bigquery
|
|
46
|
+
Requires-Dist: google-cloud-bigquery>=3.15; extra == "bigquery"
|
|
45
47
|
Provides-Extra: all-db
|
|
46
48
|
Requires-Dist: clickhouse-driver>=0.2.0; extra == "all-db"
|
|
47
49
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "all-db"
|
|
48
50
|
Requires-Dist: pymysql>=1.0.0; extra == "all-db"
|
|
49
51
|
Requires-Dist: duckdb>=1.1; extra == "all-db"
|
|
50
52
|
Requires-Dist: snowflake-connector-python>=3.12; extra == "all-db"
|
|
53
|
+
Requires-Dist: google-cloud-bigquery>=3.15; extra == "all-db"
|
|
51
54
|
Provides-Extra: prophet
|
|
52
55
|
Requires-Dist: prophet>=1.1.0; extra == "prophet"
|
|
53
56
|
Provides-Extra: timesfm
|
|
@@ -65,6 +68,7 @@ Requires-Dist: psycopg2-binary>=2.9.0; extra == "all"
|
|
|
65
68
|
Requires-Dist: pymysql>=1.0.0; extra == "all"
|
|
66
69
|
Requires-Dist: duckdb>=1.1; extra == "all"
|
|
67
70
|
Requires-Dist: snowflake-connector-python>=3.12; extra == "all"
|
|
71
|
+
Requires-Dist: google-cloud-bigquery>=3.15; extra == "all"
|
|
68
72
|
Requires-Dist: prophet>=1.1.0; extra == "all"
|
|
69
73
|
Requires-Dist: timesfm>=0.1.0; extra == "all"
|
|
70
74
|
Requires-Dist: sqlglot>=20.0; extra == "all"
|
|
@@ -110,7 +114,7 @@ Dynamic: license-file
|
|
|
110
114
|
- **@mentions** — tag users/groups in alerts, each channel formats natively
|
|
111
115
|
- **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
|
|
112
116
|
- **Project-level error alerts** — catch DB outages and pipeline crashes once per run
|
|
113
|
-
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed); Snowflake as
|
|
117
|
+
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed, incl. MotherDuck cloud paths); Snowflake and BigQuery as hybrid-mode sources
|
|
114
118
|
- **Hybrid mode** — read metric SQL from a billed-per-query warehouse while all pipeline state lives in a cheap local database
|
|
115
119
|
- **Idempotent** — resume from interruptions, no duplicate processing
|
|
116
120
|
- **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
|
|
@@ -126,8 +130,9 @@ With database drivers:
|
|
|
126
130
|
|
|
127
131
|
```bash
|
|
128
132
|
pip install detectkit[clickhouse] # ClickHouse
|
|
129
|
-
pip install detectkit[duckdb] # DuckDB - single file, no server
|
|
133
|
+
pip install detectkit[duckdb] # DuckDB - single file, no server (also MotherDuck md: paths)
|
|
130
134
|
pip install detectkit[snowflake] # Snowflake - source-only (hybrid mode)
|
|
135
|
+
pip install detectkit[bigquery] # BigQuery - source-only (hybrid mode)
|
|
131
136
|
pip install detectkit[all-db] # All databases
|
|
132
137
|
```
|
|
133
138
|
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
- **@mentions** — tag users/groups in alerts, each channel formats natively
|
|
25
25
|
- **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
|
|
26
26
|
- **Project-level error alerts** — catch DB outages and pipeline crashes once per run
|
|
27
|
-
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed); Snowflake as
|
|
27
|
+
- **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed, incl. MotherDuck cloud paths); Snowflake and BigQuery as hybrid-mode sources
|
|
28
28
|
- **Hybrid mode** — read metric SQL from a billed-per-query warehouse while all pipeline state lives in a cheap local database
|
|
29
29
|
- **Idempotent** — resume from interruptions, no duplicate processing
|
|
30
30
|
- **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
|
|
@@ -40,8 +40,9 @@ With database drivers:
|
|
|
40
40
|
|
|
41
41
|
```bash
|
|
42
42
|
pip install detectkit[clickhouse] # ClickHouse
|
|
43
|
-
pip install detectkit[duckdb] # DuckDB - single file, no server
|
|
43
|
+
pip install detectkit[duckdb] # DuckDB - single file, no server (also MotherDuck md: paths)
|
|
44
44
|
pip install detectkit[snowflake] # Snowflake - source-only (hybrid mode)
|
|
45
|
+
pip install detectkit[bigquery] # BigQuery - source-only (hybrid mode)
|
|
45
46
|
pip install detectkit[all-db] # All databases
|
|
46
47
|
```
|
|
47
48
|
|
|
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
|
|
|
4
4
|
A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "0.
|
|
7
|
+
__version__ = "0.66.0"
|
|
8
8
|
|
|
9
9
|
from detectkit.core.interval import Interval
|
|
10
10
|
from detectkit.core.models import ColumnDefinition, TableModel
|
|
@@ -14,7 +14,7 @@ conventions.
|
|
|
14
14
|
**Database access for _you_ (recommended, not required).** detectkit itself
|
|
15
15
|
connects to the database directly via its drivers — it **never** needs an MCP to
|
|
16
16
|
run. But you assist far better with **read access to the same database** (e.g. a
|
|
17
|
-
database MCP for the project's ClickHouse / PostgreSQL / MySQL / MariaDB / DuckDB / Snowflake (source-only, hybrid mode)): you can inspect
|
|
17
|
+
database MCP for the project's ClickHouse / PostgreSQL / MySQL / MariaDB / DuckDB (incl. MotherDuck `md:` paths) / Snowflake / BigQuery (the last two source-only, hybrid mode)): you can inspect
|
|
18
18
|
a metric's series, find real incidents to label for `dtk autotune`, sanity-check
|
|
19
19
|
a metric query before running it, and confirm detections — instead of asking the
|
|
20
20
|
user to run every query by hand. Without it, fall back to marking incidents
|
|
@@ -80,8 +80,8 @@ metric's **load** step ever connects to it — detect/alert and every other
|
|
|
80
80
|
command don't — though every `dtk run` fail-fast validates the resolved name
|
|
81
81
|
up front regardless of `--steps` (a typo exits 1 before any pipeline work).
|
|
82
82
|
The `source_profile` may point at a **source-only** warehouse profile such as
|
|
83
|
-
`snowflake` (usable *only* as a source, never for `_dtk_*` state
|
|
84
|
-
`project.md`), which is exactly the case hybrid mode is built for.
|
|
83
|
+
`snowflake` or `bigquery` (usable *only* as a source, never for `_dtk_*` state
|
|
84
|
+
— see `project.md`), which is exactly the case hybrid mode is built for.
|
|
85
85
|
See the [Hybrid Mode guide](https://dtk.pipelab.dev/guides/hybrid-mode/).
|
|
86
86
|
|
|
87
87
|
`ai_context` is **OSI-compatible grounding** you can add to any metric with **no
|
|
@@ -6,9 +6,11 @@ metrics with automatic anomaly detection and multi-channel alerting. It is
|
|
|
6
6
|
them with one command. Core logic is pure numpy (no pandas). **ClickHouse,
|
|
7
7
|
PostgreSQL, MySQL/MariaDB and DuckDB are all fully supported** — only the
|
|
8
8
|
connection and the SQL dialect of your metric queries differ between them
|
|
9
|
-
(DuckDB additionally needs no server at all — just a local file
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
(DuckDB additionally needs no server at all — just a local file, or a
|
|
10
|
+
`md:<database>` path to run fully on **MotherDuck**, DuckDB's cloud);
|
|
11
|
+
**Snowflake** and **BigQuery** are supported as **source-only** backends
|
|
12
|
+
(hybrid mode — they run a metric's load SQL, but `_dtk_*` state must live in one
|
|
13
|
+
of the full backends).
|
|
12
14
|
|
|
13
15
|
## The pipeline: load → detect → alert
|
|
14
16
|
|
|
@@ -58,8 +58,8 @@ profile whose database runs metric SQL, while *every* `_dtk_*` table
|
|
|
58
58
|
else. A per-metric `source_profile` overrides it (`metrics.md`); resolves
|
|
59
59
|
**metric → project → unset**, same precedence as `loading_delay`. Unset on
|
|
60
60
|
both = today's behavior (one profile does everything). The source profile can
|
|
61
|
-
be a full backend **or** a **source-only** type
|
|
62
|
-
here, never as the state profile
|
|
61
|
+
be a full backend **or** a **source-only** type (`snowflake`, `bigquery`) —
|
|
62
|
+
valid only here, never as the state profile. Only the LOAD step's
|
|
63
63
|
metric-SQL query is affected — detect/alert and every other command
|
|
64
64
|
(`dtk autotune`/`tune`/`ui`/`clean`/`unlock`) only ever touch the state
|
|
65
65
|
profile. A source-side failure raises `SourceDatabaseError` (message leads
|
|
@@ -140,14 +140,14 @@ alert_channels:
|
|
|
140
140
|
### Database profiles
|
|
141
141
|
|
|
142
142
|
> ClickHouse, PostgreSQL, MySQL, MariaDB and DuckDB are all supported as
|
|
143
|
-
> **state** backends (they hold the `_dtk_*` tables); **Snowflake**
|
|
144
|
-
> **source-only** — see
|
|
143
|
+
> **state** backends (they hold the `_dtk_*` tables); **Snowflake** and
|
|
144
|
+
> **BigQuery** are **source-only** — see their blocks below.
|
|
145
145
|
> ClickHouse/MySQL/MariaDB use two *databases*; PostgreSQL connects to one
|
|
146
146
|
> `database` and uses two *schemas*; DuckDB is a single *file* (or `:memory:`
|
|
147
147
|
> for tests) holding two *schemas*.
|
|
148
148
|
> `dtk init --db-type {clickhouse,postgres,mysql,mariadb}` scaffolds the right
|
|
149
|
-
> shape — DuckDB and
|
|
150
|
-
> `--db-type` choice).
|
|
149
|
+
> shape — DuckDB, Snowflake and BigQuery profiles are written by hand (not yet
|
|
150
|
+
> a `--db-type` choice).
|
|
151
151
|
|
|
152
152
|
**ClickHouse**:
|
|
153
153
|
```yaml
|
|
@@ -201,23 +201,32 @@ auto-detected at connect (`SELECT VERSION()`), so MariaDB gets its own
|
|
|
201
201
|
`VALUES()`-form upsert instead of MySQL 8.0.19's row-alias form.
|
|
202
202
|
`pip install 'detectkit[mariadb]'`.
|
|
203
203
|
|
|
204
|
-
**DuckDB** (embedded, single file; two schemas like PostgreSQL
|
|
204
|
+
**DuckDB** (embedded, single file; two schemas like PostgreSQL — or
|
|
205
|
+
**MotherDuck**, DuckDB's cloud, via a `md:` path):
|
|
205
206
|
```yaml
|
|
206
207
|
profiles:
|
|
207
208
|
prod:
|
|
208
209
|
type: duckdb
|
|
209
|
-
path: /var/lib/detectkit/warehouse.duckdb # required — file path,
|
|
210
|
+
path: /var/lib/detectkit/warehouse.duckdb # required — file path, ":memory:",
|
|
211
|
+
# # or "md:<database>" for MotherDuck
|
|
210
212
|
internal_schema: detectkit # optional — _dtk_* tables (default: "detectkit")
|
|
211
213
|
data_schema: main # optional — data queries (default: "main")
|
|
212
|
-
read_only: false # optional (default: false)
|
|
214
|
+
read_only: false # optional (default: false; local files only)
|
|
215
|
+
motherduck_token: "{{ env_var('MOTHERDUCK_TOKEN') }}" # optional — only for "md:" paths
|
|
213
216
|
settings: {} # optional — extra duckdb.connect() config options
|
|
214
217
|
```
|
|
215
|
-
> Single-writer: DuckDB allows only **one**
|
|
216
|
-
> at a time — a `dtk ui`/`dtk tune` session left open will
|
|
217
|
-
> separately spawned `dtk run` against the same file. Point a
|
|
218
|
-
> consumer at it with `read_only: true`.
|
|
218
|
+
> Single-writer (**local files only**): a local DuckDB file allows only **one**
|
|
219
|
+
> read-write connection at a time — a `dtk ui`/`dtk tune` session left open will
|
|
220
|
+
> clash with a separately spawned `dtk run` against the same file. Point a
|
|
221
|
+
> read-only consumer at it with `read_only: true`. A `md:<database>` MotherDuck
|
|
222
|
+
> path is a **served** database with no single-writer rule — `dtk ui` and a
|
|
223
|
+
> spawned `dtk run` coexist, and `read_only` doesn't apply.
|
|
219
224
|
> `path: ":memory:"` has no on-disk state, so resume/idempotency breaks across
|
|
220
225
|
> process restarts — use it for tests/scratch only, never a real project.
|
|
226
|
+
> `motherduck_token` (env-interpolated) authenticates `md:` paths; unset, the
|
|
227
|
+
> `motherduck` extension falls back to a `motherduck_token` environment
|
|
228
|
+
> variable (lowercase — the extension's own lookup name). MotherDuck is a full
|
|
229
|
+
> state backend — `_dtk_*` tables can live there — and needs no extra beyond
|
|
221
230
|
> `pip install 'detectkit[duckdb]'`.
|
|
222
231
|
|
|
223
232
|
**Snowflake** (**source-only** — hybrid mode; runs a metric's load SQL, never
|
|
@@ -250,6 +259,38 @@ profiles:
|
|
|
250
259
|
> hybrid mode (load from Snowflake, keep cheap local state) is the point.
|
|
251
260
|
> `pip install 'detectkit[snowflake]'`.
|
|
252
261
|
|
|
262
|
+
**BigQuery** (**source-only** — hybrid mode; runs a metric's load SQL, never
|
|
263
|
+
holds `_dtk_*` state):
|
|
264
|
+
```yaml
|
|
265
|
+
profiles:
|
|
266
|
+
warehouse:
|
|
267
|
+
type: bigquery
|
|
268
|
+
project: my-analytics-project # required — GCP project id billed for the queries
|
|
269
|
+
credentials_json_path: /etc/detectkit/bq-sa.json # optional — service-account JSON key file
|
|
270
|
+
# # (unset -> Application Default Credentials)
|
|
271
|
+
location: EU # optional — job location (e.g. EU / US)
|
|
272
|
+
dataset: analytics # optional — default dataset so unqualified table names resolve
|
|
273
|
+
api_endpoint: null # optional — endpoint override; plain-http (the emulator) -> anonymous
|
|
274
|
+
# # auth when no key file; https endpoints authenticate normally
|
|
275
|
+
settings: # optional — extra QueryJobConfig attributes applied to every query
|
|
276
|
+
maximum_bytes_billed: 1000000000
|
|
277
|
+
```
|
|
278
|
+
> **Source-only**: a `bigquery` profile is valid **only** as a metric's or the
|
|
279
|
+
> project's `source_profile` (hybrid mode) — `dtk run` refuses it as a state
|
|
280
|
+
> profile (`--profile`/`default_profile`), so pair it with a full backend
|
|
281
|
+
> (DuckDB/Postgres/ClickHouse) that holds the `_dtk_*` tables. See `metrics.md`
|
|
282
|
+
> and the [Hybrid Mode guide](https://dtk.pipelab.dev/guides/hybrid-mode/).
|
|
283
|
+
> Auth: `credentials_json_path` (a service-account key file) when set, else
|
|
284
|
+
> **Application Default Credentials** (gcloud ADC / an attached service account
|
|
285
|
+
> / Workload Identity); host/port/user/password are unused. Timestamps: a
|
|
286
|
+
> BigQuery `TIMESTAMP` column comes back tz-aware UTC (handled by the loader),
|
|
287
|
+
> `DATETIME` comes back naive — prefer `TIMESTAMP` (or a cast) for the metric's
|
|
288
|
+
> timestamp column. Billing note: on-demand queries bill a **10 MiB minimum**
|
|
289
|
+
> of bytes processed per query per referenced table, so frequent small
|
|
290
|
+
> monitoring queries are disproportionately expensive — load from BigQuery,
|
|
291
|
+
> keep cheap local state; `settings: {maximum_bytes_billed: ...}` caps what a
|
|
292
|
+
> single query may scan. `pip install 'detectkit[bigquery]'`.
|
|
293
|
+
|
|
253
294
|
### Alert channels
|
|
254
295
|
|
|
255
296
|
Defined once in `profiles.yml`, referenced by name in each metric's
|
|
@@ -127,11 +127,14 @@ _COMMENTED_EXAMPLES = {
|
|
|
127
127
|
"duckdb": """ # Example DuckDB profile (in-process, single-file; no host/port/user/password)
|
|
128
128
|
# duckdb_dev:
|
|
129
129
|
# type: duckdb
|
|
130
|
-
# path: ./detectkit.duckdb # or ':memory:'
|
|
130
|
+
# path: ./detectkit.duckdb # or ':memory:' (transient, tests-only), or
|
|
131
|
+
# # # 'md:<database>' for MotherDuck (cloud; add
|
|
132
|
+
# # # motherduck_token: "{{ env_var('MOTHERDUCK_TOKEN') }}")
|
|
131
133
|
# internal_schema: detectkit
|
|
132
134
|
# data_schema: main
|
|
133
|
-
# # Only ONE read-write connection at a time
|
|
134
|
-
# #
|
|
135
|
+
# # Only ONE read-write connection at a time (local files; md: paths are
|
|
136
|
+
# # served and don't have this limit) — don't point a scheduled `dtk run`
|
|
137
|
+
# # and a long-lived `dtk ui` at the same local file simultaneously.
|
|
135
138
|
""",
|
|
136
139
|
}
|
|
137
140
|
|
|
@@ -29,27 +29,31 @@ class ProfileConfig(BaseModel):
|
|
|
29
29
|
duckdb) can hold detectkit state (the ``_dtk_*`` tables) — and, since
|
|
30
30
|
state managers also expose ``execute_query``, double as hybrid-mode
|
|
31
31
|
sources.
|
|
32
|
-
- **Source-only backends** (``SOURCE_ONLY_TYPES``: snowflake)
|
|
33
|
-
only connect + read query. They are valid **only** as a
|
|
34
|
-
``source_profile``; :meth:`create_manager` refuses them
|
|
35
|
-
profile with a clear error.
|
|
32
|
+
- **Source-only backends** (``SOURCE_ONLY_TYPES``: snowflake, bigquery)
|
|
33
|
+
implement only connect + read query. They are valid **only** as a
|
|
34
|
+
metric/project ``source_profile``; :meth:`create_manager` refuses them
|
|
35
|
+
as a state profile with a clear error.
|
|
36
36
|
|
|
37
37
|
Attributes:
|
|
38
38
|
type: Database type ("clickhouse", "postgres", "mysql", "mariadb",
|
|
39
|
-
"duckdb"; source-only: "snowflake")
|
|
39
|
+
"duckdb"; source-only: "snowflake", "bigquery")
|
|
40
40
|
host: Database host (unused for DuckDB — it is an in-process, file-backed
|
|
41
|
-
database with no network endpoint — and for Snowflake,
|
|
42
|
-
|
|
43
|
-
port: Database port (unused for DuckDB/Snowflake; required
|
|
44
|
-
other backend)
|
|
45
|
-
user: Database user (unused for DuckDB; required explicitly
|
|
46
|
-
|
|
47
|
-
|
|
41
|
+
database with no network endpoint — and for Snowflake/BigQuery,
|
|
42
|
+
which connect by `account` / `project`)
|
|
43
|
+
port: Database port (unused for DuckDB/Snowflake/BigQuery; required
|
|
44
|
+
for every other backend)
|
|
45
|
+
user: Database user (unused for DuckDB/BigQuery; required explicitly
|
|
46
|
+
for Snowflake)
|
|
47
|
+
password: Database password (unused for DuckDB/BigQuery; for Snowflake
|
|
48
|
+
either `password` or `private_key_path` must be set)
|
|
48
49
|
database: Connection-target database (PostgreSQL/MySQL/MariaDB; the
|
|
49
50
|
session default database for Snowflake; unused for DuckDB — use
|
|
50
51
|
`path` instead)
|
|
51
|
-
path: Path to the DuckDB database file,
|
|
52
|
-
in-process database
|
|
52
|
+
path: Path to the DuckDB database file, ":memory:" for a transient
|
|
53
|
+
in-process database, or "md:<database>" for a MotherDuck cloud
|
|
54
|
+
database (DuckDB only)
|
|
55
|
+
motherduck_token: MotherDuck service token for "md:" paths (DuckDB
|
|
56
|
+
only; unset -> the extension reads the motherduck_token env var)
|
|
53
57
|
account: Snowflake account identifier (e.g. "myorg-myaccount")
|
|
54
58
|
warehouse: Snowflake virtual warehouse to run load queries on
|
|
55
59
|
role: Snowflake role for the session
|
|
@@ -57,6 +61,15 @@ class ProfileConfig(BaseModel):
|
|
|
57
61
|
private_key_path: Path to a PEM private key for Snowflake key-pair
|
|
58
62
|
auth (recommended for service accounts)
|
|
59
63
|
private_key_passphrase: Passphrase of the private key, if encrypted
|
|
64
|
+
project: GCP project id billed for the queries (BigQuery only)
|
|
65
|
+
credentials_json_path: Path to a service-account JSON key file
|
|
66
|
+
(BigQuery only; unset -> Application Default Credentials)
|
|
67
|
+
location: Job location, e.g. "EU" (BigQuery only; unset -> inferred
|
|
68
|
+
from the referenced datasets)
|
|
69
|
+
dataset: Default dataset for unqualified table names in load queries
|
|
70
|
+
(BigQuery only)
|
|
71
|
+
api_endpoint: API endpoint override — the BigQuery emulator or a
|
|
72
|
+
private endpoint (BigQuery only)
|
|
60
73
|
internal_database: Database/schema for internal tables
|
|
61
74
|
internal_schema: Schema for internal tables (PostgreSQL/DuckDB only)
|
|
62
75
|
data_database: Database for user data tables
|
|
@@ -76,10 +89,10 @@ class ProfileConfig(BaseModel):
|
|
|
76
89
|
)
|
|
77
90
|
# Source-only backends: connect + execute_query only (no DDL/upserts/
|
|
78
91
|
# locks); valid ONLY as a metric/project `source_profile` (hybrid mode).
|
|
79
|
-
SOURCE_ONLY_TYPES: ClassVar[frozenset[str]] = frozenset({"snowflake"})
|
|
80
|
-
# Backends with no network port to validate (in-process file DB;
|
|
81
|
-
# account-resolved cloud
|
|
82
|
-
_PORTLESS_TYPES: ClassVar[frozenset[str]] = frozenset({"duckdb", "snowflake"})
|
|
92
|
+
SOURCE_ONLY_TYPES: ClassVar[frozenset[str]] = frozenset({"snowflake", "bigquery"})
|
|
93
|
+
# Backends with no network port to validate (in-process file DB;
|
|
94
|
+
# account/project-resolved cloud drivers).
|
|
95
|
+
_PORTLESS_TYPES: ClassVar[frozenset[str]] = frozenset({"duckdb", "snowflake", "bigquery"})
|
|
83
96
|
|
|
84
97
|
type: str = Field(..., description="Database type")
|
|
85
98
|
host: str = Field(default="localhost", description="Database host (unused for DuckDB)")
|
|
@@ -96,20 +109,29 @@ class ProfileConfig(BaseModel):
|
|
|
96
109
|
default=None, description="Database to connect to (PostgreSQL/MySQL/MariaDB)"
|
|
97
110
|
)
|
|
98
111
|
|
|
99
|
-
# DuckDB-only: the database file path (or ":memory:"
|
|
100
|
-
# password/database above are simply
|
|
101
|
-
# rejected, since e.g. `host` always
|
|
112
|
+
# DuckDB-only: the database file path (or ":memory:", or "md:<database>"
|
|
113
|
+
# for MotherDuck). host/port/user/password/database above are simply
|
|
114
|
+
# ignored for this backend rather than rejected, since e.g. `host` always
|
|
115
|
+
# carries its "localhost" default.
|
|
102
116
|
path: str | None = Field(
|
|
103
117
|
default=None,
|
|
104
|
-
description="Database file path,
|
|
118
|
+
description="Database file path, ':memory:' for a transient in-process "
|
|
119
|
+
"database, or 'md:<database>' for a MotherDuck cloud database (DuckDB only)",
|
|
105
120
|
)
|
|
106
121
|
read_only: bool = Field(
|
|
107
122
|
default=False,
|
|
108
123
|
description=(
|
|
109
|
-
"Open the database read-only (DuckDB only
|
|
110
|
-
"
|
|
124
|
+
"Open the database read-only (DuckDB local files only — MotherDuck "
|
|
125
|
+
"has no read-only attach) — lets a reader profile coexist with the "
|
|
126
|
+
"one process holding the file read-write"
|
|
111
127
|
),
|
|
112
128
|
)
|
|
129
|
+
motherduck_token: str | None = Field(
|
|
130
|
+
default=None,
|
|
131
|
+
description="MotherDuck service token for 'md:' paths (DuckDB only; "
|
|
132
|
+
"env-interpolated — unset falls back to the motherduck_token "
|
|
133
|
+
"environment variable read by the extension itself)",
|
|
134
|
+
)
|
|
113
135
|
|
|
114
136
|
# Snowflake-only (source-only backend). host/port are meaningless for it:
|
|
115
137
|
# the driver resolves the endpoint from `account`.
|
|
@@ -141,6 +163,34 @@ class ProfileConfig(BaseModel):
|
|
|
141
163
|
description="Passphrase of the private key, when it is encrypted (Snowflake only)",
|
|
142
164
|
)
|
|
143
165
|
|
|
166
|
+
# BigQuery-only (source-only backend). host/port/user/password are
|
|
167
|
+
# meaningless for it: the client resolves the endpoint from `project` and
|
|
168
|
+
# authenticates via a service-account key file or ADC.
|
|
169
|
+
project: str | None = Field(
|
|
170
|
+
default=None,
|
|
171
|
+
description="GCP project id billed for the queries, e.g. 'my-project' (BigQuery only)",
|
|
172
|
+
)
|
|
173
|
+
credentials_json_path: str | None = Field(
|
|
174
|
+
default=None,
|
|
175
|
+
description="Path to a service-account JSON key file (BigQuery only; "
|
|
176
|
+
"unset -> Application Default Credentials)",
|
|
177
|
+
)
|
|
178
|
+
location: str | None = Field(
|
|
179
|
+
default=None,
|
|
180
|
+
description="Job location, e.g. 'EU' (BigQuery only; unset -> inferred "
|
|
181
|
+
"from the referenced datasets)",
|
|
182
|
+
)
|
|
183
|
+
dataset: str | None = Field(
|
|
184
|
+
default=None,
|
|
185
|
+
description="Default dataset for unqualified table names in load queries (BigQuery only)",
|
|
186
|
+
)
|
|
187
|
+
api_endpoint: str | None = Field(
|
|
188
|
+
default=None,
|
|
189
|
+
description="API endpoint override (BigQuery only) — a plain-http endpoint "
|
|
190
|
+
"(the BigQuery emulator) uses anonymous credentials when no key file is "
|
|
191
|
+
"set; https endpoints (regional/private) authenticate normally",
|
|
192
|
+
)
|
|
193
|
+
|
|
144
194
|
# Internal location for _dtk_* tables
|
|
145
195
|
internal_database: str | None = Field(
|
|
146
196
|
default=None, description="Database for internal tables (ClickHouse/MySQL/MariaDB)"
|
|
@@ -215,6 +265,23 @@ class ProfileConfig(BaseModel):
|
|
|
215
265
|
)
|
|
216
266
|
return self
|
|
217
267
|
|
|
268
|
+
@model_validator(mode="after")
|
|
269
|
+
def _validate_bigquery_fields(self) -> "ProfileConfig":
|
|
270
|
+
"""BigQuery profiles need an explicit GCP project id.
|
|
271
|
+
|
|
272
|
+
Auth needs nothing further: `credentials_json_path` when set, else
|
|
273
|
+
Application Default Credentials (gcloud ADC / attached service
|
|
274
|
+
account / Workload Identity).
|
|
275
|
+
"""
|
|
276
|
+
if self.type != "bigquery":
|
|
277
|
+
return self
|
|
278
|
+
if not self.project:
|
|
279
|
+
raise ValueError(
|
|
280
|
+
"bigquery profiles must set 'project' (the GCP project id billed "
|
|
281
|
+
"for the queries, e.g. 'my-project')"
|
|
282
|
+
)
|
|
283
|
+
return self
|
|
284
|
+
|
|
218
285
|
def get_internal_location(self) -> str:
|
|
219
286
|
"""
|
|
220
287
|
Get internal location (database or schema).
|
|
@@ -364,14 +431,15 @@ class ProfileConfig(BaseModel):
|
|
|
364
431
|
if not self.path:
|
|
365
432
|
raise ValueError(
|
|
366
433
|
"DuckDB profiles must set 'path' (the database file path, "
|
|
367
|
-
"
|
|
368
|
-
"
|
|
434
|
+
"':memory:' for a transient, tests/preview-only in-process "
|
|
435
|
+
"database, or 'md:<database>' for MotherDuck)"
|
|
369
436
|
)
|
|
370
437
|
return DuckDBDatabaseManager(
|
|
371
438
|
path=self.path,
|
|
372
439
|
internal_schema=self.get_internal_location(),
|
|
373
440
|
data_schema=self.get_data_location(),
|
|
374
441
|
read_only=self.read_only,
|
|
442
|
+
motherduck_token=self.motherduck_token,
|
|
375
443
|
settings=self.settings,
|
|
376
444
|
ensure_locations=ensure_locations,
|
|
377
445
|
)
|
|
@@ -410,6 +478,19 @@ class ProfileConfig(BaseModel):
|
|
|
410
478
|
role=self.role,
|
|
411
479
|
settings=self.settings,
|
|
412
480
|
)
|
|
481
|
+
if self.type == "bigquery":
|
|
482
|
+
from detectkit.database.bigquery_manager import BigQuerySourceManager
|
|
483
|
+
|
|
484
|
+
# Guaranteed by `_validate_bigquery_fields` at construction time.
|
|
485
|
+
assert self.project is not None
|
|
486
|
+
return BigQuerySourceManager(
|
|
487
|
+
project=self.project,
|
|
488
|
+
credentials_json_path=self.credentials_json_path,
|
|
489
|
+
location=self.location,
|
|
490
|
+
dataset=self.dataset,
|
|
491
|
+
api_endpoint=self.api_endpoint,
|
|
492
|
+
settings=self.settings,
|
|
493
|
+
)
|
|
413
494
|
return self.create_manager()
|
|
414
495
|
|
|
415
496
|
|