detectkit 0.59.0__tar.gz → 0.61.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. {detectkit-0.59.0/detectkit.egg-info → detectkit-0.61.0}/PKG-INFO +7 -2
  2. {detectkit-0.59.0 → detectkit-0.61.0}/README.md +2 -1
  3. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/__init__.py +1 -1
  4. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/CLAUDE.section.md +1 -1
  5. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/metrics.md +14 -0
  6. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/project.md +45 -3
  7. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md +6 -2
  8. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/init.py +13 -1
  9. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/run.py +135 -68
  10. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/metric_config.py +36 -0
  11. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/profile.py +86 -19
  12. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/project_config.py +15 -0
  13. detectkit-0.61.0/detectkit/database/duckdb_manager.py +329 -0
  14. detectkit-0.61.0/detectkit/loaders/errors.py +29 -0
  15. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/loaders/metric_loader.py +23 -3
  16. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/_base.py +80 -0
  17. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/_load_step.py +6 -1
  18. {detectkit-0.59.0 → detectkit-0.61.0/detectkit.egg-info}/PKG-INFO +7 -2
  19. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit.egg-info/SOURCES.txt +2 -0
  20. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit.egg-info/requires.txt +5 -0
  21. {detectkit-0.59.0 → detectkit-0.61.0}/pyproject.toml +8 -1
  22. {detectkit-0.59.0 → detectkit-0.61.0}/LICENSE +0 -0
  23. {detectkit-0.59.0 → detectkit-0.61.0}/MANIFEST.in +0 -0
  24. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/__init__.py +0 -0
  25. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/__init__.py +0 -0
  26. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/base.py +0 -0
  27. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/branding.py +0 -0
  28. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/discord.py +0 -0
  29. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/email.py +0 -0
  30. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/factory.py +0 -0
  31. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/googlechat.py +0 -0
  32. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/mattermost.py +0 -0
  33. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/ntfy.py +0 -0
  34. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/slack.py +0 -0
  35. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/teams.py +0 -0
  36. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/telegram.py +0 -0
  37. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/channels/webhook.py +0 -0
  38. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/__init__.py +0 -0
  39. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_base.py +0 -0
  40. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
  41. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_decision.py +0 -0
  42. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
  43. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_recovery.py +0 -0
  44. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_replay.py +0 -0
  45. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/_types.py +0 -0
  46. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
  47. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/__init__.py +0 -0
  48. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/_base.py +0 -0
  49. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/_types.py +0 -0
  50. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/autotuner.py +0 -0
  51. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/axis_spec.py +0 -0
  52. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/config_emitter.py +0 -0
  53. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/crossval.py +0 -0
  54. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/detector_select.py +0 -0
  55. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/distribution.py +0 -0
  56. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/grid_search.py +0 -0
  57. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/labels.py +0 -0
  58. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/result.py +0 -0
  59. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/runner.py +0 -0
  60. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/scoring.py +0 -0
  61. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/seasonality_search.py +0 -0
  62. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/settings.py +0 -0
  63. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/autotune/window_select.py +0 -0
  64. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/__init__.py +0 -0
  65. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/_output.py +0 -0
  66. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/alerting.md +0 -0
  67. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/autotune.md +0 -0
  68. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/cli.md +0 -0
  69. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/detectors.md +0 -0
  70. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/rules/overview.md +0 -0
  71. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/skills/dtk-autotune/SKILL.md +0 -0
  72. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/skills/dtk-feedback/SKILL.md +0 -0
  73. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/skills/dtk-new-metric/SKILL.md +0 -0
  74. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/assets/claude/skills/dtk-tune/SKILL.md +0 -0
  75. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/__init__.py +0 -0
  76. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/autotune.py +0 -0
  77. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/clean.py +0 -0
  78. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/init_claude.py +0 -0
  79. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/osi.py +0 -0
  80. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/test_alert.py +0 -0
  81. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/tune.py +0 -0
  82. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/ui.py +0 -0
  83. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/commands/unlock.py +0 -0
  84. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/cli/main.py +0 -0
  85. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/__init__.py +0 -0
  86. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/metric_io.py +0 -0
  87. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/config/validator.py +0 -0
  88. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/core/__init__.py +0 -0
  89. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/core/interval.py +0 -0
  90. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/core/models.py +0 -0
  91. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/__init__.py +0 -0
  92. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/_sql_manager.py +0 -0
  93. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/clickhouse_manager.py +0 -0
  94. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/__init__.py +0 -0
  95. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_alert_states.py +0 -0
  96. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_autotune_runs.py +0 -0
  97. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_base.py +0 -0
  98. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_datapoints.py +0 -0
  99. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_detections.py +0 -0
  100. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_maintenance.py +0 -0
  101. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_metrics.py +0 -0
  102. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_schema.py +0 -0
  103. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/_tasks.py +0 -0
  104. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/internal_tables/manager.py +0 -0
  105. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/manager.py +0 -0
  106. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/mysql_manager.py +0 -0
  107. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/postgres_manager.py +0 -0
  108. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/database/tables.py +0 -0
  109. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/__init__.py +0 -0
  110. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/base.py +0 -0
  111. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/factory.py +0 -0
  112. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/seasonality.py +0 -0
  113. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/__init__.py +0 -0
  114. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/_windowed.py +0 -0
  115. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/autoreg.py +0 -0
  116. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/iqr.py +0 -0
  117. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/mad.py +0 -0
  118. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/manual_bounds.py +0 -0
  119. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/detectors/statistical/zscore.py +0 -0
  120. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/loaders/__init__.py +0 -0
  121. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/loaders/query_template.py +0 -0
  122. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/__init__.py +0 -0
  123. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/error_dispatch.py +0 -0
  124. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/__init__.py +0 -0
  125. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
  126. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
  127. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/_types.py +0 -0
  128. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/orchestration/task_manager/manager.py +0 -0
  129. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/reporting/__init__.py +0 -0
  130. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/reporting/assets/report.js +0 -0
  131. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/reporting/builder.py +0 -0
  132. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/reporting/html_report.py +0 -0
  133. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/__init__.py +0 -0
  134. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/errors.py +0 -0
  135. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/exporter.py +0 -0
  136. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/importer.py +0 -0
  137. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/osi_model.py +0 -0
  138. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/semantic/query_gen.py +0 -0
  139. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/__init__.py +0 -0
  140. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/assets/tune.js +0 -0
  141. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/config_writer.py +0 -0
  142. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/html.py +0 -0
  143. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/payload.py +0 -0
  144. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/tuning/server.py +0 -0
  145. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/__init__.py +0 -0
  146. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/assets/ui.js +0 -0
  147. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/html.py +0 -0
  148. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/jobs.py +0 -0
  149. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/metric_files.py +0 -0
  150. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/overview.py +0 -0
  151. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/ui/server.py +0 -0
  152. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/utils/__init__.py +0 -0
  153. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/utils/datetime_utils.py +0 -0
  154. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/utils/env_interpolation.py +0 -0
  155. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/utils/json_utils.py +0 -0
  156. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit/utils/stats.py +0 -0
  157. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit.egg-info/dependency_links.txt +0 -0
  158. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit.egg-info/entry_points.txt +0 -0
  159. {detectkit-0.59.0 → detectkit-0.61.0}/detectkit.egg-info/top_level.txt +0 -0
  160. {detectkit-0.59.0 → detectkit-0.61.0}/requirements.txt +0 -0
  161. {detectkit-0.59.0 → detectkit-0.61.0}/setup.cfg +0 -0
  162. {detectkit-0.59.0 → detectkit-0.61.0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectkit
3
- Version: 0.59.0
3
+ Version: 0.61.0
4
4
  Summary: Metric monitoring with automatic anomaly detection
5
5
  Author: detectkit team
6
6
  License: MIT
@@ -38,10 +38,13 @@ Provides-Extra: mysql
38
38
  Requires-Dist: pymysql>=1.0.0; extra == "mysql"
39
39
  Provides-Extra: mariadb
40
40
  Requires-Dist: pymysql>=1.0.0; extra == "mariadb"
41
+ Provides-Extra: duckdb
42
+ Requires-Dist: duckdb>=1.1; extra == "duckdb"
41
43
  Provides-Extra: all-db
42
44
  Requires-Dist: clickhouse-driver>=0.2.0; extra == "all-db"
43
45
  Requires-Dist: psycopg2-binary>=2.9.0; extra == "all-db"
44
46
  Requires-Dist: pymysql>=1.0.0; extra == "all-db"
47
+ Requires-Dist: duckdb>=1.1; extra == "all-db"
45
48
  Provides-Extra: prophet
46
49
  Requires-Dist: prophet>=1.1.0; extra == "prophet"
47
50
  Provides-Extra: timesfm
@@ -55,6 +58,7 @@ Provides-Extra: all
55
58
  Requires-Dist: clickhouse-driver>=0.2.0; extra == "all"
56
59
  Requires-Dist: psycopg2-binary>=2.9.0; extra == "all"
57
60
  Requires-Dist: pymysql>=1.0.0; extra == "all"
61
+ Requires-Dist: duckdb>=1.1; extra == "all"
58
62
  Requires-Dist: prophet>=1.1.0; extra == "all"
59
63
  Requires-Dist: timesfm>=0.1.0; extra == "all"
60
64
  Requires-Dist: sqlglot>=20.0; extra == "all"
@@ -99,7 +103,8 @@ Dynamic: license-file
99
103
  - **@mentions** — tag users/groups in alerts, each channel formats natively
100
104
  - **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
101
105
  - **Project-level error alerts** — catch DB outages and pipeline crashes once per run
102
- - **Database agnostic** — ClickHouse, PostgreSQL, MySQL
106
+ - **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed)
107
+ - **Hybrid mode** — read metric SQL from a billed-per-query warehouse while all pipeline state lives in a cheap local database
103
108
  - **Idempotent** — resume from interruptions, no duplicate processing
104
109
  - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
105
110
  - **AI-native onboarding** — `dtk init-claude` sets up Claude Code context (CLAUDE.md + rules + three skills) so an assistant can scaffold metrics, configure databases, and file feedback upstream
@@ -24,7 +24,8 @@
24
24
  - **@mentions** — tag users/groups in alerts, each channel formats natively
25
25
  - **Alert lifecycle** — consecutive anomalies, fraction-of-window rule (`anomaly_window` + `min_anomaly_share`), cooldown, recovery notifications, no-data alerts
26
26
  - **Project-level error alerts** — catch DB outages and pipeline crashes once per run
27
- - **Database agnostic** — ClickHouse, PostgreSQL, MySQL
27
+ - **Database agnostic** — ClickHouse, PostgreSQL, MySQL/MariaDB, DuckDB (no server needed)
28
+ - **Hybrid mode** — read metric SQL from a billed-per-query warehouse while all pipeline state lives in a cheap local database
28
29
  - **Idempotent** — resume from interruptions, no duplicate processing
29
30
  - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
30
31
  - **AI-native onboarding** — `dtk init-claude` sets up Claude Code context (CLAUDE.md + rules + three skills) so an assistant can scaffold metrics, configure databases, and file feedback upstream
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
4
4
  A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
5
5
  """
6
6
 
7
- __version__ = "0.59.0"
7
+ __version__ = "0.61.0"
8
8
 
9
9
  from detectkit.core.interval import Interval
10
10
  from detectkit.core.models import ColumnDefinition, TableModel
@@ -14,7 +14,7 @@ conventions.
14
14
  **Database access for _you_ (recommended, not required).** detectkit itself
15
15
  connects to the database directly via its drivers — it **never** needs an MCP to
16
16
  run. But you assist far better with **read access to the same database** (e.g. a
17
- database MCP for the project's ClickHouse / PostgreSQL / MySQL / MariaDB): you can inspect
17
+ database MCP for the project's ClickHouse / PostgreSQL / MySQL / MariaDB / DuckDB): you can inspect
18
18
  a metric's series, find real incidents to label for `dtk autotune`, sanity-check
19
19
  a metric query before running it, and confirm detections — instead of asking the
20
20
  user to run every query by hand. Without it, fall back to marking incidents
@@ -11,6 +11,7 @@ keep them in sync. Detectors are covered in `detectors.md`, alerting in
11
11
  name: api_response_time # required, unique across the project
12
12
  description: API p95 latency # optional, shown in alerts
13
13
  profile: prod # optional, overrides project default_profile
14
+ source_profile: warehouse # optional — hybrid mode: source-DB profile for THIS metric's SQL (see below)
14
15
  enabled: true # optional, false → skipped by `dtk run`
15
16
  tags: [critical, api] # optional, used by `--select tag:<t>`
16
17
  ai_context: # optional — OSI-compatible grounding (descriptive only)
@@ -67,6 +68,19 @@ false_alert_budget: 0.3 # optional — `dtk tune` target false-alert rate
67
68
  project-wide default; unset → project, then a built-in `0.5`. Tuning-only — it never
68
69
  affects the load/detect/alert pipeline.
69
70
 
71
+ `source_profile` is **hybrid mode** for this one metric: the `profiles.yml`
72
+ profile whose database runs *this metric's* SQL query, while `_dtk_*` state
73
+ (datapoints, detections, task locks, alert state) stays in the active
74
+ **state** profile — the one `dtk run` is already connected to for everything
75
+ else. Overrides the project-wide `source_profile` (`project.md`); resolves
76
+ **metric → project → unset** (unset on both = today's behavior — one profile
77
+ for everything). Not the same field as `profile` above (`profile` is dead —
78
+ not applied at runtime by `dtk run`; `source_profile` is live). Only the
79
+ metric's **load** step ever connects to it — detect/alert and every other
80
+ command don't — though every `dtk run` fail-fast validates the resolved name
81
+ up front regardless of `--steps` (a typo exits 1 before any pipeline work).
82
+ See the [Hybrid Mode guide](https://dtk.pipelab.dev/guides/hybrid-mode/).
83
+
70
84
  `ai_context` is **OSI-compatible grounding** you can add to any metric with **no
71
85
  OSI model needed** — the metric's business meaning (`instructions`), alternative
72
86
  names (`synonyms`) and example values (`examples`), mirroring the
@@ -33,6 +33,7 @@ timeouts: # per-step, seconds
33
33
  alert_help_url: null # optional, see below — "How to read this alert" link
34
34
  false_alert_budget: null # optional — `dtk tune` target FDR (0,1]; per-metric override wins
35
35
  loading_delay: null # optional — default data-maturity delay; per-metric override wins
36
+ source_profile: null # optional — hybrid mode: default source-DB profile; per-metric override wins
36
37
 
37
38
  error_alerting: # optional, see below
38
39
  enabled: false
@@ -50,6 +51,26 @@ overrides it (`0` opts that metric out); see `metrics.md` for the full
50
51
  load/no-data behavior and the detection-latency trade-off. Only set this
51
52
  project-wide when your metrics genuinely share the same upstream schedule.
52
53
 
54
+ `source_profile` is **hybrid mode**: a project-wide default `profiles.yml`
55
+ profile whose database runs metric SQL, while *every* `_dtk_*` table
56
+ (datapoints, detections, task locks, alert state) stays in the active
57
+ **state** profile — the one `dtk run` is already connected to for everything
58
+ else. A per-metric `source_profile` overrides it (`metrics.md`); resolves
59
+ **metric → project → unset**, same precedence as `loading_delay`. Unset on
60
+ both = today's behavior (one profile does everything). Only the LOAD step's
61
+ metric-SQL query is affected — detect/alert and every other command
62
+ (`dtk autotune`/`tune`/`ui`/`clean`/`unlock`) only ever touch the state
63
+ profile. A source-side failure raises `SourceDatabaseError` (message leads
64
+ with `source database (profile '<name>'): ...`), distinct from a plain
65
+ exception for a state-side failure — useful for telling a warehouse outage
66
+ apart from a state-DB outage in `error_alerting`'s `{error_type}` /
67
+ `{error_message}`. `dtk run` validates every resolved `source_profile` name
68
+ against `profiles.yml` before opening any connection (unknown name → exit
69
+ `1`, no partial run). Full config example + operational caveats (connecting a
70
+ profile always runs `CREATE DATABASE`/`SCHEMA IF NOT EXISTS` for its own
71
+ `internal_*` location too, even a source-only one) in the [Hybrid Mode
72
+ guide](https://dtk.pipelab.dev/guides/hybrid-mode/).
73
+
53
74
  ### `alert_help_url` — "How to read this alert" link
54
75
 
55
76
  Every default-rendered alert on every channel carries a `How to read this alert`
@@ -116,10 +137,12 @@ alert_channels:
116
137
 
117
138
  ### Database profiles
118
139
 
119
- > ClickHouse, PostgreSQL, MySQL and MariaDB are all fully supported.
140
+ > ClickHouse, PostgreSQL, MySQL, MariaDB and DuckDB are all supported.
120
141
  > ClickHouse/MySQL/MariaDB use two *databases*; PostgreSQL connects to one
121
- > `database` and uses two *schemas*.
122
- > `dtk init --db-type {clickhouse,postgres,mysql,mariadb}` scaffolds the right shape.
142
+ > `database` and uses two *schemas*; DuckDB is a single *file* (or `:memory:`
143
+ > for tests) holding two *schemas*.
144
+ > `dtk init --db-type {clickhouse,postgres,mysql,mariadb}` scaffolds the right
145
+ > shape — DuckDB profiles are written by hand (not yet a `--db-type` choice).
123
146
 
124
147
  **ClickHouse**:
125
148
  ```yaml
@@ -173,6 +196,25 @@ auto-detected at connect (`SELECT VERSION()`), so MariaDB gets its own
173
196
  `VALUES()`-form upsert instead of MySQL 8.0.19's row-alias form.
174
197
  `pip install 'detectkit[mariadb]'`.
175
198
 
199
+ **DuckDB** (embedded, single file; two schemas like PostgreSQL):
200
+ ```yaml
201
+ profiles:
202
+ prod:
203
+ type: duckdb
204
+ path: /var/lib/detectkit/warehouse.duckdb # required — file path, or ":memory:"
205
+ internal_schema: detectkit # optional — _dtk_* tables (default: "detectkit")
206
+ data_schema: main # optional — data queries (default: "main")
207
+ read_only: false # optional (default: false)
208
+ settings: {} # optional — extra duckdb.connect() config options
209
+ ```
210
+ > Single-writer: DuckDB allows only **one** read-write connection to the file
211
+ > at a time — a `dtk ui`/`dtk tune` session left open will clash with a
212
+ > separately spawned `dtk run` against the same file. Point a read-only
213
+ > consumer at it with `read_only: true`.
214
+ > `path: ":memory:"` has no on-disk state, so resume/idempotency breaks across
215
+ > process restarts — use it for tests/scratch only, never a real project.
216
+ > `pip install 'detectkit[duckdb]'`.
217
+
176
218
  ### Alert channels
177
219
 
178
220
  Defined once in `profiles.yml`, referenced by name in each metric's
@@ -50,13 +50,17 @@ side by side, ask which one to set up.
50
50
 
51
51
  ## Step 1 — Pick the database backend
52
52
 
53
- **ClickHouse, PostgreSQL, MySQL and MariaDB are all fully supported.** Ask
53
+ **ClickHouse, PostgreSQL, MySQL, MariaDB and DuckDB are all supported.** Ask
54
54
  which one the project uses (default to ClickHouse if unsure). `dtk init
55
55
  --db-type {clickhouse,postgres,mysql,mariadb}` scaffolds `profiles.yml` for the
56
- chosen backend. The location fields differ:
56
+ chosen backend (DuckDB isn't a `--db-type` choice yet — write its profile by
57
+ hand; see `project.md`). The location fields differ:
57
58
  - **ClickHouse** / **MySQL** / **MariaDB** — two *databases*: `internal_database` / `data_database`.
58
59
  - **PostgreSQL** — connect to a `database` (must already exist), then two
59
60
  *schemas*: `internal_schema` / `data_schema`.
61
+ - **DuckDB** — a single **file** (`path`, or `":memory:"` for tests only),
62
+ then two *schemas*: `internal_schema` / `data_schema`. No host/port/user/
63
+ password — it's an embedded, single-writer database (see `project.md`).
60
64
 
61
65
  `type: mariadb` is an identical profile alias to `type: mysql` (same location
62
66
  fields; the vendor is auto-detected at connect). The metric query SQL dialect
@@ -123,6 +123,15 @@ _COMMENTED_EXAMPLES = {
123
123
  # password: ""
124
124
  # internal_database: detectkit
125
125
  # data_database: analytics
126
+ """,
127
+ "duckdb": """ # Example DuckDB profile (in-process, single-file; no host/port/user/password)
128
+ # duckdb_dev:
129
+ # type: duckdb
130
+ # path: ./detectkit.duckdb # or ':memory:' for a transient, tests-only DB
131
+ # internal_schema: detectkit
132
+ # data_schema: main
133
+ # # Only ONE read-write connection at a time — don't point a scheduled
134
+ # # `dtk run` and a long-lived `dtk ui` at the same file simultaneously.
126
135
  """,
127
136
  }
128
137
 
@@ -351,7 +360,10 @@ timeouts:
351
360
  # live under a top-level 'profiles:' mapping). The active dev/prod profiles
352
361
  # are scaffolded for the chosen --db-type; the other backends are included
353
362
  # as commented examples. See the per-database docs for connection details.
354
- other_backends = [t for t in ("clickhouse", "postgres", "mysql") if t != db_type]
363
+ # "duckdb" is not a `--db-type` choice (there is no active/prod scaffold
364
+ # for it — an in-process file DB has no separate dev/prod endpoints), so
365
+ # it always shows up as a commented example, unlike the other backends.
366
+ other_backends = [t for t in ("clickhouse", "postgres", "mysql", "duckdb") if t != db_type]
355
367
  commented = "\n".join(_COMMENTED_EXAMPLES[t] for t in other_backends)
356
368
  profiles_config = (
357
369
  "# Database connection profiles\n\n"
@@ -13,7 +13,7 @@ from typing import Any
13
13
 
14
14
  import click
15
15
 
16
- from detectkit.config.metric_config import MetricConfig
16
+ from detectkit.config.metric_config import MetricConfig, resolve_source_profile
17
17
  from detectkit.config.profile import ProfilesConfig
18
18
  from detectkit.config.project_config import ProjectConfig
19
19
  from detectkit.config.validator import (
@@ -326,6 +326,21 @@ def _run_impl(
326
326
  summary["error"] = f"Error loading profiles.yml: {e}"
327
327
  return 1
328
328
 
329
+ # Fail-fast: every selected metric's RESOLVED source_profile (hybrid
330
+ # mode) must name a real profile. This is a cheap name check — no
331
+ # connections opened — so a typo'd source_profile fails the whole run
332
+ # up front instead of surfacing deep inside whichever metric's LOAD step
333
+ # happens to hit it first.
334
+ source_profile_error = _validate_source_profiles(metrics, project_config, profiles_config)
335
+ if source_profile_error:
336
+ # A config typo, not an outage: exit 1 without paging error_alerting,
337
+ # consistent with the sibling config-error paths (bad selector,
338
+ # unparseable profiles.yml). The error alert channel is reserved for
339
+ # DB-down / DDL / runtime failures.
340
+ click.echo(click.style(f"Error: {source_profile_error}", fg="red", bold=True))
341
+ summary["error"] = f"Error: {source_profile_error}"
342
+ return 1
343
+
329
344
  # Create database manager
330
345
  try:
331
346
  db_manager = profiles_config.create_manager(profile)
@@ -372,84 +387,136 @@ def _run_impl(
372
387
  summary["error"] = f"Error initializing internal tables: {e}"
373
388
  return 1
374
389
 
390
+ # The active STATE profile's actual name (mirrors ProfilesConfig.get_profile's
391
+ # own None -> default_profile resolution) — lets the task manager recognize
392
+ # a metric's source_profile that happens to name the state profile itself,
393
+ # and reuse db_manager instead of opening a duplicate connection.
394
+ state_profile_name: str | None
395
+ if profile is not None:
396
+ state_profile_name = profile
397
+ else:
398
+ state_profile_name = getattr(profiles_config, "default_profile", None)
399
+
375
400
  # Create task manager
376
401
  task_manager = TaskManager(
377
402
  internal_manager=internal_manager,
378
403
  db_manager=db_manager,
379
404
  profiles_config=profiles_config,
380
405
  project_config=project_config,
406
+ state_profile_name=state_profile_name,
381
407
  )
382
408
 
383
- # Process each metric
384
- for index, (metric_path, config) in enumerate(metrics):
385
- result = process_metric(
386
- metric_path=metric_path,
387
- config=config,
388
- project_root=project_root,
389
- task_manager=task_manager,
390
- steps=step_list,
391
- from_date=from_dt,
392
- to_date=to_dt,
393
- full_refresh=full_refresh,
394
- force=force,
395
- )
396
- summary["metrics"].append(
397
- {
398
- "name": config.name,
399
- "status": _status_str(result["status"]),
400
- "steps_completed": [s.value for s in result["steps_completed"]],
401
- "datapoints_loaded": result["datapoints_loaded"],
402
- "anomalies_detected": result["anomalies_detected"],
403
- "alerts_sent": result["alerts_sent"],
404
- "error": result["error"],
405
- }
406
- )
407
-
408
- # Project-level error alert was dispatched — stop processing the
409
- # rest of the metrics. The DB / source is presumed unreachable;
410
- # subsequent metrics would all fail with the same error.
411
- if result.get("abort_run"):
412
- click.echo(
413
- click.style(
414
- "✗ Aborting run after project error alert. " "Remaining metrics skipped.",
415
- fg="red",
416
- bold=True,
417
- )
409
+ try:
410
+ # Process each metric
411
+ for index, (metric_path, config) in enumerate(metrics):
412
+ result = process_metric(
413
+ metric_path=metric_path,
414
+ config=config,
415
+ project_root=project_root,
416
+ task_manager=task_manager,
417
+ steps=step_list,
418
+ from_date=from_dt,
419
+ to_date=to_dt,
420
+ full_refresh=full_refresh,
421
+ force=force,
422
+ )
423
+ summary["metrics"].append(
424
+ {
425
+ "name": config.name,
426
+ "status": _status_str(result["status"]),
427
+ "steps_completed": [s.value for s in result["steps_completed"]],
428
+ "datapoints_loaded": result["datapoints_loaded"],
429
+ "anomalies_detected": result["anomalies_detected"],
430
+ "alerts_sent": result["alerts_sent"],
431
+ "error": result["error"],
432
+ }
418
433
  )
419
- summary["aborted"] = True
420
- for _, skipped_config in metrics[index + 1 :]:
421
- summary["metrics"].append(
422
- {
423
- "name": skipped_config.name,
424
- "status": "skipped",
425
- "steps_completed": [],
426
- "datapoints_loaded": 0,
427
- "anomalies_detected": 0,
428
- "alerts_sent": 0,
429
- "error": None,
430
- }
431
- )
432
- break
433
434
 
434
- # Optional: emit a self-contained HTML report from the freshly-persisted
435
- # internal tables (values + bands + anomalies + replayed alerts).
436
- if report_path is not None:
437
- try:
438
- emit_metric_report(
439
- config=config,
440
- project_root=project_root,
441
- internal_manager=internal_manager,
442
- report_path=report_path,
443
- project_name=getattr(project_config, "name", None),
444
- project_loading_delay=getattr(project_config, "loading_delay", None),
445
- from_dt=from_dt,
446
- to_dt=to_dt,
435
+ # Project-level error alert was dispatched stop processing the
436
+ # rest of the metrics. The DB / source is presumed unreachable;
437
+ # subsequent metrics would all fail with the same error.
438
+ if result.get("abort_run"):
439
+ click.echo(
440
+ click.style(
441
+ "✗ Aborting run after project error alert. " "Remaining metrics skipped.",
442
+ fg="red",
443
+ bold=True,
444
+ )
447
445
  )
448
- except Exception as report_error: # never fail the run on a report
449
- click.echo(click.style(f" │ Report skipped: {report_error}", fg="yellow"))
450
-
451
- failed = any(m["status"] == "failed" for m in summary["metrics"])
452
- return 1 if (failed or summary["aborted"]) else 0
446
+ summary["aborted"] = True
447
+ for _, skipped_config in metrics[index + 1 :]:
448
+ summary["metrics"].append(
449
+ {
450
+ "name": skipped_config.name,
451
+ "status": "skipped",
452
+ "steps_completed": [],
453
+ "datapoints_loaded": 0,
454
+ "anomalies_detected": 0,
455
+ "alerts_sent": 0,
456
+ "error": None,
457
+ }
458
+ )
459
+ break
460
+
461
+ # Optional: emit a self-contained HTML report from the freshly-persisted
462
+ # internal tables (values + bands + anomalies + replayed alerts).
463
+ if report_path is not None:
464
+ try:
465
+ emit_metric_report(
466
+ config=config,
467
+ project_root=project_root,
468
+ internal_manager=internal_manager,
469
+ report_path=report_path,
470
+ project_name=getattr(project_config, "name", None),
471
+ project_loading_delay=getattr(project_config, "loading_delay", None),
472
+ from_dt=from_dt,
473
+ to_dt=to_dt,
474
+ )
475
+ except Exception as report_error: # never fail the run on a report
476
+ click.echo(click.style(f" │ Report skipped: {report_error}", fg="yellow"))
477
+
478
+ failed = any(m["status"] == "failed" for m in summary["metrics"])
479
+ return 1 if (failed or summary["aborted"]) else 0
480
+ finally:
481
+ # Close every pooled hybrid-mode SOURCE manager (never db_manager
482
+ # itself — this function owns that connection's lifecycle, and it
483
+ # was never explicitly closed here either, before or after hybrid
484
+ # mode existed). Duck-typed lookup: a test double standing in for
485
+ # TaskManager need not implement it.
486
+ close_sources = getattr(task_manager, "close_sources", None)
487
+ if close_sources is not None:
488
+ close_sources()
489
+
490
+
491
+ def _validate_source_profiles(
492
+ metrics: list[tuple[Path, MetricConfig]],
493
+ project_config: ProjectConfig,
494
+ profiles_config: ProfilesConfig,
495
+ ) -> str | None:
496
+ """Cheap, connection-free check that every metric's RESOLVED
497
+ ``source_profile`` (hybrid mode) names a real profile.
498
+
499
+ Returns an error message naming every offending metric, or ``None`` when
500
+ every resolved name is either unset (no hybrid override) or a known
501
+ profile. Runs before any database manager is built.
502
+ """
503
+ project_source_profile = getattr(project_config, "source_profile", None)
504
+ known_profiles = getattr(profiles_config, "profiles", {}) or {}
505
+ unknown: dict[str, str] = {}
506
+ for _, config in metrics:
507
+ name = resolve_source_profile(
508
+ getattr(config, "source_profile", None), project_source_profile
509
+ )
510
+ if name is not None and name not in known_profiles:
511
+ unknown[config.name] = name
512
+ if not unknown:
513
+ return None
514
+ details = ", ".join(f"{m} -> '{p}'" for m, p in sorted(unknown.items()))
515
+ available = ", ".join(sorted(known_profiles.keys()))
516
+ return (
517
+ f"Unknown source_profile referenced by metric(s): {details}. "
518
+ f"Available profiles: {available}"
519
+ )
453
520
 
454
521
 
455
522
  def _resolve_report_path(report_path: str, project_root: Path, metric_name: str) -> Path:
@@ -627,6 +627,8 @@ class MetricConfig(BaseModel):
627
627
  description: Optional metric description (supports multi-line text)
628
628
  tags: Optional list of tags for metric selection (e.g., ["critical", "api"])
629
629
  profile: Profile name to use (overrides default_profile from project config)
630
+ source_profile: Hybrid mode — profile whose database runs this metric's
631
+ SQL, while _dtk_* state stays in the active (state) profile
630
632
  query: Inline SQL query (mutually exclusive with query_file)
631
633
  query_file: Path to SQL file (mutually exclusive with query)
632
634
  query_columns: Column name mapping for query results
@@ -684,6 +686,22 @@ class MetricConfig(BaseModel):
684
686
  profile: str | None = Field(
685
687
  default=None, description="Profile name to use (overrides default_profile)"
686
688
  )
689
+ # Hybrid mode: run this metric's SQL against a different database than the
690
+ # one holding _dtk_* state. Unlike `profile` above (which is dead — only
691
+ # round-tripped by autotune's config emitter), this is live: the load step
692
+ # resolves it to a pooled `ProfilesConfig.create_manager(...)` and executes
693
+ # the metric's query through it, while every other step (detect/alert) and
694
+ # save_datapoints itself keep using the active state profile. Falls back to
695
+ # ProjectConfig.source_profile, then None (= use the state profile).
696
+ source_profile: str | None = Field(
697
+ default=None,
698
+ description=(
699
+ "profiles.yml profile whose database runs this metric's SQL; "
700
+ "_dtk_* state stays in the active (state) profile. Falls back to "
701
+ "the project-level source_profile, then to the state profile "
702
+ "itself."
703
+ ),
704
+ )
687
705
  query: str | None = Field(default=None, description="Inline SQL query")
688
706
  query_file: Path | None = Field(default=None, description="Path to SQL file")
689
707
  query_columns: QueryColumnsConfig | None = Field(
@@ -1054,6 +1072,24 @@ def resolve_loading_delay_seconds(
1054
1072
  return 0
1055
1073
 
1056
1074
 
1075
+ def resolve_source_profile(
1076
+ metric_source_profile: str | None, project_source_profile: str | None
1077
+ ) -> str | None:
1078
+ """Resolve the profiles.yml profile that should run a metric's SQL:
1079
+ metric -> project -> ``None``.
1080
+
1081
+ ``None`` means "run the SQL through the same connection detectkit is
1082
+ already using for ``_dtk_*`` state" — i.e. hybrid mode is off for this
1083
+ metric and the meaning of ``dtk run --profile`` / ``default_profile`` is
1084
+ unchanged. The first configured value wins, mirroring
1085
+ :func:`resolve_loading_delay_seconds`'s metric -> project precedence.
1086
+ """
1087
+ for value in (metric_source_profile, project_source_profile):
1088
+ if value is not None:
1089
+ return value
1090
+ return None
1091
+
1092
+
1057
1093
  def resolve_grid_phase_seconds(loading_start_time: str | None, interval_seconds: int) -> int:
1058
1094
  """Phase of the metric's interval grid on the epoch clock, in ``[0, interval)``.
1059
1095