detectkit 0.10.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. {detectkit-0.10.0/detectkit.egg-info → detectkit-0.11.0}/PKG-INFO +4 -2
  2. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/__init__.py +1 -1
  3. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/overview.md +3 -3
  4. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/project.md +18 -20
  5. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md +13 -8
  6. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/init.py +185 -95
  7. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/main.py +11 -3
  8. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/profile.py +39 -3
  9. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/models.py +11 -0
  10. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/__init__.py +6 -0
  11. detectkit-0.11.0/detectkit/database/_sql_manager.py +398 -0
  12. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/clickhouse_manager.py +38 -16
  13. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_alert_states.py +14 -29
  14. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_datapoints.py +6 -5
  15. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_detections.py +9 -11
  16. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_maintenance.py +5 -7
  17. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_schema.py +5 -1
  18. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/manager.py +73 -0
  19. detectkit-0.11.0/detectkit/database/mysql_manager.py +132 -0
  20. detectkit-0.11.0/detectkit/database/postgres_manager.py +118 -0
  21. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/tables.py +3 -0
  22. {detectkit-0.10.0 → detectkit-0.11.0/detectkit.egg-info}/PKG-INFO +4 -2
  23. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/SOURCES.txt +3 -0
  24. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/requires.txt +3 -1
  25. {detectkit-0.10.0 → detectkit-0.11.0}/pyproject.toml +5 -1
  26. {detectkit-0.10.0 → detectkit-0.11.0}/LICENSE +0 -0
  27. {detectkit-0.10.0 → detectkit-0.11.0}/MANIFEST.in +0 -0
  28. {detectkit-0.10.0 → detectkit-0.11.0}/README.md +0 -0
  29. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/__init__.py +0 -0
  30. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/__init__.py +0 -0
  31. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/base.py +0 -0
  32. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/email.py +0 -0
  33. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/factory.py +0 -0
  34. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/mattermost.py +0 -0
  35. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/slack.py +0 -0
  36. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/telegram.py +0 -0
  37. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/webhook.py +0 -0
  38. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/__init__.py +0 -0
  39. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_base.py +0 -0
  40. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
  41. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_decision.py +0 -0
  42. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
  43. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_recovery.py +0 -0
  44. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_types.py +0 -0
  45. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
  46. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/__init__.py +0 -0
  47. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/_output.py +0 -0
  48. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/CLAUDE.section.md +0 -0
  49. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/alerting.md +0 -0
  50. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/cli.md +0 -0
  51. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/detectors.md +0 -0
  52. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/metrics.md +0 -0
  53. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/skills/dtk-new-metric/SKILL.md +0 -0
  54. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/__init__.py +0 -0
  55. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/clean.py +0 -0
  56. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/init_claude.py +0 -0
  57. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/run.py +0 -0
  58. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/test_alert.py +0 -0
  59. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/unlock.py +0 -0
  60. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/__init__.py +0 -0
  61. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/metric_config.py +0 -0
  62. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/project_config.py +0 -0
  63. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/validator.py +0 -0
  64. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/__init__.py +0 -0
  65. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/interval.py +0 -0
  66. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/__init__.py +0 -0
  67. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_base.py +0 -0
  68. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_metrics.py +0 -0
  69. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_tasks.py +0 -0
  70. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/manager.py +0 -0
  71. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/__init__.py +0 -0
  72. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/base.py +0 -0
  73. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/factory.py +0 -0
  74. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/seasonality.py +0 -0
  75. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/__init__.py +0 -0
  76. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/_windowed.py +0 -0
  77. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/iqr.py +0 -0
  78. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/mad.py +0 -0
  79. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/manual_bounds.py +0 -0
  80. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/zscore.py +0 -0
  81. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/__init__.py +0 -0
  82. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/metric_loader.py +0 -0
  83. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/query_template.py +0 -0
  84. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/__init__.py +0 -0
  85. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/error_dispatch.py +0 -0
  86. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/__init__.py +0 -0
  87. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
  88. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_base.py +0 -0
  89. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
  90. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_load_step.py +0 -0
  91. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_types.py +0 -0
  92. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/manager.py +0 -0
  93. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/__init__.py +0 -0
  94. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/datetime_utils.py +0 -0
  95. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/env_interpolation.py +0 -0
  96. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/json_utils.py +0 -0
  97. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/stats.py +0 -0
  98. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/dependency_links.txt +0 -0
  99. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/entry_points.txt +0 -0
  100. {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/top_level.txt +0 -0
  101. {detectkit-0.10.0 → detectkit-0.11.0}/requirements.txt +0 -0
  102. {detectkit-0.10.0 → detectkit-0.11.0}/setup.cfg +0 -0
  103. {detectkit-0.10.0 → detectkit-0.11.0}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectkit
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Metric monitoring with automatic anomaly detection
5
5
  Author: detectkit team
6
6
  License: MIT
@@ -61,7 +61,9 @@ Requires-Dist: black>=23.0; extra == "dev"
61
61
  Requires-Dist: mypy>=1.0; extra == "dev"
62
62
  Requires-Dist: ruff>=0.1.0; extra == "dev"
63
63
  Provides-Extra: integration
64
- Requires-Dist: testcontainers[clickhouse]>=4.0; extra == "integration"
64
+ Requires-Dist: testcontainers[clickhouse,mysql,postgres]>=4.0; extra == "integration"
65
+ Requires-Dist: psycopg2-binary>=2.9.0; extra == "integration"
66
+ Requires-Dist: pymysql>=1.0.0; extra == "integration"
65
67
  Dynamic: license-file
66
68
 
67
69
  # detectkit
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
4
4
  A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
5
5
  """
6
6
 
7
- __version__ = "0.10.0"
7
+ __version__ = "0.11.0"
8
8
 
9
9
  from detectkit.core.interval import Interval
10
10
  from detectkit.core.models import ColumnDefinition, TableModel
@@ -3,9 +3,9 @@
3
3
  detectkit is a Python library and CLI (`dtk`) for monitoring time-series
4
4
  metrics with automatic anomaly detection and multi-channel alerting. It is
5
5
  **dbt-like**: metrics live as YAML + SQL in a project directory, and you run
6
- them with one command. Core logic is pure numpy (no pandas). **ClickHouse is
7
- the only implemented backend today**; PostgreSQL and MySQL are planned (their
8
- profiles validate but creating a manager raises `NotImplementedError`).
6
+ them with one command. Core logic is pure numpy (no pandas). **ClickHouse,
7
+ PostgreSQL and MySQL are all fully supported** only the connection and the SQL
8
+ dialect of your metric queries differ between them.
9
9
 
10
10
  ## The pipeline: load → detect → alert
11
11
 
@@ -78,11 +78,11 @@ alert_channels:
78
78
 
79
79
  ### Database profiles
80
80
 
81
- > ClickHouse is the only implemented backend today. PostgreSQL and MySQL are
82
- > planned their profiles validate, but creating a manager raises
83
- > `NotImplementedError("... coming soon")`.
81
+ > ClickHouse, PostgreSQL and MySQL are all fully supported. ClickHouse/MySQL use
82
+ > two *databases*; PostgreSQL connects to one `database` and uses two *schemas*.
83
+ > `dtk init --db-type {clickhouse,postgres,mysql}` scaffolds the right shape.
84
84
 
85
- **ClickHouse** (priority backend):
85
+ **ClickHouse**:
86
86
  ```yaml
87
87
  profiles:
88
88
  prod:
@@ -98,36 +98,34 @@ profiles:
98
98
  max_memory_usage: 10000000000
99
99
  ```
100
100
 
101
- **PostgreSQL** (planned, not yet implemented):
101
+ **PostgreSQL** (connect to `database`, tables in schemas):
102
102
  ```yaml
103
103
  profiles:
104
104
  prod:
105
105
  type: postgres
106
106
  host: localhost
107
107
  port: 5432
108
- database: analytics # required
108
+ database: detectkit # required — must already exist
109
109
  user: postgres
110
110
  password: "..."
111
- internal_schema: detectkit # required — _dtk_* tables
111
+ internal_schema: detectkit # required — _dtk_* tables (auto-created)
112
112
  data_schema: public # required — data queries
113
- pool_size: 5 # optional
114
- max_overflow: 10 # optional
113
+ settings: {} # optional — extra psycopg2.connect kwargs
115
114
  ```
116
115
 
117
- **MySQL** (planned, not yet implemented):
116
+ **MySQL** (8.0+; two databases):
118
117
  ```yaml
119
118
  profiles:
120
119
  prod:
121
120
  type: mysql
122
121
  host: localhost
123
122
  port: 3306
124
- database: analytics # required
125
123
  user: root
126
124
  password: "..."
127
- internal_database: detectkit # required
125
+ internal_database: detectkit # required — _dtk_* tables (auto-created)
128
126
  data_database: analytics # required
129
- charset: utf8mb4 # optional
130
- autocommit: true # optional
127
+ database: analytics # optional — default db for the connection
128
+ settings: {} # optional — extra pymysql.connect kwargs
131
129
  ```
132
130
 
133
131
  ### Alert channels
@@ -185,12 +183,12 @@ alert_channels:
185
183
  ## Notes
186
184
 
187
185
  - **First-run setup:** the `profiles.yml` that `dtk init` writes is a
188
- placeholder its `dev` profile points `internal_database` / `data_database`
189
- at example values (`detectkit` / `default`) on `localhost`. Edit the host,
190
- credentials and both database names to match your environment before running
191
- (the **`dtk-setup-project`** skill walks this). There is no `database:` field —
192
- ClickHouse needs both `internal_database` and `data_database`, or the run
193
- raises `internal_database must be set for ClickHouse`.
186
+ placeholder scaffolded for `--db-type` (default ClickHouse) its `dev`
187
+ profile points the location fields at example values on `localhost`. Edit the
188
+ host, credentials and location names to match your environment before running
189
+ (the **`dtk-setup-project`** skill walks this). ClickHouse/MySQL use
190
+ `internal_database` / `data_database` (no `database:` field on ClickHouse);
191
+ PostgreSQL connects to a `database` and uses `internal_schema` / `data_schema`.
194
192
  - `dtk run` (without `--profile`) uses the `default_profile` declared in
195
193
  **`profiles.yml`**; the `default_profile` in `detectkit_project.yml` is not
196
194
  read at runtime — keep them in sync to avoid confusion.
@@ -40,12 +40,17 @@ side by side, ask which one to set up.
40
40
 
41
41
  ## Step 1 — Pick the database backend
42
42
 
43
- **ClickHouse is the only implemented backend today.** PostgreSQL and MySQL
44
- profiles validate, but `dtk run` raises `NotImplementedError("PostgreSQL
45
- support coming soon")` / `"MySQL support coming soon"` when it builds the
46
- manager. If the user wants Postgres/MySQL, say plainly that runs won't work
47
- yet; you can still write the profile shape for later, but set expectations.
48
- Assume ClickHouse unless told otherwise.
43
+ **ClickHouse, PostgreSQL and MySQL are all fully supported.** Ask which one the
44
+ project uses (default to ClickHouse if unsure). `dtk init --db-type
45
+ {clickhouse,postgres,mysql}` scaffolds `profiles.yml` for the chosen backend.
46
+ The location fields differ:
47
+ - **ClickHouse** / **MySQL** two *databases*: `internal_database` / `data_database`.
48
+ - **PostgreSQL** connect to a `database` (must already exist), then two
49
+ *schemas*: `internal_schema` / `data_schema`.
50
+
51
+ The metric query SQL dialect also differs (e.g. `toStartOfInterval` on
52
+ ClickHouse vs `date_trunc`/`to_timestamp` on Postgres vs `FROM_UNIXTIME` on
53
+ MySQL). Everything else — detectors, alerting, the CLI — is identical.
49
54
 
50
55
  ## Step 2 — Connection details (gather, don't guess)
51
56
 
@@ -74,8 +79,8 @@ values):
74
79
  data, e.g. `detectkit` or `monitoring`.
75
80
  - `data_database` — where the source tables your metric queries read from live.
76
81
 
77
- For Postgres these are `internal_schema` / `data_schema`; for MySQL they are
78
- `internal_database` / `data_database` (relevant only once those backends land).
82
+ For Postgres these are `internal_schema` / `data_schema` (inside the connected
83
+ `database`); for MySQL they are `internal_database` / `data_database`.
79
84
 
80
85
  ## Step 4 — Profile name & `default_profile`
81
86
 
@@ -8,93 +8,9 @@ from pathlib import Path
8
8
 
9
9
  import click
10
10
 
11
-
12
- def run_init(project_name: str, target_dir: str):
13
- """
14
- Initialize a new detectkit project.
15
-
16
- Args:
17
- project_name: Name of the project (or path - will extract basename)
18
- target_dir: Directory to create project in
19
-
20
- Creates:
21
- project_name/
22
- ├── detectkit_project.yml
23
- ├── profiles.yml
24
- ├── metrics/
25
- │ └── .gitkeep
26
- └── sql/
27
- └── .gitkeep
28
- """
29
- # Extract just the directory name in case user passes a full path
30
- project_name_clean = Path(project_name).name
31
- target_path = Path(target_dir) / project_name_clean
32
-
33
- # Check if project already exists
34
- if target_path.exists():
35
- click.echo(
36
- click.style(
37
- f"Error: Directory '{target_path}' already exists!",
38
- fg="red",
39
- bold=True,
40
- )
41
- )
42
- return
43
-
44
- # Create project directory
45
- click.echo(f"Creating detectkit project '{project_name_clean}' in {target_dir}...")
46
-
47
- target_path.mkdir(parents=True, exist_ok=True)
48
-
49
- # Create subdirectories
50
- (target_path / "metrics").mkdir(exist_ok=True)
51
- (target_path / "sql").mkdir(exist_ok=True)
52
-
53
- # Create .gitkeep files
54
- (target_path / "metrics" / ".gitkeep").touch()
55
- (target_path / "sql" / ".gitkeep").touch()
56
-
57
- # Create detectkit_project.yml
58
- project_config = f"""# detectkit project configuration
59
- name: {project_name_clean}
60
- version: '1.0'
61
-
62
- # Directory paths. These are the real config keys (nested under `paths:`);
63
- # a flat `metrics_path:` / `sql_path:` is not a recognized field and is ignored.
64
- paths:
65
- metrics: metrics
66
- sql: sql
67
- templates: templates
68
-
69
- # Default profile to use (must exist in profiles.yml)
70
- default_profile: dev
71
-
72
- # Default table names (can be overridden in metrics)
73
- tables:
74
- datapoints: _dtk_datapoints
75
- detections: _dtk_detections
76
- tasks: _dtk_tasks
77
-
78
- # Default timeouts (seconds)
79
- timeouts:
80
- load: 1800 # 30 minutes
81
- detect: 3600 # 1 hour
82
- alert: 300 # 5 minutes
83
- """
84
-
85
- (target_path / "detectkit_project.yml").write_text(project_config)
86
-
87
- # Create profiles.yml (must validate against ProfilesConfig: connections
88
- # live under a top-level 'profiles:' mapping). ClickHouse needs BOTH
89
- # `internal_database` (for the _dtk_* tables) and `data_database` (where the
90
- # metric queries read from) — there is no `database:` field, so the dev
91
- # profile below is runnable as-is against a local ClickHouse.
92
- profiles_config = """# Database connection profiles
93
-
94
- default_profile: dev
95
-
96
- profiles:
97
- # Local dev — runnable against a local ClickHouse once the databases exist.
11
+ # Active dev/prod profile blocks per backend (indented under `profiles:`).
12
+ _ACTIVE_PROFILES = {
13
+ "clickhouse": """ # Local dev — runnable against a local ClickHouse once the databases exist.
98
14
  # ClickHouse needs BOTH locations (there is no `database:` field):
99
15
  # internal_database -> where detectkit's own _dtk_* tables live
100
16
  # data_database -> where your metric source tables live
@@ -116,27 +32,106 @@ profiles:
116
32
  password: "{{ env_var('CLICKHOUSE_PASSWORD') }}"
117
33
  internal_database: detectkit # _dtk_* tables
118
34
  data_database: monitoring # your source data
35
+ """,
36
+ "postgres": """ # Local dev — runnable against a local PostgreSQL. PostgreSQL uses SCHEMAS:
37
+ # database -> the database to connect to (must already exist)
38
+ # internal_schema -> schema for detectkit's own _dtk_* tables (auto-created)
39
+ # data_schema -> schema your metric source tables live in
40
+ dev:
41
+ type: postgres
42
+ host: localhost
43
+ port: 5432
44
+ user: postgres
45
+ password: postgres
46
+ database: detectkit
47
+ internal_schema: detectkit
48
+ data_schema: public
49
+
50
+ # Production — keep secrets in env vars, never commit credentials.
51
+ prod:
52
+ type: postgres
53
+ host: "{{ env_var('POSTGRES_HOST') }}"
54
+ port: 5432
55
+ user: "{{ env_var('POSTGRES_USER') }}"
56
+ password: "{{ env_var('POSTGRES_PASSWORD') }}"
57
+ database: "{{ env_var('POSTGRES_DB') }}"
58
+ internal_schema: detectkit
59
+ data_schema: public
60
+ """,
61
+ "mysql": """ # Local dev — runnable against a local MySQL (8.0+). MySQL uses DATABASES:
62
+ # internal_database -> database for detectkit's own _dtk_* tables (auto-created)
63
+ # data_database -> database your metric source tables live in
64
+ dev:
65
+ type: mysql
66
+ host: localhost
67
+ port: 3306
68
+ user: root
69
+ password: ""
70
+ internal_database: detectkit
71
+ data_database: analytics
119
72
 
120
- # Example PostgreSQL profile (uses internal_schema / data_schema)
73
+ # Production keep secrets in env vars, never commit credentials.
74
+ prod:
75
+ type: mysql
76
+ host: "{{ env_var('MYSQL_HOST') }}"
77
+ port: 3306
78
+ user: "{{ env_var('MYSQL_USER') }}"
79
+ password: "{{ env_var('MYSQL_PASSWORD') }}"
80
+ internal_database: detectkit
81
+ data_database: monitoring
82
+ """,
83
+ }
84
+
85
+ # Commented single-profile examples for the backends that are NOT active.
86
+ _COMMENTED_EXAMPLES = {
87
+ "clickhouse": """ # Example ClickHouse profile (needs internal_database + data_database)
88
+ # clickhouse_dev:
89
+ # type: clickhouse
90
+ # host: localhost
91
+ # port: 9000
92
+ # user: default
93
+ # password: ""
94
+ # internal_database: detectkit
95
+ # data_database: default
96
+ """,
97
+ "postgres": """ # Example PostgreSQL profile (connect to `database`; tables live in schemas)
121
98
  # postgres_dev:
122
99
  # type: postgres
123
100
  # host: localhost
124
101
  # port: 5432
125
102
  # user: postgres
126
103
  # password: postgres
104
+ # database: detectkit
127
105
  # internal_schema: detectkit
128
106
  # data_schema: public
129
-
130
- # Example MySQL profile
107
+ """,
108
+ "mysql": """ # Example MySQL profile (8.0+; internal_database + data_database)
131
109
  # mysql_dev:
132
110
  # type: mysql
133
111
  # host: localhost
134
112
  # port: 3306
135
113
  # user: root
136
- # password: root
114
+ # password: ""
137
115
  # internal_database: detectkit
138
116
  # data_database: analytics
139
-
117
+ """,
118
+ }
119
+
120
+ # Timestamp-bucketing expression for the example metric query, per dialect.
121
+ _BUCKET_SQL = {
122
+ "clickhouse": "toStartOfInterval(event_time, INTERVAL {{ interval_seconds }} SECOND)",
123
+ "postgres": (
124
+ "to_timestamp(floor(extract(epoch from event_time) / {{ interval_seconds }})"
125
+ " * {{ interval_seconds }})"
126
+ ),
127
+ "mysql": (
128
+ "FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(event_time) / {{ interval_seconds }})"
129
+ " * {{ interval_seconds }})"
130
+ ),
131
+ }
132
+
133
+ # Alert-channel section (backend-independent); appended after the profiles.
134
+ _ALERT_CHANNELS = """
140
135
  # Alert channels (referenced by name from a metric's alerting.channels)
141
136
  alert_channels:
142
137
  # Mattermost. Supported keys: webhook_url, username, icon_emoji, channel,
@@ -179,9 +174,103 @@ alert_channels:
179
174
  # Authorization: "Bearer {{ env_var('WEBHOOK_TOKEN') }}"
180
175
  """
181
176
 
177
+
178
+ def run_init(project_name: str, target_dir: str, db_type: str = "clickhouse"):
179
+ """
180
+ Initialize a new detectkit project.
181
+
182
+ Args:
183
+ project_name: Name of the project (or path - will extract basename)
184
+ target_dir: Directory to create project in
185
+
186
+ Creates:
187
+ project_name/
188
+ ├── detectkit_project.yml
189
+ ├── profiles.yml
190
+ ├── metrics/
191
+ │ └── .gitkeep
192
+ └── sql/
193
+ └── .gitkeep
194
+ """
195
+ # Extract just the directory name in case user passes a full path
196
+ project_name_clean = Path(project_name).name
197
+ target_path = Path(target_dir) / project_name_clean
198
+
199
+ # Check if project already exists
200
+ if target_path.exists():
201
+ click.echo(
202
+ click.style(
203
+ f"Error: Directory '{target_path}' already exists!",
204
+ fg="red",
205
+ bold=True,
206
+ )
207
+ )
208
+ return
209
+
210
+ # Create project directory
211
+ click.echo(f"Creating detectkit project '{project_name_clean}' in {target_dir}...")
212
+
213
+ target_path.mkdir(parents=True, exist_ok=True)
214
+
215
+ # Create subdirectories
216
+ (target_path / "metrics").mkdir(exist_ok=True)
217
+ (target_path / "sql").mkdir(exist_ok=True)
218
+
219
+ # Create .gitkeep files
220
+ (target_path / "metrics" / ".gitkeep").touch()
221
+ (target_path / "sql" / ".gitkeep").touch()
222
+
223
+ # Create detectkit_project.yml
224
+ project_config = f"""# detectkit project configuration
225
+ name: {project_name_clean}
226
+ version: '1.0'
227
+
228
+ # Directory paths. These are the real config keys (nested under `paths:`);
229
+ # a flat `metrics_path:` / `sql_path:` is not a recognized field and is ignored.
230
+ paths:
231
+ metrics: metrics
232
+ sql: sql
233
+ templates: templates
234
+
235
+ # Default profile to use (must exist in profiles.yml)
236
+ default_profile: dev
237
+
238
+ # Default table names (can be overridden in metrics)
239
+ tables:
240
+ datapoints: _dtk_datapoints
241
+ detections: _dtk_detections
242
+ tasks: _dtk_tasks
243
+
244
+ # Default timeouts (seconds)
245
+ timeouts:
246
+ load: 1800 # 30 minutes
247
+ detect: 3600 # 1 hour
248
+ alert: 300 # 5 minutes
249
+ """
250
+
251
+ (target_path / "detectkit_project.yml").write_text(project_config)
252
+
253
+ # Create profiles.yml (must validate against ProfilesConfig: connections
254
+ # live under a top-level 'profiles:' mapping). The active dev/prod profiles
255
+ # are scaffolded for the chosen --db-type; the other backends are included
256
+ # as commented examples. See the per-database docs for connection details.
257
+ other_backends = [t for t in ("clickhouse", "postgres", "mysql") if t != db_type]
258
+ commented = "\n".join(_COMMENTED_EXAMPLES[t] for t in other_backends)
259
+ profiles_config = (
260
+ "# Database connection profiles\n\n"
261
+ "default_profile: dev\n\n"
262
+ "profiles:\n"
263
+ f"{_ACTIVE_PROFILES[db_type]}\n"
264
+ f"{commented}"
265
+ f"{_ALERT_CHANNELS}"
266
+ )
267
+
182
268
  (target_path / "profiles.yml").write_text(profiles_config)
183
269
 
184
- # Create example metric (must validate against MetricConfig)
270
+ # Create example metric (must validate against MetricConfig). The
271
+ # timestamp-bucketing expression is dialect-specific; it is substituted for
272
+ # the __DTK_BUCKET__ sentinel below so the Jinja `{{ }}` placeholders in the
273
+ # rest of the query are left untouched.
185
274
  example_metric = """# Example metric configuration
186
275
  name: example_cpu_usage
187
276
  description: CPU usage monitoring example
@@ -191,7 +280,7 @@ description: CPU usage monitoring example
191
280
  # {{ interval_seconds }} - metric interval in seconds
192
281
  query: |
193
282
  SELECT
194
- toStartOfInterval(event_time, INTERVAL {{ interval_seconds }} SECOND) AS timestamp,
283
+ __DTK_BUCKET__ AS timestamp,
195
284
  avg(cpu_usage) AS value
196
285
  FROM system_metrics
197
286
  WHERE event_time >= '{{ dtk_start_time }}'
@@ -252,6 +341,7 @@ tags:
252
341
  - system
253
342
  """
254
343
 
344
+ example_metric = example_metric.replace("__DTK_BUCKET__", _BUCKET_SQL[db_type])
255
345
  (target_path / "metrics" / "example_cpu_usage.yml").write_text(example_metric)
256
346
 
257
347
  # Create README
@@ -315,7 +405,7 @@ See https://github.com/alexeiveselov92/detectkit for full documentation.
315
405
  click.echo()
316
406
  click.echo("Next steps:")
317
407
  click.echo(f" 1. cd {project_name}")
318
- click.echo(" 2. Configure database connection in profiles.yml")
408
+ click.echo(f" 2. Configure your {db_type} connection in profiles.yml")
319
409
  click.echo(" 3. Create or edit metric definitions in metrics/")
320
410
  click.echo(" 4. Run: dtk run --select example_cpu_usage")
321
411
  click.echo()
@@ -36,23 +36,31 @@ def cli():
36
36
  default=".",
37
37
  help="Directory to create project in (default: current directory)",
38
38
  )
39
- def init(project_name: str, target_dir: str):
39
+ @click.option(
40
+ "--db-type",
41
+ type=click.Choice(["clickhouse", "postgres", "mysql"]),
42
+ default="clickhouse",
43
+ show_default=True,
44
+ help="Database backend to scaffold the dev/prod profiles and example query for.",
45
+ )
46
+ def init(project_name: str, target_dir: str, db_type: str):
40
47
  """
41
48
  Initialize a new detectkit project.
42
49
 
43
50
  Creates project structure with configuration files and directories:
44
51
  - detectkit_project.yml (project config)
45
- - profiles.yml (database connections)
52
+ - profiles.yml (database connections — for the chosen --db-type)
46
53
  - metrics/ (metric definitions)
47
54
  - sql/ (SQL queries)
48
55
 
49
56
  Example:
50
57
  dtk init my_monitoring_project
51
58
  dtk init analytics --target-dir /opt/projects
59
+ dtk init my_project --db-type postgres
52
60
  """
53
61
  from detectkit.cli.commands.init import run_init
54
62
 
55
- run_init(project_name, target_dir)
63
+ run_init(project_name, target_dir, db_type=db_type)
56
64
 
57
65
 
58
66
  @cli.command(name="init-claude")
@@ -41,6 +41,13 @@ class ProfileConfig(BaseModel):
41
41
  user: str = Field(default="default", description="Database user")
42
42
  password: str = Field(default="", description="Database password")
43
43
 
44
+ # Connection-target database. Required for PostgreSQL (the database to
45
+ # connect to, inside which internal_schema/data_schema live); optional for
46
+ # MySQL; unused for ClickHouse.
47
+ database: str | None = Field(
48
+ default=None, description="Database to connect to (PostgreSQL/MySQL)"
49
+ )
50
+
44
51
  # Internal location for _dtk_* tables
45
52
  internal_database: str | None = Field(
46
53
  default=None, description="Database for internal tables (ClickHouse/MySQL)"
@@ -136,7 +143,9 @@ class ProfileConfig(BaseModel):
136
143
  Database manager instance
137
144
 
138
145
  Raises:
139
- NotImplementedError: If database type not yet implemented
146
+ ValueError: If the database type is unsupported, or required
147
+ connection fields (e.g. PostgreSQL ``database``) are missing
148
+ ImportError: If the backend's driver is not installed
140
149
  """
141
150
  if self.type == "clickhouse":
142
151
  return ClickHouseDatabaseManager(
@@ -149,9 +158,36 @@ class ProfileConfig(BaseModel):
149
158
  settings=self.settings,
150
159
  )
151
160
  elif self.type == "postgres":
152
- raise NotImplementedError("PostgreSQL support coming soon")
161
+ from detectkit.database.postgres_manager import PostgresDatabaseManager
162
+
163
+ if not self.database:
164
+ raise ValueError(
165
+ "PostgreSQL profiles must set 'database' (the database to "
166
+ "connect to, inside which internal_schema/data_schema live)"
167
+ )
168
+ return PostgresDatabaseManager(
169
+ host=self.host,
170
+ port=self.port,
171
+ user=self.user,
172
+ password=self.password,
173
+ database=self.database,
174
+ internal_schema=self.get_internal_location(),
175
+ data_schema=self.get_data_location(),
176
+ settings=self.settings,
177
+ )
153
178
  elif self.type == "mysql":
154
- raise NotImplementedError("MySQL support coming soon")
179
+ from detectkit.database.mysql_manager import MySQLDatabaseManager
180
+
181
+ return MySQLDatabaseManager(
182
+ host=self.host,
183
+ port=self.port,
184
+ user=self.user,
185
+ password=self.password,
186
+ database=self.database,
187
+ internal_database=self.get_internal_location(),
188
+ data_database=self.get_data_location(),
189
+ settings=self.settings,
190
+ )
155
191
  else:
156
192
  raise ValueError(f"Unsupported database type: {self.type}")
157
193
 
@@ -47,6 +47,12 @@ class TableModel:
47
47
  engine: Database engine (ClickHouse-specific, e.g., "MergeTree")
48
48
  order_by: Columns for ORDER BY clause (ClickHouse-specific)
49
49
  indexes: Additional indexes to create
50
+ version_column: Column that drives last-writer-wins deduplication.
51
+ On ClickHouse this is the version encoded in the engine string
52
+ (e.g. ``ReplacingMergeTree(created_at)``); on SQL backends with an
53
+ enforced primary key it drives a version-aware upsert so a re-insert
54
+ with a newer ``version_column`` replaces the existing row. ``None``
55
+ for tables that do not deduplicate by version (e.g. ``_dtk_tasks``).
50
56
 
51
57
  Example:
52
58
  >>> model = TableModel(
@@ -65,6 +71,7 @@ class TableModel:
65
71
  engine: str | None = None
66
72
  order_by: list[str] | None = None
67
73
  indexes: list[str] = field(default_factory=list)
74
+ version_column: str | None = None
68
75
 
69
76
  def __post_init__(self):
70
77
  """Validate table model."""
@@ -86,6 +93,10 @@ class TableModel:
86
93
  if order_col not in column_names:
87
94
  raise ValueError(f"ORDER BY column '{order_col}' not found in table columns")
88
95
 
96
+ # Validate version column exists (if specified)
97
+ if self.version_column and self.version_column not in column_names:
98
+ raise ValueError(f"Version column '{self.version_column}' not found in table columns")
99
+
89
100
  def get_column(self, name: str) -> ColumnDefinition | None:
90
101
  """
91
102
  Get column definition by name.
@@ -1,8 +1,11 @@
1
1
  """Database managers for detectk."""
2
2
 
3
+ from detectkit.database._sql_manager import SQLDatabaseManager
3
4
  from detectkit.database.clickhouse_manager import ClickHouseDatabaseManager
4
5
  from detectkit.database.internal_tables import InternalTablesManager
5
6
  from detectkit.database.manager import BaseDatabaseManager
7
+ from detectkit.database.mysql_manager import MySQLDatabaseManager
8
+ from detectkit.database.postgres_manager import PostgresDatabaseManager
6
9
  from detectkit.database.tables import (
7
10
  INTERNAL_TABLES,
8
11
  TABLE_DATAPOINTS,
@@ -16,6 +19,9 @@ from detectkit.database.tables import (
16
19
  __all__ = [
17
20
  "BaseDatabaseManager",
18
21
  "ClickHouseDatabaseManager",
22
+ "SQLDatabaseManager",
23
+ "PostgresDatabaseManager",
24
+ "MySQLDatabaseManager",
19
25
  "InternalTablesManager",
20
26
  "TABLE_DATAPOINTS",
21
27
  "TABLE_DETECTIONS",