detectkit 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {detectkit-0.10.0/detectkit.egg-info → detectkit-0.11.0}/PKG-INFO +4 -2
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/__init__.py +1 -1
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/overview.md +3 -3
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/project.md +18 -20
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md +13 -8
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/init.py +185 -95
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/main.py +11 -3
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/profile.py +39 -3
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/models.py +11 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/__init__.py +6 -0
- detectkit-0.11.0/detectkit/database/_sql_manager.py +398 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/clickhouse_manager.py +38 -16
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_alert_states.py +14 -29
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_datapoints.py +6 -5
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_detections.py +9 -11
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_maintenance.py +5 -7
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_schema.py +5 -1
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/manager.py +73 -0
- detectkit-0.11.0/detectkit/database/mysql_manager.py +132 -0
- detectkit-0.11.0/detectkit/database/postgres_manager.py +118 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/tables.py +3 -0
- {detectkit-0.10.0 → detectkit-0.11.0/detectkit.egg-info}/PKG-INFO +4 -2
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/SOURCES.txt +3 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/requires.txt +3 -1
- {detectkit-0.10.0 → detectkit-0.11.0}/pyproject.toml +5 -1
- {detectkit-0.10.0 → detectkit-0.11.0}/LICENSE +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/MANIFEST.in +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/README.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/base.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/email.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/factory.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/mattermost.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/slack.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/telegram.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/channels/webhook.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_base.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_decision.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_recovery.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/_types.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/_output.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/CLAUDE.section.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/alerting.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/cli.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/detectors.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/rules/metrics.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/skills/dtk-new-metric/SKILL.md +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/clean.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/init_claude.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/run.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/test_alert.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/commands/unlock.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/metric_config.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/project_config.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/config/validator.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/core/interval.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_base.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_metrics.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/_tasks.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/database/internal_tables/manager.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/base.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/factory.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/seasonality.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/_windowed.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/iqr.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/mad.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/manual_bounds.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/detectors/statistical/zscore.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/metric_loader.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/loaders/query_template.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/error_dispatch.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_base.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_load_step.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/_types.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/orchestration/task_manager/manager.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/__init__.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/datetime_utils.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/env_interpolation.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/json_utils.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit/utils/stats.py +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/dependency_links.txt +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/entry_points.txt +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/detectkit.egg-info/top_level.txt +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/requirements.txt +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/setup.cfg +0 -0
- {detectkit-0.10.0 → detectkit-0.11.0}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: detectkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Metric monitoring with automatic anomaly detection
|
|
5
5
|
Author: detectkit team
|
|
6
6
|
License: MIT
|
|
@@ -61,7 +61,9 @@ Requires-Dist: black>=23.0; extra == "dev"
|
|
|
61
61
|
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
62
62
|
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
63
63
|
Provides-Extra: integration
|
|
64
|
-
Requires-Dist: testcontainers[clickhouse]>=4.0; extra == "integration"
|
|
64
|
+
Requires-Dist: testcontainers[clickhouse,mysql,postgres]>=4.0; extra == "integration"
|
|
65
|
+
Requires-Dist: psycopg2-binary>=2.9.0; extra == "integration"
|
|
66
|
+
Requires-Dist: pymysql>=1.0.0; extra == "integration"
|
|
65
67
|
Dynamic: license-file
|
|
66
68
|
|
|
67
69
|
# detectkit
|
|
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
|
|
|
4
4
|
A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "0.
|
|
7
|
+
__version__ = "0.11.0"
|
|
8
8
|
|
|
9
9
|
from detectkit.core.interval import Interval
|
|
10
10
|
from detectkit.core.models import ColumnDefinition, TableModel
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
detectkit is a Python library and CLI (`dtk`) for monitoring time-series
|
|
4
4
|
metrics with automatic anomaly detection and multi-channel alerting. It is
|
|
5
5
|
**dbt-like**: metrics live as YAML + SQL in a project directory, and you run
|
|
6
|
-
them with one command. Core logic is pure numpy (no pandas). **ClickHouse
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
them with one command. Core logic is pure numpy (no pandas). **ClickHouse,
|
|
7
|
+
PostgreSQL and MySQL are all fully supported** — only the connection and the SQL
|
|
8
|
+
dialect of your metric queries differ between them.
|
|
9
9
|
|
|
10
10
|
## The pipeline: load → detect → alert
|
|
11
11
|
|
|
@@ -78,11 +78,11 @@ alert_channels:
|
|
|
78
78
|
|
|
79
79
|
### Database profiles
|
|
80
80
|
|
|
81
|
-
> ClickHouse
|
|
82
|
-
>
|
|
83
|
-
> `
|
|
81
|
+
> ClickHouse, PostgreSQL and MySQL are all fully supported. ClickHouse/MySQL use
|
|
82
|
+
> two *databases*; PostgreSQL connects to one `database` and uses two *schemas*.
|
|
83
|
+
> `dtk init --db-type {clickhouse,postgres,mysql}` scaffolds the right shape.
|
|
84
84
|
|
|
85
|
-
**ClickHouse
|
|
85
|
+
**ClickHouse**:
|
|
86
86
|
```yaml
|
|
87
87
|
profiles:
|
|
88
88
|
prod:
|
|
@@ -98,36 +98,34 @@ profiles:
|
|
|
98
98
|
max_memory_usage: 10000000000
|
|
99
99
|
```
|
|
100
100
|
|
|
101
|
-
**PostgreSQL** (
|
|
101
|
+
**PostgreSQL** (connect to `database`, tables in schemas):
|
|
102
102
|
```yaml
|
|
103
103
|
profiles:
|
|
104
104
|
prod:
|
|
105
105
|
type: postgres
|
|
106
106
|
host: localhost
|
|
107
107
|
port: 5432
|
|
108
|
-
database:
|
|
108
|
+
database: detectkit # required — must already exist
|
|
109
109
|
user: postgres
|
|
110
110
|
password: "..."
|
|
111
|
-
internal_schema: detectkit # required — _dtk_* tables
|
|
111
|
+
internal_schema: detectkit # required — _dtk_* tables (auto-created)
|
|
112
112
|
data_schema: public # required — data queries
|
|
113
|
-
|
|
114
|
-
max_overflow: 10 # optional
|
|
113
|
+
settings: {} # optional — extra psycopg2.connect kwargs
|
|
115
114
|
```
|
|
116
115
|
|
|
117
|
-
**MySQL** (
|
|
116
|
+
**MySQL** (8.0+; two databases):
|
|
118
117
|
```yaml
|
|
119
118
|
profiles:
|
|
120
119
|
prod:
|
|
121
120
|
type: mysql
|
|
122
121
|
host: localhost
|
|
123
122
|
port: 3306
|
|
124
|
-
database: analytics # required
|
|
125
123
|
user: root
|
|
126
124
|
password: "..."
|
|
127
|
-
internal_database: detectkit # required
|
|
125
|
+
internal_database: detectkit # required — _dtk_* tables (auto-created)
|
|
128
126
|
data_database: analytics # required
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
database: analytics # optional — default db for the connection
|
|
128
|
+
settings: {} # optional — extra pymysql.connect kwargs
|
|
131
129
|
```
|
|
132
130
|
|
|
133
131
|
### Alert channels
|
|
@@ -185,12 +183,12 @@ alert_channels:
|
|
|
185
183
|
## Notes
|
|
186
184
|
|
|
187
185
|
- **First-run setup:** the `profiles.yml` that `dtk init` writes is a
|
|
188
|
-
placeholder
|
|
189
|
-
at example values
|
|
190
|
-
credentials and
|
|
191
|
-
(the **`dtk-setup-project`** skill walks this).
|
|
192
|
-
|
|
193
|
-
|
|
186
|
+
placeholder scaffolded for `--db-type` (default ClickHouse) — its `dev`
|
|
187
|
+
profile points the location fields at example values on `localhost`. Edit the
|
|
188
|
+
host, credentials and location names to match your environment before running
|
|
189
|
+
(the **`dtk-setup-project`** skill walks this). ClickHouse/MySQL use
|
|
190
|
+
`internal_database` / `data_database` (no `database:` field on ClickHouse);
|
|
191
|
+
PostgreSQL connects to a `database` and uses `internal_schema` / `data_schema`.
|
|
194
192
|
- `dtk run` (without `--profile`) uses the `default_profile` declared in
|
|
195
193
|
**`profiles.yml`**; the `default_profile` in `detectkit_project.yml` is not
|
|
196
194
|
read at runtime — keep them in sync to avoid confusion.
|
{detectkit-0.10.0 → detectkit-0.11.0}/detectkit/cli/assets/claude/skills/dtk-setup-project/SKILL.md
RENAMED
|
@@ -40,12 +40,17 @@ side by side, ask which one to set up.
|
|
|
40
40
|
|
|
41
41
|
## Step 1 — Pick the database backend
|
|
42
42
|
|
|
43
|
-
**ClickHouse
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
**ClickHouse, PostgreSQL and MySQL are all fully supported.** Ask which one the
|
|
44
|
+
project uses (default to ClickHouse if unsure). `dtk init --db-type
|
|
45
|
+
{clickhouse,postgres,mysql}` scaffolds `profiles.yml` for the chosen backend.
|
|
46
|
+
The location fields differ:
|
|
47
|
+
- **ClickHouse** / **MySQL** — two *databases*: `internal_database` / `data_database`.
|
|
48
|
+
- **PostgreSQL** — connect to a `database` (must already exist), then two
|
|
49
|
+
*schemas*: `internal_schema` / `data_schema`.
|
|
50
|
+
|
|
51
|
+
The metric query SQL dialect also differs (e.g. `toStartOfInterval` on
|
|
52
|
+
ClickHouse vs `date_trunc`/`to_timestamp` on Postgres vs `FROM_UNIXTIME` on
|
|
53
|
+
MySQL). Everything else — detectors, alerting, the CLI — is identical.
|
|
49
54
|
|
|
50
55
|
## Step 2 — Connection details (gather, don't guess)
|
|
51
56
|
|
|
@@ -74,8 +79,8 @@ values):
|
|
|
74
79
|
data, e.g. `detectkit` or `monitoring`.
|
|
75
80
|
- `data_database` — where the source tables your metric queries read from live.
|
|
76
81
|
|
|
77
|
-
For Postgres these are `internal_schema` / `data_schema
|
|
78
|
-
`
|
|
82
|
+
For Postgres these are `internal_schema` / `data_schema` (inside the connected
|
|
83
|
+
`database`); for MySQL they are `internal_database` / `data_database`.
|
|
79
84
|
|
|
80
85
|
## Step 4 — Profile name & `default_profile`
|
|
81
86
|
|
|
@@ -8,93 +8,9 @@ from pathlib import Path
|
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"""
|
|
14
|
-
Initialize a new detectkit project.
|
|
15
|
-
|
|
16
|
-
Args:
|
|
17
|
-
project_name: Name of the project (or path - will extract basename)
|
|
18
|
-
target_dir: Directory to create project in
|
|
19
|
-
|
|
20
|
-
Creates:
|
|
21
|
-
project_name/
|
|
22
|
-
├── detectkit_project.yml
|
|
23
|
-
├── profiles.yml
|
|
24
|
-
├── metrics/
|
|
25
|
-
│ └── .gitkeep
|
|
26
|
-
└── sql/
|
|
27
|
-
└── .gitkeep
|
|
28
|
-
"""
|
|
29
|
-
# Extract just the directory name in case user passes a full path
|
|
30
|
-
project_name_clean = Path(project_name).name
|
|
31
|
-
target_path = Path(target_dir) / project_name_clean
|
|
32
|
-
|
|
33
|
-
# Check if project already exists
|
|
34
|
-
if target_path.exists():
|
|
35
|
-
click.echo(
|
|
36
|
-
click.style(
|
|
37
|
-
f"Error: Directory '{target_path}' already exists!",
|
|
38
|
-
fg="red",
|
|
39
|
-
bold=True,
|
|
40
|
-
)
|
|
41
|
-
)
|
|
42
|
-
return
|
|
43
|
-
|
|
44
|
-
# Create project directory
|
|
45
|
-
click.echo(f"Creating detectkit project '{project_name_clean}' in {target_dir}...")
|
|
46
|
-
|
|
47
|
-
target_path.mkdir(parents=True, exist_ok=True)
|
|
48
|
-
|
|
49
|
-
# Create subdirectories
|
|
50
|
-
(target_path / "metrics").mkdir(exist_ok=True)
|
|
51
|
-
(target_path / "sql").mkdir(exist_ok=True)
|
|
52
|
-
|
|
53
|
-
# Create .gitkeep files
|
|
54
|
-
(target_path / "metrics" / ".gitkeep").touch()
|
|
55
|
-
(target_path / "sql" / ".gitkeep").touch()
|
|
56
|
-
|
|
57
|
-
# Create detectkit_project.yml
|
|
58
|
-
project_config = f"""# detectkit project configuration
|
|
59
|
-
name: {project_name_clean}
|
|
60
|
-
version: '1.0'
|
|
61
|
-
|
|
62
|
-
# Directory paths. These are the real config keys (nested under `paths:`);
|
|
63
|
-
# a flat `metrics_path:` / `sql_path:` is not a recognized field and is ignored.
|
|
64
|
-
paths:
|
|
65
|
-
metrics: metrics
|
|
66
|
-
sql: sql
|
|
67
|
-
templates: templates
|
|
68
|
-
|
|
69
|
-
# Default profile to use (must exist in profiles.yml)
|
|
70
|
-
default_profile: dev
|
|
71
|
-
|
|
72
|
-
# Default table names (can be overridden in metrics)
|
|
73
|
-
tables:
|
|
74
|
-
datapoints: _dtk_datapoints
|
|
75
|
-
detections: _dtk_detections
|
|
76
|
-
tasks: _dtk_tasks
|
|
77
|
-
|
|
78
|
-
# Default timeouts (seconds)
|
|
79
|
-
timeouts:
|
|
80
|
-
load: 1800 # 30 minutes
|
|
81
|
-
detect: 3600 # 1 hour
|
|
82
|
-
alert: 300 # 5 minutes
|
|
83
|
-
"""
|
|
84
|
-
|
|
85
|
-
(target_path / "detectkit_project.yml").write_text(project_config)
|
|
86
|
-
|
|
87
|
-
# Create profiles.yml (must validate against ProfilesConfig: connections
|
|
88
|
-
# live under a top-level 'profiles:' mapping). ClickHouse needs BOTH
|
|
89
|
-
# `internal_database` (for the _dtk_* tables) and `data_database` (where the
|
|
90
|
-
# metric queries read from) — there is no `database:` field, so the dev
|
|
91
|
-
# profile below is runnable as-is against a local ClickHouse.
|
|
92
|
-
profiles_config = """# Database connection profiles
|
|
93
|
-
|
|
94
|
-
default_profile: dev
|
|
95
|
-
|
|
96
|
-
profiles:
|
|
97
|
-
# Local dev — runnable against a local ClickHouse once the databases exist.
|
|
11
|
+
# Active dev/prod profile blocks per backend (indented under `profiles:`).
|
|
12
|
+
_ACTIVE_PROFILES = {
|
|
13
|
+
"clickhouse": """ # Local dev — runnable against a local ClickHouse once the databases exist.
|
|
98
14
|
# ClickHouse needs BOTH locations (there is no `database:` field):
|
|
99
15
|
# internal_database -> where detectkit's own _dtk_* tables live
|
|
100
16
|
# data_database -> where your metric source tables live
|
|
@@ -116,27 +32,106 @@ profiles:
|
|
|
116
32
|
password: "{{ env_var('CLICKHOUSE_PASSWORD') }}"
|
|
117
33
|
internal_database: detectkit # _dtk_* tables
|
|
118
34
|
data_database: monitoring # your source data
|
|
35
|
+
""",
|
|
36
|
+
"postgres": """ # Local dev — runnable against a local PostgreSQL. PostgreSQL uses SCHEMAS:
|
|
37
|
+
# database -> the database to connect to (must already exist)
|
|
38
|
+
# internal_schema -> schema for detectkit's own _dtk_* tables (auto-created)
|
|
39
|
+
# data_schema -> schema your metric source tables live in
|
|
40
|
+
dev:
|
|
41
|
+
type: postgres
|
|
42
|
+
host: localhost
|
|
43
|
+
port: 5432
|
|
44
|
+
user: postgres
|
|
45
|
+
password: postgres
|
|
46
|
+
database: detectkit
|
|
47
|
+
internal_schema: detectkit
|
|
48
|
+
data_schema: public
|
|
49
|
+
|
|
50
|
+
# Production — keep secrets in env vars, never commit credentials.
|
|
51
|
+
prod:
|
|
52
|
+
type: postgres
|
|
53
|
+
host: "{{ env_var('POSTGRES_HOST') }}"
|
|
54
|
+
port: 5432
|
|
55
|
+
user: "{{ env_var('POSTGRES_USER') }}"
|
|
56
|
+
password: "{{ env_var('POSTGRES_PASSWORD') }}"
|
|
57
|
+
database: "{{ env_var('POSTGRES_DB') }}"
|
|
58
|
+
internal_schema: detectkit
|
|
59
|
+
data_schema: public
|
|
60
|
+
""",
|
|
61
|
+
"mysql": """ # Local dev — runnable against a local MySQL (8.0+). MySQL uses DATABASES:
|
|
62
|
+
# internal_database -> database for detectkit's own _dtk_* tables (auto-created)
|
|
63
|
+
# data_database -> database your metric source tables live in
|
|
64
|
+
dev:
|
|
65
|
+
type: mysql
|
|
66
|
+
host: localhost
|
|
67
|
+
port: 3306
|
|
68
|
+
user: root
|
|
69
|
+
password: ""
|
|
70
|
+
internal_database: detectkit
|
|
71
|
+
data_database: analytics
|
|
119
72
|
|
|
120
|
-
#
|
|
73
|
+
# Production — keep secrets in env vars, never commit credentials.
|
|
74
|
+
prod:
|
|
75
|
+
type: mysql
|
|
76
|
+
host: "{{ env_var('MYSQL_HOST') }}"
|
|
77
|
+
port: 3306
|
|
78
|
+
user: "{{ env_var('MYSQL_USER') }}"
|
|
79
|
+
password: "{{ env_var('MYSQL_PASSWORD') }}"
|
|
80
|
+
internal_database: detectkit
|
|
81
|
+
data_database: monitoring
|
|
82
|
+
""",
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
# Commented single-profile examples for the backends that are NOT active.
|
|
86
|
+
_COMMENTED_EXAMPLES = {
|
|
87
|
+
"clickhouse": """ # Example ClickHouse profile (needs internal_database + data_database)
|
|
88
|
+
# clickhouse_dev:
|
|
89
|
+
# type: clickhouse
|
|
90
|
+
# host: localhost
|
|
91
|
+
# port: 9000
|
|
92
|
+
# user: default
|
|
93
|
+
# password: ""
|
|
94
|
+
# internal_database: detectkit
|
|
95
|
+
# data_database: default
|
|
96
|
+
""",
|
|
97
|
+
"postgres": """ # Example PostgreSQL profile (connect to `database`; tables live in schemas)
|
|
121
98
|
# postgres_dev:
|
|
122
99
|
# type: postgres
|
|
123
100
|
# host: localhost
|
|
124
101
|
# port: 5432
|
|
125
102
|
# user: postgres
|
|
126
103
|
# password: postgres
|
|
104
|
+
# database: detectkit
|
|
127
105
|
# internal_schema: detectkit
|
|
128
106
|
# data_schema: public
|
|
129
|
-
|
|
130
|
-
# Example MySQL profile
|
|
107
|
+
""",
|
|
108
|
+
"mysql": """ # Example MySQL profile (8.0+; internal_database + data_database)
|
|
131
109
|
# mysql_dev:
|
|
132
110
|
# type: mysql
|
|
133
111
|
# host: localhost
|
|
134
112
|
# port: 3306
|
|
135
113
|
# user: root
|
|
136
|
-
# password:
|
|
114
|
+
# password: ""
|
|
137
115
|
# internal_database: detectkit
|
|
138
116
|
# data_database: analytics
|
|
139
|
-
|
|
117
|
+
""",
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
# Timestamp-bucketing expression for the example metric query, per dialect.
|
|
121
|
+
_BUCKET_SQL = {
|
|
122
|
+
"clickhouse": "toStartOfInterval(event_time, INTERVAL {{ interval_seconds }} SECOND)",
|
|
123
|
+
"postgres": (
|
|
124
|
+
"to_timestamp(floor(extract(epoch from event_time) / {{ interval_seconds }})"
|
|
125
|
+
" * {{ interval_seconds }})"
|
|
126
|
+
),
|
|
127
|
+
"mysql": (
|
|
128
|
+
"FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(event_time) / {{ interval_seconds }})"
|
|
129
|
+
" * {{ interval_seconds }})"
|
|
130
|
+
),
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
# Alert-channel section (backend-independent); appended after the profiles.
|
|
134
|
+
_ALERT_CHANNELS = """
|
|
140
135
|
# Alert channels (referenced by name from a metric's alerting.channels)
|
|
141
136
|
alert_channels:
|
|
142
137
|
# Mattermost. Supported keys: webhook_url, username, icon_emoji, channel,
|
|
@@ -179,9 +174,103 @@ alert_channels:
|
|
|
179
174
|
# Authorization: "Bearer {{ env_var('WEBHOOK_TOKEN') }}"
|
|
180
175
|
"""
|
|
181
176
|
|
|
177
|
+
|
|
178
|
+
def run_init(project_name: str, target_dir: str, db_type: str = "clickhouse"):
|
|
179
|
+
"""
|
|
180
|
+
Initialize a new detectkit project.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
project_name: Name of the project (or path - will extract basename)
|
|
184
|
+
target_dir: Directory to create project in
|
|
185
|
+
|
|
186
|
+
Creates:
|
|
187
|
+
project_name/
|
|
188
|
+
├── detectkit_project.yml
|
|
189
|
+
├── profiles.yml
|
|
190
|
+
├── metrics/
|
|
191
|
+
│ └── .gitkeep
|
|
192
|
+
└── sql/
|
|
193
|
+
└── .gitkeep
|
|
194
|
+
"""
|
|
195
|
+
# Extract just the directory name in case user passes a full path
|
|
196
|
+
project_name_clean = Path(project_name).name
|
|
197
|
+
target_path = Path(target_dir) / project_name_clean
|
|
198
|
+
|
|
199
|
+
# Check if project already exists
|
|
200
|
+
if target_path.exists():
|
|
201
|
+
click.echo(
|
|
202
|
+
click.style(
|
|
203
|
+
f"Error: Directory '{target_path}' already exists!",
|
|
204
|
+
fg="red",
|
|
205
|
+
bold=True,
|
|
206
|
+
)
|
|
207
|
+
)
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
# Create project directory
|
|
211
|
+
click.echo(f"Creating detectkit project '{project_name_clean}' in {target_dir}...")
|
|
212
|
+
|
|
213
|
+
target_path.mkdir(parents=True, exist_ok=True)
|
|
214
|
+
|
|
215
|
+
# Create subdirectories
|
|
216
|
+
(target_path / "metrics").mkdir(exist_ok=True)
|
|
217
|
+
(target_path / "sql").mkdir(exist_ok=True)
|
|
218
|
+
|
|
219
|
+
# Create .gitkeep files
|
|
220
|
+
(target_path / "metrics" / ".gitkeep").touch()
|
|
221
|
+
(target_path / "sql" / ".gitkeep").touch()
|
|
222
|
+
|
|
223
|
+
# Create detectkit_project.yml
|
|
224
|
+
project_config = f"""# detectkit project configuration
|
|
225
|
+
name: {project_name_clean}
|
|
226
|
+
version: '1.0'
|
|
227
|
+
|
|
228
|
+
# Directory paths. These are the real config keys (nested under `paths:`);
|
|
229
|
+
# a flat `metrics_path:` / `sql_path:` is not a recognized field and is ignored.
|
|
230
|
+
paths:
|
|
231
|
+
metrics: metrics
|
|
232
|
+
sql: sql
|
|
233
|
+
templates: templates
|
|
234
|
+
|
|
235
|
+
# Default profile to use (must exist in profiles.yml)
|
|
236
|
+
default_profile: dev
|
|
237
|
+
|
|
238
|
+
# Default table names (can be overridden in metrics)
|
|
239
|
+
tables:
|
|
240
|
+
datapoints: _dtk_datapoints
|
|
241
|
+
detections: _dtk_detections
|
|
242
|
+
tasks: _dtk_tasks
|
|
243
|
+
|
|
244
|
+
# Default timeouts (seconds)
|
|
245
|
+
timeouts:
|
|
246
|
+
load: 1800 # 30 minutes
|
|
247
|
+
detect: 3600 # 1 hour
|
|
248
|
+
alert: 300 # 5 minutes
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
(target_path / "detectkit_project.yml").write_text(project_config)
|
|
252
|
+
|
|
253
|
+
# Create profiles.yml (must validate against ProfilesConfig: connections
|
|
254
|
+
# live under a top-level 'profiles:' mapping). The active dev/prod profiles
|
|
255
|
+
# are scaffolded for the chosen --db-type; the other backends are included
|
|
256
|
+
# as commented examples. See the per-database docs for connection details.
|
|
257
|
+
other_backends = [t for t in ("clickhouse", "postgres", "mysql") if t != db_type]
|
|
258
|
+
commented = "\n".join(_COMMENTED_EXAMPLES[t] for t in other_backends)
|
|
259
|
+
profiles_config = (
|
|
260
|
+
"# Database connection profiles\n\n"
|
|
261
|
+
"default_profile: dev\n\n"
|
|
262
|
+
"profiles:\n"
|
|
263
|
+
f"{_ACTIVE_PROFILES[db_type]}\n"
|
|
264
|
+
f"{commented}"
|
|
265
|
+
f"{_ALERT_CHANNELS}"
|
|
266
|
+
)
|
|
267
|
+
|
|
182
268
|
(target_path / "profiles.yml").write_text(profiles_config)
|
|
183
269
|
|
|
184
|
-
# Create example metric (must validate against MetricConfig)
|
|
270
|
+
# Create example metric (must validate against MetricConfig). The
|
|
271
|
+
# timestamp-bucketing expression is dialect-specific; it is substituted for
|
|
272
|
+
# the __DTK_BUCKET__ sentinel below so the Jinja `{{ }}` placeholders in the
|
|
273
|
+
# rest of the query are left untouched.
|
|
185
274
|
example_metric = """# Example metric configuration
|
|
186
275
|
name: example_cpu_usage
|
|
187
276
|
description: CPU usage monitoring example
|
|
@@ -191,7 +280,7 @@ description: CPU usage monitoring example
|
|
|
191
280
|
# {{ interval_seconds }} - metric interval in seconds
|
|
192
281
|
query: |
|
|
193
282
|
SELECT
|
|
194
|
-
|
|
283
|
+
__DTK_BUCKET__ AS timestamp,
|
|
195
284
|
avg(cpu_usage) AS value
|
|
196
285
|
FROM system_metrics
|
|
197
286
|
WHERE event_time >= '{{ dtk_start_time }}'
|
|
@@ -252,6 +341,7 @@ tags:
|
|
|
252
341
|
- system
|
|
253
342
|
"""
|
|
254
343
|
|
|
344
|
+
example_metric = example_metric.replace("__DTK_BUCKET__", _BUCKET_SQL[db_type])
|
|
255
345
|
(target_path / "metrics" / "example_cpu_usage.yml").write_text(example_metric)
|
|
256
346
|
|
|
257
347
|
# Create README
|
|
@@ -315,7 +405,7 @@ See https://github.com/alexeiveselov92/detectkit for full documentation.
|
|
|
315
405
|
click.echo()
|
|
316
406
|
click.echo("Next steps:")
|
|
317
407
|
click.echo(f" 1. cd {project_name}")
|
|
318
|
-
click.echo(" 2. Configure
|
|
408
|
+
click.echo(f" 2. Configure your {db_type} connection in profiles.yml")
|
|
319
409
|
click.echo(" 3. Create or edit metric definitions in metrics/")
|
|
320
410
|
click.echo(" 4. Run: dtk run --select example_cpu_usage")
|
|
321
411
|
click.echo()
|
|
@@ -36,23 +36,31 @@ def cli():
|
|
|
36
36
|
default=".",
|
|
37
37
|
help="Directory to create project in (default: current directory)",
|
|
38
38
|
)
|
|
39
|
-
|
|
39
|
+
@click.option(
|
|
40
|
+
"--db-type",
|
|
41
|
+
type=click.Choice(["clickhouse", "postgres", "mysql"]),
|
|
42
|
+
default="clickhouse",
|
|
43
|
+
show_default=True,
|
|
44
|
+
help="Database backend to scaffold the dev/prod profiles and example query for.",
|
|
45
|
+
)
|
|
46
|
+
def init(project_name: str, target_dir: str, db_type: str):
|
|
40
47
|
"""
|
|
41
48
|
Initialize a new detectkit project.
|
|
42
49
|
|
|
43
50
|
Creates project structure with configuration files and directories:
|
|
44
51
|
- detectkit_project.yml (project config)
|
|
45
|
-
- profiles.yml (database connections)
|
|
52
|
+
- profiles.yml (database connections — for the chosen --db-type)
|
|
46
53
|
- metrics/ (metric definitions)
|
|
47
54
|
- sql/ (SQL queries)
|
|
48
55
|
|
|
49
56
|
Example:
|
|
50
57
|
dtk init my_monitoring_project
|
|
51
58
|
dtk init analytics --target-dir /opt/projects
|
|
59
|
+
dtk init my_project --db-type postgres
|
|
52
60
|
"""
|
|
53
61
|
from detectkit.cli.commands.init import run_init
|
|
54
62
|
|
|
55
|
-
run_init(project_name, target_dir)
|
|
63
|
+
run_init(project_name, target_dir, db_type=db_type)
|
|
56
64
|
|
|
57
65
|
|
|
58
66
|
@cli.command(name="init-claude")
|
|
@@ -41,6 +41,13 @@ class ProfileConfig(BaseModel):
|
|
|
41
41
|
user: str = Field(default="default", description="Database user")
|
|
42
42
|
password: str = Field(default="", description="Database password")
|
|
43
43
|
|
|
44
|
+
# Connection-target database. Required for PostgreSQL (the database to
|
|
45
|
+
# connect to, inside which internal_schema/data_schema live); optional for
|
|
46
|
+
# MySQL; unused for ClickHouse.
|
|
47
|
+
database: str | None = Field(
|
|
48
|
+
default=None, description="Database to connect to (PostgreSQL/MySQL)"
|
|
49
|
+
)
|
|
50
|
+
|
|
44
51
|
# Internal location for _dtk_* tables
|
|
45
52
|
internal_database: str | None = Field(
|
|
46
53
|
default=None, description="Database for internal tables (ClickHouse/MySQL)"
|
|
@@ -136,7 +143,9 @@ class ProfileConfig(BaseModel):
|
|
|
136
143
|
Database manager instance
|
|
137
144
|
|
|
138
145
|
Raises:
|
|
139
|
-
|
|
146
|
+
ValueError: If the database type is unsupported, or required
|
|
147
|
+
connection fields (e.g. PostgreSQL ``database``) are missing
|
|
148
|
+
ImportError: If the backend's driver is not installed
|
|
140
149
|
"""
|
|
141
150
|
if self.type == "clickhouse":
|
|
142
151
|
return ClickHouseDatabaseManager(
|
|
@@ -149,9 +158,36 @@ class ProfileConfig(BaseModel):
|
|
|
149
158
|
settings=self.settings,
|
|
150
159
|
)
|
|
151
160
|
elif self.type == "postgres":
|
|
152
|
-
|
|
161
|
+
from detectkit.database.postgres_manager import PostgresDatabaseManager
|
|
162
|
+
|
|
163
|
+
if not self.database:
|
|
164
|
+
raise ValueError(
|
|
165
|
+
"PostgreSQL profiles must set 'database' (the database to "
|
|
166
|
+
"connect to, inside which internal_schema/data_schema live)"
|
|
167
|
+
)
|
|
168
|
+
return PostgresDatabaseManager(
|
|
169
|
+
host=self.host,
|
|
170
|
+
port=self.port,
|
|
171
|
+
user=self.user,
|
|
172
|
+
password=self.password,
|
|
173
|
+
database=self.database,
|
|
174
|
+
internal_schema=self.get_internal_location(),
|
|
175
|
+
data_schema=self.get_data_location(),
|
|
176
|
+
settings=self.settings,
|
|
177
|
+
)
|
|
153
178
|
elif self.type == "mysql":
|
|
154
|
-
|
|
179
|
+
from detectkit.database.mysql_manager import MySQLDatabaseManager
|
|
180
|
+
|
|
181
|
+
return MySQLDatabaseManager(
|
|
182
|
+
host=self.host,
|
|
183
|
+
port=self.port,
|
|
184
|
+
user=self.user,
|
|
185
|
+
password=self.password,
|
|
186
|
+
database=self.database,
|
|
187
|
+
internal_database=self.get_internal_location(),
|
|
188
|
+
data_database=self.get_data_location(),
|
|
189
|
+
settings=self.settings,
|
|
190
|
+
)
|
|
155
191
|
else:
|
|
156
192
|
raise ValueError(f"Unsupported database type: {self.type}")
|
|
157
193
|
|
|
@@ -47,6 +47,12 @@ class TableModel:
|
|
|
47
47
|
engine: Database engine (ClickHouse-specific, e.g., "MergeTree")
|
|
48
48
|
order_by: Columns for ORDER BY clause (ClickHouse-specific)
|
|
49
49
|
indexes: Additional indexes to create
|
|
50
|
+
version_column: Column that drives last-writer-wins deduplication.
|
|
51
|
+
On ClickHouse this is the version encoded in the engine string
|
|
52
|
+
(e.g. ``ReplacingMergeTree(created_at)``); on SQL backends with an
|
|
53
|
+
enforced primary key it drives a version-aware upsert so a re-insert
|
|
54
|
+
with a newer ``version_column`` replaces the existing row. ``None``
|
|
55
|
+
for tables that do not deduplicate by version (e.g. ``_dtk_tasks``).
|
|
50
56
|
|
|
51
57
|
Example:
|
|
52
58
|
>>> model = TableModel(
|
|
@@ -65,6 +71,7 @@ class TableModel:
|
|
|
65
71
|
engine: str | None = None
|
|
66
72
|
order_by: list[str] | None = None
|
|
67
73
|
indexes: list[str] = field(default_factory=list)
|
|
74
|
+
version_column: str | None = None
|
|
68
75
|
|
|
69
76
|
def __post_init__(self):
|
|
70
77
|
"""Validate table model."""
|
|
@@ -86,6 +93,10 @@ class TableModel:
|
|
|
86
93
|
if order_col not in column_names:
|
|
87
94
|
raise ValueError(f"ORDER BY column '{order_col}' not found in table columns")
|
|
88
95
|
|
|
96
|
+
# Validate version column exists (if specified)
|
|
97
|
+
if self.version_column and self.version_column not in column_names:
|
|
98
|
+
raise ValueError(f"Version column '{self.version_column}' not found in table columns")
|
|
99
|
+
|
|
89
100
|
def get_column(self, name: str) -> ColumnDefinition | None:
|
|
90
101
|
"""
|
|
91
102
|
Get column definition by name.
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
"""Database managers for detectk."""
|
|
2
2
|
|
|
3
|
+
from detectkit.database._sql_manager import SQLDatabaseManager
|
|
3
4
|
from detectkit.database.clickhouse_manager import ClickHouseDatabaseManager
|
|
4
5
|
from detectkit.database.internal_tables import InternalTablesManager
|
|
5
6
|
from detectkit.database.manager import BaseDatabaseManager
|
|
7
|
+
from detectkit.database.mysql_manager import MySQLDatabaseManager
|
|
8
|
+
from detectkit.database.postgres_manager import PostgresDatabaseManager
|
|
6
9
|
from detectkit.database.tables import (
|
|
7
10
|
INTERNAL_TABLES,
|
|
8
11
|
TABLE_DATAPOINTS,
|
|
@@ -16,6 +19,9 @@ from detectkit.database.tables import (
|
|
|
16
19
|
__all__ = [
|
|
17
20
|
"BaseDatabaseManager",
|
|
18
21
|
"ClickHouseDatabaseManager",
|
|
22
|
+
"SQLDatabaseManager",
|
|
23
|
+
"PostgresDatabaseManager",
|
|
24
|
+
"MySQLDatabaseManager",
|
|
19
25
|
"InternalTablesManager",
|
|
20
26
|
"TABLE_DATAPOINTS",
|
|
21
27
|
"TABLE_DETECTIONS",
|