detectkit 0.7.0__tar.gz → 0.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {detectkit-0.7.0/detectkit.egg-info → detectkit-0.8.1}/PKG-INFO +5 -2
  2. {detectkit-0.7.0 → detectkit-0.8.1}/README.md +4 -1
  3. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/__init__.py +1 -1
  4. detectkit-0.8.1/detectkit/cli/commands/clean.py +333 -0
  5. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/commands/run.py +15 -2
  6. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/main.py +64 -0
  7. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_alert_states.py +31 -0
  8. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_detections.py +27 -1
  9. detectkit-0.8.1/detectkit/database/internal_tables/_maintenance.py +70 -0
  10. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/manager.py +2 -0
  11. {detectkit-0.7.0 → detectkit-0.8.1/detectkit.egg-info}/PKG-INFO +5 -2
  12. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit.egg-info/SOURCES.txt +2 -0
  13. {detectkit-0.7.0 → detectkit-0.8.1}/LICENSE +0 -0
  14. {detectkit-0.7.0 → detectkit-0.8.1}/MANIFEST.in +0 -0
  15. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/__init__.py +0 -0
  16. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/__init__.py +0 -0
  17. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/base.py +0 -0
  18. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/email.py +0 -0
  19. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/factory.py +0 -0
  20. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/mattermost.py +0 -0
  21. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/slack.py +0 -0
  22. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/telegram.py +0 -0
  23. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/channels/webhook.py +0 -0
  24. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/__init__.py +0 -0
  25. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_base.py +0 -0
  26. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_cooldown.py +0 -0
  27. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_decision.py +0 -0
  28. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_dispatch.py +0 -0
  29. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_recovery.py +0 -0
  30. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/_types.py +0 -0
  31. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/alerting/orchestrator/orchestrator.py +0 -0
  32. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/__init__.py +0 -0
  33. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/commands/__init__.py +0 -0
  34. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/commands/init.py +0 -0
  35. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/commands/test_alert.py +0 -0
  36. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/cli/commands/unlock.py +0 -0
  37. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/config/__init__.py +0 -0
  38. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/config/metric_config.py +0 -0
  39. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/config/profile.py +0 -0
  40. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/config/project_config.py +0 -0
  41. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/config/validator.py +0 -0
  42. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/core/__init__.py +0 -0
  43. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/core/interval.py +0 -0
  44. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/core/models.py +0 -0
  45. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/__init__.py +0 -0
  46. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/clickhouse_manager.py +0 -0
  47. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/__init__.py +0 -0
  48. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_base.py +0 -0
  49. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_datapoints.py +0 -0
  50. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_metrics.py +0 -0
  51. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_schema.py +0 -0
  52. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/internal_tables/_tasks.py +0 -0
  53. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/manager.py +0 -0
  54. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/database/tables.py +0 -0
  55. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/__init__.py +0 -0
  56. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/base.py +0 -0
  57. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/factory.py +0 -0
  58. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/seasonality.py +0 -0
  59. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/__init__.py +0 -0
  60. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/_windowed.py +0 -0
  61. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/iqr.py +0 -0
  62. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/mad.py +0 -0
  63. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/manual_bounds.py +0 -0
  64. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/detectors/statistical/zscore.py +0 -0
  65. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/loaders/__init__.py +0 -0
  66. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/loaders/metric_loader.py +0 -0
  67. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/loaders/query_template.py +0 -0
  68. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/__init__.py +0 -0
  69. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/error_dispatch.py +0 -0
  70. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/__init__.py +0 -0
  71. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/_alert_step.py +0 -0
  72. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/_base.py +0 -0
  73. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/_detect_step.py +0 -0
  74. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/_load_step.py +0 -0
  75. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/_types.py +0 -0
  76. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/orchestration/task_manager/manager.py +0 -0
  77. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/utils/__init__.py +0 -0
  78. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/utils/datetime_utils.py +0 -0
  79. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/utils/env_interpolation.py +0 -0
  80. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/utils/json_utils.py +0 -0
  81. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit/utils/stats.py +0 -0
  82. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit.egg-info/dependency_links.txt +0 -0
  83. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit.egg-info/entry_points.txt +0 -0
  84. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit.egg-info/requires.txt +0 -0
  85. {detectkit-0.7.0 → detectkit-0.8.1}/detectkit.egg-info/top_level.txt +0 -0
  86. {detectkit-0.7.0 → detectkit-0.8.1}/pyproject.toml +0 -0
  87. {detectkit-0.7.0 → detectkit-0.8.1}/requirements.txt +0 -0
  88. {detectkit-0.7.0 → detectkit-0.8.1}/setup.cfg +0 -0
  89. {detectkit-0.7.0 → detectkit-0.8.1}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectkit
3
- Version: 0.7.0
3
+ Version: 0.8.1
4
4
  Summary: Metric monitoring with automatic anomaly detection
5
5
  Author: detectkit team
6
6
  License: MIT
@@ -84,7 +84,7 @@ Dynamic: license-file
84
84
  - **Project-level error alerts** — catch DB outages and pipeline crashes once per run
85
85
  - **Database agnostic** — ClickHouse, PostgreSQL, MySQL
86
86
  - **Idempotent** — resume from interruptions, no duplicate processing
87
- - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, tag-based selectors
87
+ - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
88
88
 
89
89
  ## Installation
90
90
 
@@ -116,6 +116,9 @@ dtk run --select cpu_usage --from 2024-01-01
116
116
 
117
117
  # Clear a stuck lock left by a crashed run (e.g. DB restarted mid-run)
118
118
  dtk unlock --select cpu_usage
119
+
120
+ # Prune data orphaned by config edits (dry-run; add --execute to apply)
121
+ dtk clean --select cpu_usage
119
122
  ```
120
123
 
121
124
  ### Metric Configuration
@@ -18,7 +18,7 @@
18
18
  - **Project-level error alerts** — catch DB outages and pipeline crashes once per run
19
19
  - **Database agnostic** — ClickHouse, PostgreSQL, MySQL
20
20
  - **Idempotent** — resume from interruptions, no duplicate processing
21
- - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, tag-based selectors
21
+ - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
22
22
 
23
23
  ## Installation
24
24
 
@@ -50,6 +50,9 @@ dtk run --select cpu_usage --from 2024-01-01
50
50
 
51
51
  # Clear a stuck lock left by a crashed run (e.g. DB restarted mid-run)
52
52
  dtk unlock --select cpu_usage
53
+
54
+ # Prune data orphaned by config edits (dry-run; add --execute to apply)
55
+ dtk clean --select cpu_usage
53
56
  ```
54
57
 
55
58
  ### Metric Configuration
@@ -4,7 +4,7 @@ detectk - Anomaly Detection for Time-Series Metrics
4
4
  A Python library for data analysts and engineers to monitor metrics with automatic anomaly detection.
5
5
  """
6
6
 
7
- __version__ = "0.7.0"
7
+ __version__ = "0.8.1"
8
8
 
9
9
  from detectkit.core.interval import Interval
10
10
  from detectkit.core.models import ColumnDefinition, TableModel
@@ -0,0 +1,333 @@
1
+ """
2
+ Implementation of 'dtk clean' command.
3
+
4
+ Removes internal data that no longer matches the project's YAML configs —
5
+ the rows left behind when an analyst edits metrics on production
6
+ (TECHNICAL_SPEC.md §14.4 / init_plan.md "Сценарий Г"). Two modes:
7
+
8
+ * ``--select`` (drift mode): for metrics that still exist, delete detection
9
+ results whose ``detector_id`` is no longer produced by the config (a
10
+ detector param/seasonality changed, or the detector was removed) and
11
+ alert-state rows whose ``alert_config_id`` is no longer produced (an
12
+ alerting block changed or was removed). Datapoints are NOT touched — they
13
+ are keyed only by (metric, timestamp) and never orphaned by a param edit;
14
+ use ``--full-refresh`` to reload those.
15
+
16
+ * ``--orphaned-metrics`` (GC mode): delete ALL rows, across every internal
17
+ table, for metric names present in the database but no longer defined by
18
+ any YAML in the project (renamed or deleted metric).
19
+
20
+ Both modes default to a dry-run that only reports what would be deleted;
21
+ pass ``--execute`` to actually delete. Selector semantics match ``dtk run``.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from pathlib import Path
27
+
28
+ import click
29
+
30
+ from detectkit.cli.commands.run import find_project_root, select_metrics
31
+ from detectkit.config.metric_config import MetricConfig
32
+ from detectkit.config.profile import ProfilesConfig
33
+ from detectkit.config.validator import validate_project_metrics
34
+ from detectkit.database.internal_tables import InternalTablesManager
35
+ from detectkit.detectors.factory import DetectorFactory
36
+ from detectkit.orchestration.task_manager._types import make_alert_config_id
37
+
38
+
39
+ def run_clean(
40
+ select: str | None,
41
+ orphaned_metrics: bool,
42
+ execute: bool,
43
+ yes: bool,
44
+ profile: str | None,
45
+ ):
46
+ """Prune stale internal data that no longer matches the project configs.
47
+
48
+ Args:
49
+ select: Metric selector (drift mode) — same semantics as ``dtk run``.
50
+ orphaned_metrics: GC mode — purge metrics no longer present in the project.
51
+ execute: Actually delete (default: dry-run, only report).
52
+ yes: Skip the confirmation prompt in GC mode.
53
+ profile: Profile name to use (defaults to project's default_profile).
54
+ """
55
+ if bool(select) == bool(orphaned_metrics):
56
+ click.echo(
57
+ click.style(
58
+ "Error: choose exactly one of --select or --orphaned-metrics.",
59
+ fg="red",
60
+ bold=True,
61
+ )
62
+ )
63
+ return
64
+
65
+ project_root = find_project_root()
66
+ if not project_root:
67
+ click.echo(click.style("Error: Not in a detectkit project directory!", fg="red", bold=True))
68
+ click.echo("Run 'dtk init <project_name>' to create a new project.")
69
+ return
70
+
71
+ click.echo(f"Project root: {project_root}")
72
+
73
+ internal_manager = _create_internal_manager(project_root, profile)
74
+ if internal_manager is None:
75
+ return
76
+
77
+ if not execute:
78
+ click.echo(
79
+ click.style("DRY-RUN — nothing will be deleted. Use --execute to apply.", fg="cyan")
80
+ )
81
+ click.echo()
82
+
83
+ if select:
84
+ _clean_drift(internal_manager, select, project_root, execute)
85
+ else:
86
+ _clean_orphaned_metrics(internal_manager, project_root, execute, yes)
87
+
88
+
89
+ # ── modes ──────────────────────────────────────────────────────────────────
90
+
91
+
92
+ def _clean_drift(
93
+ internal_manager: InternalTablesManager,
94
+ select: str,
95
+ project_root: Path,
96
+ execute: bool,
97
+ ) -> None:
98
+ """Prune detector/alert data whose hash is no longer produced by the config."""
99
+ try:
100
+ metrics = select_metrics(select, project_root)
101
+ except ValueError as e:
102
+ click.echo(click.style(f"Error: {e}", fg="red", bold=True))
103
+ return
104
+
105
+ if not metrics:
106
+ click.echo(click.style(f"No metrics found matching selector: {select}", fg="yellow"))
107
+ return
108
+
109
+ click.echo(f"Found {len(metrics)} metric(s) to inspect")
110
+ click.echo()
111
+
112
+ total_det_groups = 0
113
+ total_alert_rows = 0
114
+
115
+ for _, config in metrics:
116
+ metric_name = config.name
117
+ try:
118
+ valid_detectors = _valid_detector_ids(config)
119
+ valid_alerts = _valid_alert_config_ids(config)
120
+ db_detectors = internal_manager.list_detector_ids(metric_name)
121
+ db_alerts = internal_manager.list_alert_config_ids(metric_name)
122
+ except Exception as e:
123
+ click.echo(click.style(f" ✗ {metric_name}: error inspecting: {e}", fg="red"), err=True)
124
+ continue
125
+
126
+ orphan_detectors = {
127
+ det_id: count for det_id, count in db_detectors.items() if det_id not in valid_detectors
128
+ }
129
+ orphan_alerts = [a for a in db_alerts if a not in valid_alerts]
130
+
131
+ if not orphan_detectors and not orphan_alerts:
132
+ click.echo(f" • {metric_name}: nothing stale")
133
+ continue
134
+
135
+ click.echo(click.style(f" {metric_name}:", bold=True))
136
+
137
+ # An empty valid set means EVERY stored row is "orphaned" — usually a
138
+ # config mid-edit, not an intent to wipe the metric. Flag it loudly.
139
+ if orphan_detectors and not valid_detectors:
140
+ click.echo(
141
+ click.style(
142
+ " ⚠ config defines no detectors — ALL detections below would be removed",
143
+ fg="yellow",
144
+ bold=True,
145
+ )
146
+ )
147
+ if orphan_alerts and not valid_alerts:
148
+ click.echo(
149
+ click.style(
150
+ " ⚠ config defines no alerting — ALL alert states below would be removed",
151
+ fg="yellow",
152
+ bold=True,
153
+ )
154
+ )
155
+
156
+ for det_id, count in sorted(orphan_detectors.items()):
157
+ total_det_groups += 1
158
+ verb = "deleting" if execute else "would delete"
159
+ click.echo(f" detector {det_id}: {verb} {count:,} detection row(s)")
160
+ if execute:
161
+ internal_manager.delete_detections(
162
+ metric_name=metric_name, detector_id=det_id, mutations_sync=True
163
+ )
164
+
165
+ for alert_id in sorted(orphan_alerts):
166
+ total_alert_rows += 1
167
+ verb = "deleting" if execute else "would delete"
168
+ click.echo(f" alert_config {alert_id}: {verb} stale alert state")
169
+ if execute:
170
+ internal_manager.delete_alert_state(metric_name, alert_id)
171
+
172
+ click.echo()
173
+ prefix = "Deleted" if execute else "Would delete"
174
+ click.echo(
175
+ click.style(
176
+ f"{prefix} {total_det_groups} orphaned detector group(s) "
177
+ f"and {total_alert_rows} orphaned alert-state row(s).",
178
+ fg="cyan",
179
+ bold=True,
180
+ )
181
+ )
182
+ if not execute and (total_det_groups or total_alert_rows):
183
+ click.echo("Re-run with --execute to apply.")
184
+
185
+
186
+ def _clean_orphaned_metrics(
187
+ internal_manager: InternalTablesManager,
188
+ project_root: Path,
189
+ execute: bool,
190
+ yes: bool,
191
+ ) -> None:
192
+ """Purge all data for metrics present in the DB but absent from the project."""
193
+ try:
194
+ project_metrics = validate_project_metrics(project_root)
195
+ project_names = {config.name for _, config in project_metrics}
196
+ except FileNotFoundError:
197
+ # No metrics/ directory at all — every DB metric is technically orphaned.
198
+ project_names = set()
199
+ except ValueError as e:
200
+ # Duplicates / parse errors: we can't trust the project set, so refuse
201
+ # to delete anything rather than risk purging valid metrics.
202
+ click.echo(
203
+ click.style(
204
+ f"Error: cannot determine project metrics ({e}). "
205
+ "Fix the configs first; aborting to avoid deleting valid data.",
206
+ fg="red",
207
+ bold=True,
208
+ )
209
+ )
210
+ return
211
+
212
+ db_names = internal_manager.list_known_metric_names()
213
+ orphans = sorted(db_names - project_names)
214
+
215
+ if not orphans:
216
+ click.echo(click.style("No orphaned metrics — database matches the project.", fg="green"))
217
+ return
218
+
219
+ click.echo(f"Found {len(orphans)} metric(s) in the database with no YAML in the project:")
220
+ click.echo()
221
+ for name in orphans:
222
+ try:
223
+ counts = internal_manager.count_metric_rows(name)
224
+ except Exception as e:
225
+ click.echo(click.style(f" ✗ {name}: error counting rows: {e}", fg="red"), err=True)
226
+ continue
227
+ total = sum(counts.values())
228
+ verb = "deleting" if execute else "would delete"
229
+ detail = ", ".join(f"{table}={count:,}" for table, count in counts.items() if count)
230
+ click.echo(
231
+ click.style(f" {name}: {verb} {total:,} row(s)", bold=True)
232
+ + (f" [{detail}]" if detail else "")
233
+ )
234
+
235
+ if not execute:
236
+ click.echo()
237
+ click.echo("Re-run with --execute to purge these metrics.")
238
+ return
239
+
240
+ # Guard: an empty project set means --execute would wipe EVERYTHING. Almost
241
+ # always a wrong directory / empty project, so demand explicit --yes.
242
+ if not project_names and not yes:
243
+ click.echo()
244
+ click.echo(
245
+ click.style(
246
+ "Refusing to purge: the project defines no metrics, so this would "
247
+ "delete ALL data. Re-run with --yes if that is really intended.",
248
+ fg="red",
249
+ bold=True,
250
+ )
251
+ )
252
+ return
253
+
254
+ if not yes:
255
+ click.echo()
256
+ if not click.confirm(
257
+ click.style(f"Permanently delete all data for {len(orphans)} metric(s)?", fg="yellow")
258
+ ):
259
+ click.echo("Aborted.")
260
+ return
261
+
262
+ purged = 0
263
+ for name in orphans:
264
+ try:
265
+ internal_manager.purge_metric(name)
266
+ purged += 1
267
+ click.echo(click.style(f" ✓ {name}: purged", fg="green"))
268
+ except Exception as e:
269
+ click.echo(click.style(f" ✗ {name}: error purging: {e}", fg="red"), err=True)
270
+
271
+ click.echo()
272
+ click.echo(
273
+ click.style(
274
+ f"Done. Purged {purged} of {len(orphans)} orphaned metric(s).", fg="cyan", bold=True
275
+ )
276
+ )
277
+
278
+
279
+ # ── helpers ──────────────────────────────────────────────────────────────────
280
+
281
+
282
+ def _valid_detector_ids(config: MetricConfig) -> set[str]:
283
+ """Detector IDs the current config produces.
284
+
285
+ Mirrors the DETECT step exactly (DetectorFactory + the same seasonality
286
+ injection) so the computed ``detector_id`` matches what the pipeline
287
+ writes — anything in the DB not in this set is stale.
288
+ """
289
+ ids: set[str] = set()
290
+ for detector_config in config.detectors or []:
291
+ params = detector_config.get_algorithm_params()
292
+ seasonality_components = detector_config.get_seasonality_components()
293
+ if seasonality_components is not None:
294
+ params["seasonality_components"] = seasonality_components
295
+ detector = DetectorFactory.create_from_config(
296
+ {"type": detector_config.type, "params": params}
297
+ )
298
+ ids.add(detector.get_detector_id())
299
+ return ids
300
+
301
+
302
+ def _valid_alert_config_ids(config: MetricConfig) -> set[str]:
303
+ """Alert-config IDs the current config produces (enabled or not).
304
+
305
+ Disabled blocks keep their hash, so a temporarily-disabled alert is NOT
306
+ treated as orphaned; only removed or functionally-changed blocks are.
307
+ """
308
+ return {make_alert_config_id(c) for c in (config.alerting or [])}
309
+
310
+
311
+ def _create_internal_manager(
312
+ project_root: Path, profile: str | None
313
+ ) -> InternalTablesManager | None:
314
+ """Load profiles.yml and build an InternalTablesManager, or report and return None."""
315
+ profiles_path = project_root / "profiles.yml"
316
+ if not profiles_path.exists():
317
+ click.echo(click.style("Error: profiles.yml not found!", fg="red", bold=True))
318
+ click.echo(f"Expected at: {profiles_path}")
319
+ return None
320
+
321
+ try:
322
+ profiles_config = ProfilesConfig.from_yaml(profiles_path)
323
+ except Exception as e:
324
+ click.echo(click.style(f"Error loading profiles.yml: {e}", fg="red", bold=True))
325
+ return None
326
+
327
+ try:
328
+ db_manager = profiles_config.create_manager(profile)
329
+ except Exception as e:
330
+ click.echo(click.style(f"Error creating database manager: {e}", fg="red", bold=True))
331
+ return None
332
+
333
+ return InternalTablesManager(db_manager)
@@ -353,8 +353,21 @@ def select_metrics(selector: str, project_root: Path) -> list[tuple[Path, Metric
353
353
  metric_paths = find_metrics_by_tag(metrics_dir, tag)
354
354
  # Path pattern selector
355
355
  elif "*" in selector or "/" in selector:
356
- pattern = selector if selector.startswith("metrics/") else f"metrics/{selector}"
357
- metric_paths = list(project_root.glob(pattern))
356
+ if selector == "*":
357
+ # "all metrics" — search recursively so nested metrics are included
358
+ # (mirrors validate_project_metrics); a plain glob of "metrics/*"
359
+ # would only see the top level.
360
+ metric_paths = [p for sub in ("**/*.yml", "**/*.yaml") for p in metrics_dir.glob(sub)]
361
+ else:
362
+ pattern = selector if selector.startswith("metrics/") else f"metrics/{selector}"
363
+ # Keep only metric files: a bare glob also matches the `.gitkeep`
364
+ # stub created by `dtk init`, any other non-YAML files, and
365
+ # directories — all of which would crash the YAML parser.
366
+ metric_paths = [
367
+ p
368
+ for p in project_root.glob(pattern)
369
+ if p.is_file() and p.suffix in (".yml", ".yaml")
370
+ ]
358
371
  # Metric name selector
359
372
  else:
360
373
  # First try filename-based search in root (backward compatibility)
@@ -218,5 +218,69 @@ def unlock(select: str, profile: str):
218
218
  run_unlock(select=select, profile=profile)
219
219
 
220
220
 
221
+ @cli.command()
222
+ @click.option(
223
+ "--select",
224
+ "-s",
225
+ help="Selector for metrics whose stale detector/alert data to prune (name, path, or tag)",
226
+ )
227
+ @click.option(
228
+ "--orphaned-metrics",
229
+ is_flag=True,
230
+ help="Purge all data for metrics no longer present in the project (renamed/deleted YAML)",
231
+ )
232
+ @click.option(
233
+ "--execute",
234
+ is_flag=True,
235
+ help="Actually delete (default: dry-run, only report what would be removed)",
236
+ )
237
+ @click.option(
238
+ "--yes",
239
+ "-y",
240
+ is_flag=True,
241
+ help="Skip the confirmation prompt (for --orphaned-metrics --execute)",
242
+ )
243
+ @click.option(
244
+ "--profile",
245
+ help="Profile to use (default: from project config)",
246
+ )
247
+ def clean(select: str, orphaned_metrics: bool, execute: bool, yes: bool, profile: str):
248
+ """
249
+ Remove internal data that no longer matches the project's YAML configs.
250
+
251
+ Over time, editing metrics on production leaves stale rows behind: changing
252
+ a detector parameter (or removing a detector) orphans its old results in
253
+ _dtk_detections, changing an alerting block orphans its state in
254
+ _dtk_alert_states, and renaming/deleting a metric orphans everything under
255
+ its old name. This command finds and removes that drift.
256
+
257
+ Both modes default to a dry-run; pass --execute to actually delete.
258
+ Selector semantics match `dtk run`.
259
+
260
+ Examples:
261
+ # Prune stale detector/alert data for one metric (dry-run)
262
+ dtk clean --select cpu_usage
263
+
264
+ # ...and actually delete it
265
+ dtk clean --select cpu_usage --execute
266
+
267
+ # Prune everything matching a tag
268
+ dtk clean --select "tag:critical" --execute
269
+
270
+ # Purge metrics that no longer exist in the project
271
+ dtk clean --orphaned-metrics
272
+ dtk clean --orphaned-metrics --execute
273
+ """
274
+ from detectkit.cli.commands.clean import run_clean
275
+
276
+ run_clean(
277
+ select=select,
278
+ orphaned_metrics=orphaned_metrics,
279
+ execute=execute,
280
+ yes=yes,
281
+ profile=profile,
282
+ )
283
+
284
+
221
285
  if __name__ == "__main__":
222
286
  cli()
@@ -113,6 +113,37 @@ class _AlertStatesMixin(_InternalTablesBase):
113
113
  }
114
114
  self._manager.insert_batch(full_table_name, insert_data, conflict_strategy="ignore")
115
115
 
116
+ def list_alert_config_ids(self, metric_name: str) -> list[str]:
117
+ """Return every ``alert_config_id`` with stored state for a metric.
118
+
119
+ Used by ``dtk clean`` to find alert-state rows left behind after an
120
+ alerting block was removed or its functional params changed (see
121
+ ``make_alert_config_id``).
122
+ """
123
+ full_table_name = self._manager.get_full_table_name(TABLE_ALERT_STATES, use_internal=True)
124
+ query = f"""
125
+ SELECT DISTINCT alert_config_id
126
+ FROM {full_table_name}
127
+ WHERE metric_name = %(metric_name)s
128
+ """
129
+ result = self._manager.execute_query(query, {"metric_name": metric_name})
130
+ return [row["alert_config_id"] for row in result if row.get("alert_config_id")]
131
+
132
+ def delete_alert_state(self, metric_name: str, alert_config_id: str) -> int:
133
+ """Delete the alert-state row for a single ``(metric, alert_config)``."""
134
+ full_table_name = self._manager.get_full_table_name(TABLE_ALERT_STATES, use_internal=True)
135
+ query = f"""
136
+ ALTER TABLE {full_table_name}
137
+ DELETE WHERE metric_name = %(metric_name)s
138
+ AND alert_config_id = %(alert_config_id)s
139
+ SETTINGS mutations_sync = 1
140
+ """
141
+ self._manager.execute_query(
142
+ query,
143
+ params={"metric_name": metric_name, "alert_config_id": alert_config_id},
144
+ )
145
+ return 0
146
+
116
147
  def get_last_alert_timestamp(
117
148
  self,
118
149
  metric_name: str,
@@ -62,8 +62,16 @@ class _DetectionsMixin(_InternalTablesBase):
62
62
  detector_id: str | None = None,
63
63
  from_timestamp: datetime | None = None,
64
64
  to_timestamp: datetime | None = None,
65
+ mutations_sync: bool = False,
65
66
  ) -> int:
66
- """Delete detection rows for the supplied filter set."""
67
+ """Delete detection rows for the supplied filter set.
68
+
69
+ ``mutations_sync=True`` waits for the ClickHouse mutation to finish
70
+ before returning (``SETTINGS mutations_sync = 1``); the default keeps
71
+ the async behaviour used by the ``--full-refresh`` hot path. The
72
+ ``dtk clean`` command passes ``True`` so a follow-up dry-run reflects
73
+ the deletion immediately.
74
+ """
67
75
  full_table_name = self._manager.get_full_table_name(TABLE_DETECTIONS, use_internal=True)
68
76
 
69
77
  where_parts = ["metric_name = %(metric_name)s"]
@@ -79,9 +87,27 @@ class _DetectionsMixin(_InternalTablesBase):
79
87
  params["to_timestamp"] = to_timestamp
80
88
 
81
89
  query = f"ALTER TABLE {full_table_name} DELETE WHERE {' AND '.join(where_parts)}"
90
+ if mutations_sync:
91
+ query += " SETTINGS mutations_sync = 1"
82
92
  self._manager.execute_query(query, params=params)
83
93
  return 0
84
94
 
95
+ def list_detector_ids(self, metric_name: str) -> dict[str, int]:
96
+ """Return ``{detector_id: row_count}`` for every detector stored for a metric.
97
+
98
+ Used by ``dtk clean`` to spot detector results left behind after a
99
+ config change altered the detector hash (see ``get_detector_id``).
100
+ """
101
+ full_table_name = self._manager.get_full_table_name(TABLE_DETECTIONS, use_internal=True)
102
+ query = f"""
103
+ SELECT detector_id, count() AS cnt
104
+ FROM {full_table_name}
105
+ WHERE metric_name = %(metric_name)s
106
+ GROUP BY detector_id
107
+ """
108
+ result = self._manager.execute_query(query, {"metric_name": metric_name})
109
+ return {row["detector_id"]: int(row["cnt"]) for row in result if row.get("detector_id")}
110
+
85
111
  def get_recent_detections(
86
112
  self,
87
113
  metric_name: str,
@@ -0,0 +1,70 @@
1
+ """Maintenance mixin: cross-table cleanup helpers for ``dtk clean``.
2
+
3
+ These support pruning data left behind when an analyst edits metric configs
4
+ on production — most importantly removing all rows for a metric whose YAML no
5
+ longer exists in the project (TECHNICAL_SPEC.md §14.4 / init_plan.md
6
+ "Сценарий Г"). They are used only by the ``dtk clean`` CLI command, never by
7
+ the run pipeline.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from detectkit.database.internal_tables._base import _InternalTablesBase
13
+ from detectkit.database.tables import (
14
+ TABLE_ALERT_STATES,
15
+ TABLE_DATAPOINTS,
16
+ TABLE_DETECTIONS,
17
+ TABLE_METRICS,
18
+ TABLE_TASKS,
19
+ )
20
+
21
+ # Every internal table is keyed by ``metric_name``, so a metric removed from
22
+ # the project (renamed or deleted YAML) leaves orphaned rows in all of them.
23
+ METRIC_KEYED_TABLES: tuple[str, ...] = (
24
+ TABLE_DATAPOINTS,
25
+ TABLE_DETECTIONS,
26
+ TABLE_TASKS,
27
+ TABLE_ALERT_STATES,
28
+ TABLE_METRICS,
29
+ )
30
+
31
+
32
+ class _MaintenanceMixin(_InternalTablesBase):
33
+ def list_known_metric_names(self) -> set[str]:
34
+ """Return every ``metric_name`` that has rows in any internal table.
35
+
36
+ Unions ``SELECT DISTINCT metric_name`` across all metric-keyed tables
37
+ so a metric is reported even if it only ever loaded datapoints (and
38
+ thus never wrote an alert state, etc.).
39
+ """
40
+ names: set[str] = set()
41
+ for table in METRIC_KEYED_TABLES:
42
+ full_table_name = self._manager.get_full_table_name(table, use_internal=True)
43
+ query = f"SELECT DISTINCT metric_name FROM {full_table_name}"
44
+ result = self._manager.execute_query(query)
45
+ names.update(row["metric_name"] for row in result if row.get("metric_name"))
46
+ return names
47
+
48
+ def count_metric_rows(self, metric_name: str) -> dict[str, int]:
49
+ """Return per-table row counts for *metric_name* (for dry-run reports)."""
50
+ counts: dict[str, int] = {}
51
+ for table in METRIC_KEYED_TABLES:
52
+ full_table_name = self._manager.get_full_table_name(table, use_internal=True)
53
+ query = f"SELECT count() AS cnt FROM {full_table_name} WHERE metric_name = %(m)s"
54
+ result = self._manager.execute_query(query, {"m": metric_name})
55
+ counts[table] = int(result[0]["cnt"]) if result else 0
56
+ return counts
57
+
58
+ def purge_metric(self, metric_name: str) -> None:
59
+ """Delete every row for *metric_name* across all internal tables.
60
+
61
+ Each delete waits for its mutation (``SETTINGS mutations_sync = 1``)
62
+ so the purge is fully applied when this returns.
63
+ """
64
+ for table in METRIC_KEYED_TABLES:
65
+ full_table_name = self._manager.get_full_table_name(table, use_internal=True)
66
+ query = (
67
+ f"ALTER TABLE {full_table_name} DELETE WHERE metric_name = %(m)s "
68
+ f"SETTINGS mutations_sync = 1"
69
+ )
70
+ self._manager.execute_query(query, {"m": metric_name})
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  from detectkit.database.internal_tables._alert_states import _AlertStatesMixin
6
6
  from detectkit.database.internal_tables._datapoints import _DatapointsMixin
7
7
  from detectkit.database.internal_tables._detections import _DetectionsMixin
8
+ from detectkit.database.internal_tables._maintenance import _MaintenanceMixin
8
9
  from detectkit.database.internal_tables._metrics import _MetricsMixin
9
10
  from detectkit.database.internal_tables._schema import _SchemaMixin
10
11
  from detectkit.database.internal_tables._tasks import _TasksMixin
@@ -17,6 +18,7 @@ class InternalTablesManager(
17
18
  _TasksMixin,
18
19
  _MetricsMixin,
19
20
  _AlertStatesMixin,
21
+ _MaintenanceMixin,
20
22
  ):
21
23
  """High-level façade over a :class:`BaseDatabaseManager` for ``_dtk_*`` tables.
22
24
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: detectkit
3
- Version: 0.7.0
3
+ Version: 0.8.1
4
4
  Summary: Metric monitoring with automatic anomaly detection
5
5
  Author: detectkit team
6
6
  License: MIT
@@ -84,7 +84,7 @@ Dynamic: license-file
84
84
  - **Project-level error alerts** — catch DB outages and pipeline crashes once per run
85
85
  - **Database agnostic** — ClickHouse, PostgreSQL, MySQL
86
86
  - **Idempotent** — resume from interruptions, no duplicate processing
87
- - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, tag-based selectors
87
+ - **CLI** — `dtk init`, `dtk run --select`, `dtk unlock`, `dtk clean`, tag-based selectors
88
88
 
89
89
  ## Installation
90
90
 
@@ -116,6 +116,9 @@ dtk run --select cpu_usage --from 2024-01-01
116
116
 
117
117
  # Clear a stuck lock left by a crashed run (e.g. DB restarted mid-run)
118
118
  dtk unlock --select cpu_usage
119
+
120
+ # Prune data orphaned by config edits (dry-run; add --execute to apply)
121
+ dtk clean --select cpu_usage
119
122
  ```
120
123
 
121
124
  ### Metric Configuration
@@ -31,6 +31,7 @@ detectkit/alerting/orchestrator/orchestrator.py
31
31
  detectkit/cli/__init__.py
32
32
  detectkit/cli/main.py
33
33
  detectkit/cli/commands/__init__.py
34
+ detectkit/cli/commands/clean.py
34
35
  detectkit/cli/commands/init.py
35
36
  detectkit/cli/commands/run.py
36
37
  detectkit/cli/commands/test_alert.py
@@ -52,6 +53,7 @@ detectkit/database/internal_tables/_alert_states.py
52
53
  detectkit/database/internal_tables/_base.py
53
54
  detectkit/database/internal_tables/_datapoints.py
54
55
  detectkit/database/internal_tables/_detections.py
56
+ detectkit/database/internal_tables/_maintenance.py
55
57
  detectkit/database/internal_tables/_metrics.py
56
58
  detectkit/database/internal_tables/_schema.py
57
59
  detectkit/database/internal_tables/_tasks.py
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes