observability-aiops 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. observability_aiops-0.1.0/.github/workflows/publish.yml +26 -0
  2. observability_aiops-0.1.0/.gitignore +7 -0
  3. observability_aiops-0.1.0/CHANGELOG.md +54 -0
  4. observability_aiops-0.1.0/LICENSE +21 -0
  5. observability_aiops-0.1.0/PKG-INFO +127 -0
  6. observability_aiops-0.1.0/README.md +111 -0
  7. observability_aiops-0.1.0/RELEASE_NOTES.md +61 -0
  8. observability_aiops-0.1.0/SECURITY.md +83 -0
  9. observability_aiops-0.1.0/mcp_server/__init__.py +1 -0
  10. observability_aiops-0.1.0/mcp_server/_shared.py +103 -0
  11. observability_aiops-0.1.0/mcp_server/server.py +39 -0
  12. observability_aiops-0.1.0/mcp_server/tools/__init__.py +1 -0
  13. observability_aiops-0.1.0/mcp_server/tools/alerts.py +56 -0
  14. observability_aiops-0.1.0/mcp_server/tools/analysis.py +59 -0
  15. observability_aiops-0.1.0/mcp_server/tools/grafana.py +70 -0
  16. observability_aiops-0.1.0/mcp_server/tools/metrics.py +72 -0
  17. observability_aiops-0.1.0/mcp_server/tools/overview.py +22 -0
  18. observability_aiops-0.1.0/mcp_server/tools/prometheus.py +31 -0
  19. observability_aiops-0.1.0/mcp_server/tools/rules.py +32 -0
  20. observability_aiops-0.1.0/mcp_server/tools/targets.py +44 -0
  21. observability_aiops-0.1.0/mcp_server/tools/writes.py +203 -0
  22. observability_aiops-0.1.0/observability_aiops/__init__.py +9 -0
  23. observability_aiops-0.1.0/observability_aiops/cli/__init__.py +9 -0
  24. observability_aiops-0.1.0/observability_aiops/cli/_common.py +78 -0
  25. observability_aiops-0.1.0/observability_aiops/cli/_root.py +57 -0
  26. observability_aiops-0.1.0/observability_aiops/cli/alert.py +49 -0
  27. observability_aiops-0.1.0/observability_aiops/cli/doctor.py +21 -0
  28. observability_aiops-0.1.0/observability_aiops/cli/init.py +135 -0
  29. observability_aiops-0.1.0/observability_aiops/cli/overview.py +16 -0
  30. observability_aiops-0.1.0/observability_aiops/cli/query.py +58 -0
  31. observability_aiops-0.1.0/observability_aiops/cli/secret.py +107 -0
  32. observability_aiops-0.1.0/observability_aiops/config.py +179 -0
  33. observability_aiops-0.1.0/observability_aiops/connection.py +209 -0
  34. observability_aiops-0.1.0/observability_aiops/doctor.py +103 -0
  35. observability_aiops-0.1.0/observability_aiops/governance/__init__.py +40 -0
  36. observability_aiops-0.1.0/observability_aiops/governance/audit.py +377 -0
  37. observability_aiops-0.1.0/observability_aiops/governance/budget.py +225 -0
  38. observability_aiops-0.1.0/observability_aiops/governance/decorators.py +474 -0
  39. observability_aiops-0.1.0/observability_aiops/governance/paths.py +23 -0
  40. observability_aiops-0.1.0/observability_aiops/governance/patterns.py +378 -0
  41. observability_aiops-0.1.0/observability_aiops/governance/policy.py +411 -0
  42. observability_aiops-0.1.0/observability_aiops/governance/sanitize.py +39 -0
  43. observability_aiops-0.1.0/observability_aiops/governance/undo.py +218 -0
  44. observability_aiops-0.1.0/observability_aiops/ops/__init__.py +1 -0
  45. observability_aiops-0.1.0/observability_aiops/ops/_util.py +61 -0
  46. observability_aiops-0.1.0/observability_aiops/ops/alerts.py +129 -0
  47. observability_aiops-0.1.0/observability_aiops/ops/analysis.py +267 -0
  48. observability_aiops-0.1.0/observability_aiops/ops/grafana.py +102 -0
  49. observability_aiops-0.1.0/observability_aiops/ops/metrics.py +120 -0
  50. observability_aiops-0.1.0/observability_aiops/ops/overview.py +76 -0
  51. observability_aiops-0.1.0/observability_aiops/ops/prom_status.py +66 -0
  52. observability_aiops-0.1.0/observability_aiops/ops/rules.py +80 -0
  53. observability_aiops-0.1.0/observability_aiops/ops/targets.py +78 -0
  54. observability_aiops-0.1.0/observability_aiops/ops/writes.py +170 -0
  55. observability_aiops-0.1.0/observability_aiops/secretstore.py +302 -0
  56. observability_aiops-0.1.0/pyproject.toml +59 -0
  57. observability_aiops-0.1.0/server.json +21 -0
  58. observability_aiops-0.1.0/skills/observability-aiops/SKILL.md +151 -0
  59. observability_aiops-0.1.0/skills/observability-aiops/references/capabilities.md +81 -0
  60. observability_aiops-0.1.0/skills/observability-aiops/references/cli-reference.md +58 -0
  61. observability_aiops-0.1.0/skills/observability-aiops/references/setup-guide.md +108 -0
  62. observability_aiops-0.1.0/smithery.yaml +9 -0
  63. observability_aiops-0.1.0/tests/test_alerts.py +84 -0
  64. observability_aiops-0.1.0/tests/test_analysis.py +97 -0
  65. observability_aiops-0.1.0/tests/test_grafana.py +62 -0
  66. observability_aiops-0.1.0/tests/test_metrics.py +66 -0
  67. observability_aiops-0.1.0/tests/test_secretstore.py +99 -0
  68. observability_aiops-0.1.0/tests/test_smoke.py +181 -0
  69. observability_aiops-0.1.0/tests/test_targets_rules.py +90 -0
  70. observability_aiops-0.1.0/tests/test_writes.py +160 -0
  71. observability_aiops-0.1.0/uv.lock +962 -0
@@ -0,0 +1,26 @@
1
+ name: Publish to PyPI
2
+
3
+ # Trusted Publishing (OIDC) — publishes from GitHub's runners with no API token,
4
+ # sidestepping the local-IP / account new-project rate limit. Configure a matching
5
+ # "trusted publisher" for this package on PyPI (see the repo release notes).
6
+ on:
7
+ release:
8
+ types: [published]
9
+ workflow_dispatch:
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ publish:
16
+ runs-on: ubuntu-latest
17
+ permissions:
18
+ id-token: write # required for PyPI Trusted Publishing (OIDC)
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+ - name: Set up uv
22
+ uses: astral-sh/setup-uv@v5
23
+ - name: Build sdist + wheel
24
+ run: uv build
25
+ - name: Publish to PyPI (Trusted Publishing)
26
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,7 @@
1
+ .venv/
2
+ dist/
3
+ __pycache__/
4
+ *.pyc
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ *.egg-info/
@@ -0,0 +1,54 @@
1
+ # Changelog
2
+
3
+ All notable changes to observability-aiops are documented here. This project adheres
4
+ to [Semantic Versioning](https://semver.org/).
5
+
6
+ ## [0.1.0] — preview
7
+
8
+ Initial preview release: governed AI-ops for a **self-hosted observability
9
+ stack** — **Prometheus** (HTTP API + PromQL), **Alertmanager** (alerts +
10
+ silences), and **Grafana** (dashboards, datasources, folders) — with a bundled
11
+ governance harness. One config can span your whole stack.
12
+ **Mock-validated only — not yet verified against a live stack.**
13
+
14
+ ### Added
15
+
16
+ - **30 MCP tools** (24 read, 6 write), every one wrapped with the bundled
17
+ `@governed_tool` harness (audit, policy, token/runaway budget, undo,
18
+ risk-tiers):
19
+ - **Metrics (Prometheus, read)** — `instant_query`, `range_query`,
20
+ `label_values`, `series_metadata`.
21
+ - **Targets & status (read)** — `list_targets` (up/down filter),
22
+ `target_scrape_health`, `dropped_targets`, `prometheus_config_status`,
23
+ `prometheus_tsdb_status`.
24
+ - **Rules (read)** — `list_rules` (recording + alerting), `rule_health`.
25
+ - **Alerts (read)** — `firing_alerts`, `pending_alerts`, `alertmanager_alerts`,
26
+ `list_silences`.
27
+ - **Grafana (read)** — `list_dashboards`, `get_dashboard`, `list_datasources`,
28
+ `datasource_health`, `list_folders`.
29
+ - **Overview (read)** — `observability_overview` (platform-aware snapshot).
30
+ - **Writes** — `create_silence` (med, time-boxed, undo→expire),
31
+ `expire_silence` (med), `create_annotation` (low), `update_dashboard`
32
+ (med, captures prior model, undo→restore), `delete_dashboard` (**high**,
33
+ dry-run, captures prior model before delete, undo→recreate),
34
+ `reload_prometheus_config` (med, records prior config hash).
35
+ - **Three flagship analyses** (read) — `firing_alert_rca` (join each firing alert
36
+ to its rule expression → likely cause + action), `target_scrape_health_analysis`
37
+ (rank down/erroring scrape targets and classify each `lastError`), and
38
+ `alert_noise_and_flap_analysis` (frequently-repeated / duplicate alerts →
39
+ dedup/rollup recommendation). Transparent heuristics that report their numbers.
40
+ - **Encrypted secret store** — Prometheus/Grafana bearer tokens stored encrypted
41
+ in `~/.observability-aiops/secrets.enc` (Fernet + scrypt); never plaintext on
42
+ disk. Legacy `OBSERVABILITY_<TARGET>_TOKEN` env var honoured as a fallback.
43
+ - **CLI** (`observability-aiops`) — `init` platform-picking wizard, `overview`,
44
+ `query instant/range/labels`, `alert firing/silences/rca`, `secret`
45
+ management, and a platform-aware `doctor` (Prometheus `/api/v1/status/buildinfo`,
46
+ Grafana `/api/health`).
47
+
48
+ ### Known limitations
49
+
50
+ - Preview / mock-only: the Prometheus and Grafana HTTP API responses are mocked
51
+ and need live verification against a real stack. Both are free/open-source and
52
+ trivial to run locally for a `doctor` check.
53
+ - Hosted/SaaS monitoring suites (Datadog, New Relic, enterprise NMS) are out of
54
+ scope by design.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 wei <zhouwei008@gmail.com>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,127 @@
1
+ Metadata-Version: 2.4
2
+ Name: observability-aiops
3
+ Version: 0.1.0
4
+ Summary: Governed AI-ops for self-hosted Prometheus + Grafana: PromQL, scrape-target & rule health, alerts/silences, dashboards, and flagship RCA/scrape-health/alert-noise analyses with a built-in governance harness (audit, budget, undo, risk tiers)
5
+ Author-email: wei <zhouwei008@gmail.com>
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Requires-Python: >=3.11
9
+ Requires-Dist: cryptography>=42.0
10
+ Requires-Dist: httpx<1.0,>=0.27
11
+ Requires-Dist: mcp[cli]<2.0,>=1.10
12
+ Requires-Dist: pyyaml<7.0,>=6.0
13
+ Requires-Dist: rich<16.0,>=13.0
14
+ Requires-Dist: typer<1.0,>=0.12
15
+ Description-Content-Type: text/markdown
16
+
17
+ <!-- mcp-name: io.github.AIops-tools/observability-aiops -->
18
+
19
+ # Observability AIops (preview)
20
+
21
+ > **Disclaimer**: Community-maintained open-source project. **Not affiliated with, endorsed by, or sponsored by the Prometheus or Grafana projects, Grafana Labs, or the Cloud Native Computing Foundation.** Prometheus, Alertmanager and Grafana are trademarks of their respective owners. MIT licensed.
22
+
23
+ Governed AI-ops for a **self-hosted observability stack** in one server —
24
+ **Prometheus** (HTTP API, PromQL, targets, rules, alerts), **Alertmanager**
25
+ (alerts + silences), and **Grafana** (dashboards, datasources, folders) — with a
26
+ **built-in governance harness**: unified audit log, policy engine, token/runaway
27
+ budget guard, undo-token recording, and graduated-autonomy risk tiers. One config
28
+ can span your whole stack; each target names its own `platform`.
29
+ **Preview — mock-validated only, not yet verified against a live stack.**
30
+
31
+ This is the **self-hosted-observability** complement to enterprise monitoring
32
+ suites: it speaks the open Prometheus/Grafana APIs an SRE actually runs, not a
33
+ vendor NMS.
34
+
35
+ ## What it does
36
+
37
+ Answers the questions an SRE actually repeats over a Prometheus/Grafana stack,
38
+ and guards the writes that follow:
39
+
40
+ - **PromQL + metadata** — instant and range queries, label-value enumeration, and
41
+ series metadata, all read-only and result-capped.
42
+ - **Scrape-target & rule health** — which targets are up/down (and *why*, from
43
+ `lastError`), which were dropped by relabeling, and which recording/alerting
44
+ rules are erroring.
45
+ - **Alerts & silences** — firing/pending Prometheus rule alerts, Alertmanager's
46
+ post-routing view, and its silences.
47
+ - **Grafana** — dashboards, datasources (+ health), and folders.
48
+ - **Flagship analyses** — transparent heuristics that show their numbers:
49
+ `firing_alert_rca` (join each firing alert to its rule expr → cause + action),
50
+ `target_scrape_health_analysis` (rank down/erroring scrapes → likely cause), and
51
+ `alert_noise_and_flap_analysis` (frequently-repeated / duplicate alerts →
52
+ dedup/rollup recommendation).
53
+ - **Governed writes** — create/expire Alertmanager silences (time-boxed), create
54
+ Grafana annotations, update/delete dashboards, and hot-reload the Prometheus
55
+ config — each audited, risk-tiered, `dry_run`-able, and the reversible ones
56
+ capture the **real fetched before-state** for undo.
57
+
58
+ ## Capability matrix (30 MCP tools)
59
+
60
+ | Group | Platform | Tools | Count | R/W |
61
+ |-------|----------|-------|:-----:|:---:|
62
+ | **Metrics** | Prometheus | `instant_query`, `range_query`, `label_values`, `series_metadata` | 4 | read |
63
+ | **Targets** | Prometheus | `list_targets`, `target_scrape_health`, `dropped_targets` | 3 | read |
64
+ | **Status** | Prometheus | `prometheus_config_status`, `prometheus_tsdb_status` | 2 | read |
65
+ | **Rules** | Prometheus | `list_rules`, `rule_health` | 2 | read |
66
+ | **Alerts** | Prometheus/Alertmanager | `firing_alerts`, `pending_alerts`, `alertmanager_alerts`, `list_silences` | 4 | read |
67
+ | **Grafana** | Grafana | `list_dashboards`, `get_dashboard`, `list_datasources`, `datasource_health`, `list_folders` | 5 | read |
68
+ | **Overview** | both | `observability_overview` | 1 | read |
69
+ | **Analyses** | Prometheus | `firing_alert_rca`, `target_scrape_health_analysis`, `alert_noise_and_flap_analysis` | 3 | read |
70
+ | **Writes** | Alertmanager | `create_silence`, `expire_silence` | 2 | write (med) |
71
+ | | Grafana | `create_annotation` | 1 | write (low) |
72
+ | | Grafana | `update_dashboard` | 1 | write (med) |
73
+ | | Grafana | `delete_dashboard` | 1 | write (**high**) |
74
+ | | Prometheus | `reload_prometheus_config` | 1 | write (med) |
75
+
76
+ The CLI exposes a convenience subset; the full 30-tool surface is via the MCP
77
+ server.
78
+
79
+ ## Quick start
80
+
81
+ ```bash
82
+ uv tool install observability-aiops # or: pipx install observability-aiops
83
+ observability-aiops init # wizard: pick platform (prometheus/grafana) + store the token (encrypted)
84
+ observability-aiops doctor # verify config, secrets, connectivity
85
+ observability-aiops overview # snapshot: firing alerts + targets up/down + rules erroring
86
+ observability-aiops query instant 'up' # run a PromQL instant query
87
+ observability-aiops alert rca # root-cause the firing alerts
88
+ ```
89
+
90
+ Run as an MCP server (stdio):
91
+
92
+ ```bash
93
+ export OBSERVABILITY_AIOPS_MASTER_PASSWORD=... # unlock secrets non-interactively
94
+ observability-aiops mcp
95
+ ```
96
+
97
+ ## Governance
98
+
99
+ Every MCP tool passes through the bundled `@governed_tool` harness:
100
+
101
+ - **Audit** — every call (params, result, status, duration, risk tier, approver,
102
+ rationale) is logged to `~/.observability-aiops/audit.db` (relocatable via
103
+ `OBSERVABILITY_AIOPS_HOME`).
104
+ - **Budget / runaway guard** — token and call budgets trip a circuit breaker on
105
+ tight poll/retry loops.
106
+ - **Risk tiers** — graduated autonomy; high-risk ops (`delete_dashboard`) can
107
+ require a named approver (`OBSERVABILITY_AUDIT_APPROVED_BY` /
108
+ `OBSERVABILITY_AUDIT_RATIONALE`).
109
+ - **Undo recording** — reversible writes capture the real before-state and record
110
+ an inverse descriptor (`create_silence`→expire, `update_dashboard`/
111
+ `delete_dashboard`→restore the captured prior model).
112
+
113
+ ## Supported scope & limitations
114
+
115
+ - **Platforms**: Prometheus HTTP API (+ a companion Alertmanager) and Grafana
116
+ HTTP API. Hosted/SaaS monitoring suites (Datadog, New Relic, enterprise NMS)
117
+ are deliberately **out of scope** for this tool.
118
+ - **Preview / mock-only.** All behaviour is validated against mocked
119
+ Prometheus/Grafana/Alertmanager responses. Both are free and open-source and
120
+ trivial to stand up in a lab (`docker run prom/prometheus`,
121
+ `grafana/grafana`), so `observability-aiops doctor` is the fastest live check
122
+ (Prometheus `/api/v1/status/buildinfo`, Grafana `/api/health`).
123
+
124
+ ## Missing a capability?
125
+
126
+ Want another read, an analysis tuned, or a platform capability that isn't here?
127
+ **Open an issue or a PR — feedback and contributions are welcome.**
@@ -0,0 +1,111 @@
1
+ <!-- mcp-name: io.github.AIops-tools/observability-aiops -->
2
+
3
+ # Observability AIops (preview)
4
+
5
+ > **Disclaimer**: Community-maintained open-source project. **Not affiliated with, endorsed by, or sponsored by the Prometheus or Grafana projects, Grafana Labs, or the Cloud Native Computing Foundation.** Prometheus, Alertmanager and Grafana are trademarks of their respective owners. MIT licensed.
6
+
7
+ Governed AI-ops for a **self-hosted observability stack** in one server —
8
+ **Prometheus** (HTTP API, PromQL, targets, rules, alerts), **Alertmanager**
9
+ (alerts + silences), and **Grafana** (dashboards, datasources, folders) — with a
10
+ **built-in governance harness**: unified audit log, policy engine, token/runaway
11
+ budget guard, undo-token recording, and graduated-autonomy risk tiers. One config
12
+ can span your whole stack; each target names its own `platform`.
13
+ **Preview — mock-validated only, not yet verified against a live stack.**
14
+
15
+ This is the **self-hosted-observability** complement to enterprise monitoring
16
+ suites: it speaks the open Prometheus/Grafana APIs an SRE actually runs, not a
17
+ vendor NMS.
18
+
19
+ ## What it does
20
+
21
+ Answers the questions an SRE actually repeats over a Prometheus/Grafana stack,
22
+ and guards the writes that follow:
23
+
24
+ - **PromQL + metadata** — instant and range queries, label-value enumeration, and
25
+ series metadata, all read-only and result-capped.
26
+ - **Scrape-target & rule health** — which targets are up/down (and *why*, from
27
+ `lastError`), which were dropped by relabeling, and which recording/alerting
28
+ rules are erroring.
29
+ - **Alerts & silences** — firing/pending Prometheus rule alerts, Alertmanager's
30
+ post-routing view, and its silences.
31
+ - **Grafana** — dashboards, datasources (+ health), and folders.
32
+ - **Flagship analyses** — transparent heuristics that show their numbers:
33
+ `firing_alert_rca` (join each firing alert to its rule expr → cause + action),
34
+ `target_scrape_health_analysis` (rank down/erroring scrapes → likely cause), and
35
+ `alert_noise_and_flap_analysis` (frequently-repeated / duplicate alerts →
36
+ dedup/rollup recommendation).
37
+ - **Governed writes** — create/expire Alertmanager silences (time-boxed), create
38
+ Grafana annotations, update/delete dashboards, and hot-reload the Prometheus
39
+ config — each audited, risk-tiered, `dry_run`-able, and the reversible ones
40
+ capture the **real fetched before-state** for undo.
41
+
42
+ ## Capability matrix (30 MCP tools)
43
+
44
+ | Group | Platform | Tools | Count | R/W |
45
+ |-------|----------|-------|:-----:|:---:|
46
+ | **Metrics** | Prometheus | `instant_query`, `range_query`, `label_values`, `series_metadata` | 4 | read |
47
+ | **Targets** | Prometheus | `list_targets`, `target_scrape_health`, `dropped_targets` | 3 | read |
48
+ | **Status** | Prometheus | `prometheus_config_status`, `prometheus_tsdb_status` | 2 | read |
49
+ | **Rules** | Prometheus | `list_rules`, `rule_health` | 2 | read |
50
+ | **Alerts** | Prometheus/Alertmanager | `firing_alerts`, `pending_alerts`, `alertmanager_alerts`, `list_silences` | 4 | read |
51
+ | **Grafana** | Grafana | `list_dashboards`, `get_dashboard`, `list_datasources`, `datasource_health`, `list_folders` | 5 | read |
52
+ | **Overview** | both | `observability_overview` | 1 | read |
53
+ | **Analyses** | Prometheus | `firing_alert_rca`, `target_scrape_health_analysis`, `alert_noise_and_flap_analysis` | 3 | read |
54
+ | **Writes** | Alertmanager | `create_silence`, `expire_silence` | 2 | write (med) |
55
+ | | Grafana | `create_annotation` | 1 | write (low) |
56
+ | | Grafana | `update_dashboard` | 1 | write (med) |
57
+ | | Grafana | `delete_dashboard` | 1 | write (**high**) |
58
+ | | Prometheus | `reload_prometheus_config` | 1 | write (med) |
59
+
60
+ The CLI exposes a convenience subset; the full 30-tool surface is via the MCP
61
+ server.
62
+
63
+ ## Quick start
64
+
65
+ ```bash
66
+ uv tool install observability-aiops # or: pipx install observability-aiops
67
+ observability-aiops init # wizard: pick platform (prometheus/grafana) + store the token (encrypted)
68
+ observability-aiops doctor # verify config, secrets, connectivity
69
+ observability-aiops overview # snapshot: firing alerts + targets up/down + rules erroring
70
+ observability-aiops query instant 'up' # run a PromQL instant query
71
+ observability-aiops alert rca # root-cause the firing alerts
72
+ ```
73
+
74
+ Run as an MCP server (stdio):
75
+
76
+ ```bash
77
+ export OBSERVABILITY_AIOPS_MASTER_PASSWORD=... # unlock secrets non-interactively
78
+ observability-aiops mcp
79
+ ```
80
+
81
+ ## Governance
82
+
83
+ Every MCP tool passes through the bundled `@governed_tool` harness:
84
+
85
+ - **Audit** — every call (params, result, status, duration, risk tier, approver,
86
+ rationale) is logged to `~/.observability-aiops/audit.db` (relocatable via
87
+ `OBSERVABILITY_AIOPS_HOME`).
88
+ - **Budget / runaway guard** — token and call budgets trip a circuit breaker on
89
+ tight poll/retry loops.
90
+ - **Risk tiers** — graduated autonomy; high-risk ops (`delete_dashboard`) can
91
+ require a named approver (`OBSERVABILITY_AUDIT_APPROVED_BY` /
92
+ `OBSERVABILITY_AUDIT_RATIONALE`).
93
+ - **Undo recording** — reversible writes capture the real before-state and record
94
+ an inverse descriptor (`create_silence`→expire, `update_dashboard`/
95
+ `delete_dashboard`→restore the captured prior model).
96
+
97
+ ## Supported scope & limitations
98
+
99
+ - **Platforms**: Prometheus HTTP API (+ a companion Alertmanager) and Grafana
100
+ HTTP API. Hosted/SaaS monitoring suites (Datadog, New Relic, enterprise NMS)
101
+ are deliberately **out of scope** for this tool.
102
+ - **Preview / mock-only.** All behaviour is validated against mocked
103
+ Prometheus/Grafana/Alertmanager responses. Both are free and open-source and
104
+ trivial to stand up in a lab (`docker run prom/prometheus`,
105
+ `grafana/grafana`), so `observability-aiops doctor` is the fastest live check
106
+ (Prometheus `/api/v1/status/buildinfo`, Grafana `/api/health`).
107
+
108
+ ## Missing a capability?
109
+
110
+ Want another read, an analysis tuned, or a platform capability that isn't here?
111
+ **Open an issue or a PR — feedback and contributions are welcome.**
@@ -0,0 +1,61 @@
1
+ # Observability AIops v0.1.0 — preview
2
+
3
+ Governed AI-ops for a **self-hosted observability stack** — **Prometheus** (HTTP
4
+ API + PromQL), **Alertmanager** (alerts + silences), and **Grafana** (dashboards,
5
+ datasources, folders) — for AI agents, with a built-in governance harness (audit,
6
+ policy, token/runaway budget, undo-token recording, graduated risk tiers) and an
7
+ encrypted credential store. Standalone — no external skill-family dependency. One
8
+ config can span your whole stack.
9
+
10
+ Positioned as the **self-hosted-observability** complement to enterprise
11
+ monitoring suites: it speaks the open Prometheus/Grafana APIs directly.
12
+
13
+ > **Preview / mock-only.** All behaviour is validated against mocked
14
+ > Prometheus/Grafana/Alertmanager responses; it has not been run against a live
15
+ > stack. Both platforms are free and open-source and trivial to stand up in a lab
16
+ > (`docker run prom/prometheus`, `grafana/grafana`). The fastest live check is
17
+ > `observability-aiops doctor`.
18
+
19
+ ## Highlights
20
+
21
+ - **30 MCP tools** (24 read, 6 write), every one wrapped with `@governed_tool`.
22
+ - **Metrics (Prometheus)** — `instant_query`, `range_query`, `label_values`,
23
+ `series_metadata`.
24
+ - **Targets & status** — `list_targets`, `target_scrape_health`,
25
+ `dropped_targets`, `prometheus_config_status`, `prometheus_tsdb_status`.
26
+ - **Rules** — `list_rules`, `rule_health`.
27
+ - **Alerts** — `firing_alerts`, `pending_alerts`, `alertmanager_alerts`,
28
+ `list_silences`.
29
+ - **Grafana** — `list_dashboards`, `get_dashboard`, `list_datasources`,
30
+ `datasource_health`, `list_folders`.
31
+ - **Overview** — `observability_overview` (platform-aware snapshot).
32
+ - **Writes** — `create_silence`/`expire_silence` (med, time-boxed),
33
+ `create_annotation` (low), `update_dashboard` (med),
34
+ `delete_dashboard` (**high**), `reload_prometheus_config` (med).
35
+ - **Three flagship analyses** — transparent heuristics that show their numbers:
36
+ `firing_alert_rca` (firing alert → rule expr → cause + action),
37
+ `target_scrape_health_analysis` (down/erroring scrapes ranked + classified),
38
+ and `alert_noise_and_flap_analysis` (noisy/duplicate alerts → dedup/rollup).
39
+ - **Encrypted secret store** (`~/.observability-aiops/secrets.enc`, Fernet +
40
+ scrypt) — Prometheus/Grafana bearer tokens, never plaintext on disk; legacy
41
+ `OBSERVABILITY_<TARGET>_TOKEN` env fallback.
42
+ - **Guarded writes** — the destructive op (`delete_dashboard`) requires dry-run +
43
+ an approver; reversible writes capture the **real fetched before-state** and
44
+ record an undo; silences are time-boxed (require a positive duration).
45
+ - **CLI** with an `init` platform-picking wizard, `secret` management, PromQL
46
+ `query`, `alert` (firing/silences/rca), and a platform-aware `doctor`.
47
+
48
+ ## Install
49
+
50
+ ```bash
51
+ uv tool install observability-aiops
52
+ observability-aiops init # pick platform (prometheus/grafana) + store the token
53
+ observability-aiops doctor
54
+ ```
55
+
56
+ ## Caveats
57
+
58
+ - Preview / mock-only: the Prometheus and Grafana HTTP API responses are mocked
59
+ and need live verification.
60
+ - Hosted/SaaS monitoring suites (Datadog, New Relic, enterprise NMS) are out of
61
+ scope by design.
@@ -0,0 +1,83 @@
1
+ # Security Policy
2
+
3
+ ## Disclaimer
4
+
5
+ Community-maintained open-source project. **Not affiliated with, endorsed by, or
6
+ sponsored by the Prometheus or Grafana projects, Grafana Labs, or the CNCF.**
7
+ Prometheus, Alertmanager and Grafana are trademarks of their respective owners.
8
+ Source is auditable under the MIT license.
9
+
10
+ ## Reporting Vulnerabilities
11
+
12
+ Report privately via a GitHub Security Advisory on
13
+ [github.com/AIops-tools/Observability-AIops](https://github.com/AIops-tools/Observability-AIops/security/advisories)
14
+ or email zhouwei008@gmail.com. Please do not open public issues for security
15
+ reports.
16
+
17
+ ## Security Design
18
+
19
+ ### Credential Management
20
+ - Per-target bearer tokens — the Grafana service-account/API token (required) or
21
+ the Prometheus bearer token (optional; many self-hosted deployments are
22
+ unauthenticated) — live **encrypted** in `~/.observability-aiops/secrets.enc`
23
+ (Fernet/AES-128 + scrypt-derived key; chmod 600), never in `config.yaml` and
24
+ never in source. The master password is never stored — only a per-store random
25
+ salt and the ciphertext are on disk.
26
+ - A legacy plaintext env var `OBSERVABILITY_<TARGET_NAME_UPPER>_TOKEN` is still
27
+ honoured as a fallback with a deprecation warning (migrate with
28
+ `observability-aiops secret migrate`).
29
+ - The token is held only in memory, sent as an `Authorization: Bearer` header,
30
+ and never logged or echoed. The config file holds only platform, scheme, host,
31
+ port, TLS setting, and an optional Alertmanager URL. PromQL is used only through
32
+ read endpoints (`/api/v1/query`, `/query_range`) — there is no write query path.
33
+
34
+ ### Governed Operations
35
+ Every MCP tool runs through the bundled `@governed_tool` harness
36
+ (`observability_aiops.governance`):
37
+ - **Audit** — every call logged to a local SQLite DB under `~/.observability-aiops/`
38
+ (relocatable via `OBSERVABILITY_AIOPS_HOME`), agent-attributed, secret-redacted.
39
+ - **Token/runaway budget** — hard ceilings (`OBSERVABILITY_MAX_TOOL_CALLS` /
40
+ `OBSERVABILITY_MAX_TOOL_SECONDS`) plus an on-by-default guard that trips a tight
41
+ poll/retry loop, preventing unbounded API consumption.
42
+ - **Graduated risk tiers** — `~/.observability-aiops/rules.yaml` `risk_tiers` gate
43
+ writes by environment/tag; the highest tiers require a recorded approver.
44
+ - **Undo-token recording** — reversible writes capture the real fetched BEFORE
45
+ state and record an inverse descriptor (e.g. `create_silence`→`expire_silence`,
46
+ `update_dashboard`/`delete_dashboard`→restore the captured prior model) so the
47
+ change can be rolled back.
48
+
49
+ ### State-Changing Operations
50
+ The destructive write — `delete_dashboard` — is `risk_level=high`, accepts a
51
+ `dry_run` preview, captures the full prior dashboard model **before** deleting,
52
+ and (under `risk_tiers`) requires a recorded approver
53
+ (`OBSERVABILITY_AUDIT_APPROVED_BY` + `OBSERVABILITY_AUDIT_RATIONALE`). Medium-risk
54
+ writes (`create_silence`/`expire_silence`, `update_dashboard`,
55
+ `reload_prometheus_config`) are audited and `dry_run`-able; silences are
56
+ **time-boxed** (require a positive duration, no open-ended silencing).
57
+ `create_annotation` is `risk_level=low`. Reversible writes capture before-state
58
+ and record an undo token.
59
+
60
+ ### SSL/TLS Verification
61
+ `verify_ssl` defaults to true; disable only for self-signed lab certificates.
62
+
63
+ ### Prompt-Injection Protection
64
+ All server-returned text (metric labels, alert messages, rule expressions,
65
+ dashboard titles, scrape errors) is passed through a `sanitize()` truncate +
66
+ control-character strip before reaching the agent.
67
+
68
+ ### Network Scope
69
+ No webhooks, no telemetry, no outbound calls beyond the configured Prometheus,
70
+ Alertmanager, and Grafana API endpoints. No post-install scripts or background
71
+ services.
72
+
73
+ ## Static Analysis
74
+
75
+ ```bash
76
+ uvx bandit -r observability_aiops/ mcp_server/
77
+ uv run ruff check .
78
+ ```
79
+
80
+ ## Supported Versions
81
+
82
+ The latest released version receives security fixes. This is a preview (0.x);
83
+ pin a version in production.
@@ -0,0 +1 @@
1
+ """MCP server package for observability-aiops."""
@@ -0,0 +1,103 @@
1
+ """Shared MCP server primitives: the FastMCP instance, connection helper,
2
+ error sanitisation, and the ``@tool_errors`` decorator.
3
+
4
+ Tool modules under ``mcp_server/tools/`` import ``mcp`` from here and register
5
+ their ``@mcp.tool()`` functions onto it. ``mcp_server/server.py`` then imports
6
+ those modules and runs the server.
7
+
8
+ Keep ``Optional[X]`` (never PEP 604 ``X | None``) in any FastMCP-reflected
9
+ tool signature — on older mcp/pydantic the union eval'd to ``types.UnionType``
10
+ crashes FastMCP's ``issubclass`` check.
11
+ """
12
+
13
+ import functools
14
+ import logging
15
+ import os
16
+ from collections.abc import Callable
17
+ from pathlib import Path
18
+ from typing import Any, Optional
19
+
20
+ from mcp.server.fastmcp import FastMCP
21
+
22
+ from observability_aiops.config import load_config
23
+ from observability_aiops.connection import ConnectionManager, ObservabilityApiError
24
+ from observability_aiops.governance import sanitize
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+ _DOCTOR_HINT = "Run 'observability-aiops doctor' to verify connectivity and credentials."
29
+
30
+
31
+ def _safe_error(exc: Exception, tool: str) -> str:
32
+ """Return an agent-safe error string; log full detail server-side only."""
33
+ logger.error("Tool %s failed", tool, exc_info=True)
34
+ _passthrough = (
35
+ ValueError,
36
+ FileNotFoundError,
37
+ KeyError,
38
+ PermissionError,
39
+ TimeoutError,
40
+ ConnectionError,
41
+ ObservabilityApiError,
42
+ )
43
+ if isinstance(exc, _passthrough):
44
+ return sanitize(str(exc), 300)
45
+ return f"{type(exc).__name__}: operation failed."
46
+
47
+
48
+ def tool_errors(shape: str = "dict") -> Callable:
49
+ """Wrap a tool body in the canonical try/except → ``_safe_error`` pattern.
50
+
51
+ Place this *between* ``@governed_tool`` and the function so the audit
52
+ decorator and FastMCP still see the original signature.
53
+ """
54
+
55
+ def decorator(func: Callable) -> Callable:
56
+ name = func.__name__
57
+
58
+ @functools.wraps(func)
59
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
60
+ try:
61
+ return func(*args, **kwargs)
62
+ except Exception as e: # noqa: BLE001 — sanitised below
63
+ msg = _safe_error(e, name)
64
+ if shape == "list":
65
+ return [{"error": msg, "hint": _DOCTOR_HINT}]
66
+ if shape == "str":
67
+ return f"Error: {msg} {_DOCTOR_HINT}"
68
+ return {"error": msg, "hint": _DOCTOR_HINT}
69
+
70
+ return wrapper
71
+
72
+ return decorator
73
+
74
+
75
+ mcp = FastMCP(
76
+ "observability-aiops",
77
+ instructions=(
78
+ "Self-hosted observability operations (preview) over Prometheus, "
79
+ "Alertmanager, and Grafana: PromQL instant/range queries, label + series "
80
+ "metadata; scrape-target health and dropped targets; recording/alerting "
81
+ "rules and their health; firing/pending alerts, Alertmanager alerts + "
82
+ "silences; Grafana dashboards, datasources, folders, and health; three "
83
+ "flagship analyses (firing-alert RCA, target-scrape-health, "
84
+ "alert-noise/flap); and governed writes — create/expire silence, create "
85
+ "annotation, update/delete dashboard, reload Prometheus config. Destructive "
86
+ "writes (delete dashboard) are risk=high with a dry_run preview and require "
87
+ "an approver. Reversible writes capture the real before-state and record an "
88
+ "undo. Every tool runs through the observability-aiops governance harness "
89
+ "(audit / budget / risk-tier / undo)."
90
+ ),
91
+ )
92
+
93
+ _conn_mgr: Optional[ConnectionManager] = None
94
+
95
+
96
+ def _get_connection(target: Optional[str] = None) -> Any:
97
+ """Return a Monitoring connection, lazily initialising the manager."""
98
+ global _conn_mgr # noqa: PLW0603
99
+ if _conn_mgr is None:
100
+ config_path_str = os.environ.get("OBSERVABILITY_AIOPS_CONFIG")
101
+ config_path = Path(config_path_str) if config_path_str else None
102
+ _conn_mgr = ConnectionManager(load_config(config_path))
103
+ return _conn_mgr.connect(target)
@@ -0,0 +1,39 @@
1
+ """MCP server wrapping observability-aiops operations (stdio transport).
2
+
3
+ Thin adapter layer: each ``@mcp.tool()`` function (in ``mcp_server/tools/``)
4
+ delegates to the ``observability_aiops`` ops package and is wrapped with the
5
+ observability-aiops ``@governed_tool`` harness (audit / budget / undo / risk-tier).
6
+
7
+ Standalone, self-governed self-hosted observability operations (preview) over
8
+ Prometheus, Alertmanager, and Grafana: PromQL, scrape-target + rule health,
9
+ alerts + silences, dashboards, flagship analyses, and governed writes.
10
+
11
+ Source: https://github.com/AIops-tools/Observability-AIops
12
+ License: MIT
13
+ """
14
+
15
+ import logging
16
+
17
+ from mcp_server._shared import _safe_error, mcp, tool_errors
18
+
19
+ # Importing the tool modules registers every @mcp.tool() onto the shared
20
+ # `mcp` instance. Order does not matter; each module is self-contained.
21
+ from mcp_server.tools import ( # noqa: F401 — side effects
22
+ alerts,
23
+ analysis,
24
+ grafana,
25
+ metrics,
26
+ overview,
27
+ prometheus,
28
+ rules,
29
+ targets,
30
+ writes,
31
+ )
32
+
33
+ __all__ = ["mcp", "main", "_safe_error", "tool_errors"]
34
+
35
+
36
+ def main() -> None:
37
+ """Run the MCP server over stdio."""
38
+ logging.basicConfig(level=logging.INFO)
39
+ mcp.run(transport="stdio")
@@ -0,0 +1 @@
1
+ """MCP tool modules. Importing each registers its @mcp.tool() functions."""