nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/observe/verification/extractor.py

@@ -0,0 +1,127 @@
+"""Extract declared metrics from service resources to form the metric contract."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from nthlayer_workers.observe.verification.models import DeclaredMetric, MetricContract, MetricSource
+
+
+@dataclass
+class Resource:
+    """Minimal resource representation for metric extraction.
+
+    Compatible with nthlayer.specs.models.Resource.
+    """
+
+    kind: str
+    spec: dict[str, Any]
+    name: str | None = None
+
+
+def extract_metric_contract(
+    service_name: str,
+    resources: list[Resource],
+) -> MetricContract:
+    """Extract all declared metrics from service resources."""
+    contract = MetricContract(service_name=service_name)
+
+    for resource in resources:
+        if resource.kind == "SLO":
+            contract.metrics.extend(_extract_slo_metrics(resource))
+        elif resource.kind == "Observability":
+            contract.metrics.extend(_extract_observability_metrics(resource))
+
+    # Deduplicate by metric name (keep first occurrence)
+    seen: set[str] = set()
+    unique_metrics = []
+    for metric in contract.metrics:
+        if metric.name not in seen:
+            seen.add(metric.name)
+            unique_metrics.append(metric)
+    contract.metrics = unique_metrics
+
+    return contract
+
+
+def _extract_slo_metrics(resource: Resource) -> list[DeclaredMetric]:
+    """Extract metrics from SLO indicator queries."""
+    metrics = []
+    spec = resource.spec or {}
+    indicators = spec.get("indicators", [])
+
+    for indicator in indicators:
+        success_ratio = indicator.get("success_ratio", {})
+        for query_key in ["total_query", "good_query", "error_query"]:
+            query = success_ratio.get(query_key, "")
+            if query:
+                for name in _extract_metrics_from_query(query):
+                    metrics.append(
+                        DeclaredMetric(
+                            name=name,
+                            source=MetricSource.SLO_INDICATOR,
+                            query=query,
+                            resource_name=resource.name,
+                        )
+                    )
+
+        latency_query = indicator.get("latency_query", "")
+        if latency_query:
+            for name in _extract_metrics_from_query(latency_query):
+                metrics.append(
+                    DeclaredMetric(
+                        name=name,
+                        source=MetricSource.SLO_INDICATOR,
+                        query=latency_query,
+                        resource_name=resource.name,
+                    )
+                )
+
+    return metrics
+
+
+def _extract_observability_metrics(resource: Resource) -> list[DeclaredMetric]:
+    """Extract metrics from Observability declarations."""
+    metrics = []
+    spec = resource.spec or {}
+
+    for metric_name in spec.get("metrics", []):
+        if isinstance(metric_name, str):
+            metrics.append(
+                DeclaredMetric(
+                    name=metric_name,
+                    source=MetricSource.OBSERVABILITY,
+                    resource_name=resource.name,
+                )
+            )
+
+    return metrics
+
+
+# PromQL functions and keywords to exclude from metric name extraction
+_PROMQL_KEYWORDS = frozenset({
+    "sum", "rate", "irate", "increase", "histogram_quantile", "avg", "min", "max",
+    "count", "stddev", "stdvar", "topk", "bottomk", "quantile", "count_values",
+    "group", "by", "without", "on", "ignoring", "group_left", "group_right",
+    "bool", "and", "or", "unless", "offset", "vector", "scalar", "abs", "absent",
+    "ceil", "floor", "round", "clamp", "clamp_max", "clamp_min", "day_of_month",
+    "day_of_week", "days_in_month", "delta", "deriv", "exp", "hour", "idelta",
+    "label_join", "label_replace", "ln", "log2", "log10", "minute", "month",
+    "predict_linear", "resets", "sort", "sort_desc", "sqrt", "time", "timestamp",
+    "year", "avg_over_time", "min_over_time", "max_over_time", "sum_over_time",
+    "count_over_time", "quantile_over_time", "stddev_over_time", "stdvar_over_time",
+    "last_over_time", "present_over_time", "changes", "le",
+})
+
+
+def _extract_metrics_from_query(query: str) -> list[str]:
+    """Extract metric names from a PromQL query."""
+    pattern = r"([a-zA-Z_:][a-zA-Z0-9_:]*)\s*(?:\{|\[|$)"
+    matches = re.findall(pattern, query)
+
+    return [
+        m for m in matches
+        if m.lower() not in _PROMQL_KEYWORDS and not m.startswith("__")
+    ]

nthlayer_workers/observe/verification/models.py

@@ -0,0 +1,101 @@
+"""Models for contract verification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class MetricSource(Enum):
+    """Source of a declared metric in service.yaml."""
+
+    SLO_INDICATOR = "slo_indicator"
+    OBSERVABILITY = "observability"
+    ALERT = "alert"
+
+
+@dataclass
+class DeclaredMetric:
+    """A metric declared in service.yaml."""
+
+    name: str
+    source: MetricSource
+    query: str | None = None
+    resource_name: str | None = None
+
+    @property
+    def is_critical(self) -> bool:
+        """SLO and alert metrics are critical for service reliability."""
+        return self.source in (MetricSource.SLO_INDICATOR, MetricSource.ALERT)
+
+
+@dataclass
+class MetricContract:
+    """The metric contract for a service."""
+
+    service_name: str
+    metrics: list[DeclaredMetric] = field(default_factory=list)
+
+    @property
+    def critical_metrics(self) -> list[DeclaredMetric]:
+        return [m for m in self.metrics if m.is_critical]
+
+    @property
+    def optional_metrics(self) -> list[DeclaredMetric]:
+        return [m for m in self.metrics if not m.is_critical]
+
+    @property
+    def unique_metric_names(self) -> set[str]:
+        return {m.name for m in self.metrics}
+
+
+@dataclass
+class VerificationResult:
+    """Result of verifying a single metric."""
+
+    metric: DeclaredMetric
+    exists: bool
+    error: str | None = None
+    sample_labels: dict | None = None
+
+    @property
+    def is_critical_failure(self) -> bool:
+        return not self.exists and self.metric.is_critical
+
+
+@dataclass
+class ContractVerificationResult:
+    """Result of verifying the entire metric contract."""
+
+    service_name: str
+    target_url: str
+    results: list[VerificationResult] = field(default_factory=list)
+
+    @property
+    def all_verified(self) -> bool:
+        return all(r.exists for r in self.results)
+
+    @property
+    def critical_verified(self) -> bool:
+        return all(r.exists for r in self.results if r.metric.is_critical)
+
+    @property
+    def missing_critical(self) -> list[VerificationResult]:
+        return [r for r in self.results if r.is_critical_failure]
+
+    @property
+    def missing_optional(self) -> list[VerificationResult]:
+        return [r for r in self.results if not r.exists and not r.metric.is_critical]
+
+    @property
+    def verified_count(self) -> int:
+        return sum(1 for r in self.results if r.exists)
+
+    @property
+    def exit_code(self) -> int:
+        """Exit code for CI/CD: 0=all verified, 1=optional missing, 2=critical missing."""
+        if self.missing_critical:
+            return 2
+        if self.missing_optional:
+            return 1
+        return 0

nthlayer_workers/observe/verification/verifier.py

@@ -0,0 +1,111 @@
+"""Prometheus metric verifier."""
+
+from __future__ import annotations
+
+import os
+
+import httpx
+import structlog
+
+from nthlayer_workers.observe.verification.models import (
+    ContractVerificationResult,
+    DeclaredMetric,
+    MetricContract,
+    VerificationResult,
+)
+
+logger = structlog.get_logger()
+
+
+class MetricVerifier:
+    """Verifies metrics exist in Prometheus."""
+
+    def __init__(
+        self,
+        prometheus_url: str,
+        username: str | None = None,
+        password: str | None = None,
+        timeout: float = 30.0,
+    ):
+        self.prometheus_url = prometheus_url.rstrip("/")
+        self.auth = (username, password) if username and password else None
+        self.timeout = timeout
+
+        if not self.auth:
+            env_user = os.environ.get("PROMETHEUS_USERNAME")
+            env_pass = os.environ.get("PROMETHEUS_PASSWORD")
+            if env_user and env_pass:
+                self.auth = (env_user, env_pass)
+
+    def verify_contract(self, contract: MetricContract) -> ContractVerificationResult:
+        """Verify all metrics in a contract exist."""
+        result = ContractVerificationResult(
+            service_name=contract.service_name,
+            target_url=self.prometheus_url,
+        )
+
+        for metric in contract.metrics:
+            verification = self.verify_metric(metric, contract.service_name)
+            result.results.append(verification)
+
+        return result
+
+    def verify_metric(self, metric: DeclaredMetric, service_name: str) -> VerificationResult:
+        """Verify a single metric exists in Prometheus."""
+        try:
+            exists, sample_labels = self._check_metric_exists(metric.name, service_name)
+            return VerificationResult(metric=metric, exists=exists, sample_labels=sample_labels)
+        except Exception as e:
+            logger.warning("metric_verification_failed", metric=metric.name, error=str(e))
+            return VerificationResult(metric=metric, exists=False, error=str(e))
+
+    def _check_metric_exists(
+        self, metric_name: str, service_name: str
+    ) -> tuple[bool, dict | None]:
+        """Check if a metric exists. Tries with service label, then without."""
+        selector = f'{metric_name}{{service="{service_name}"}}'
+        exists, labels = self._query_series(selector)
+        if exists:
+            return True, labels
+
+        selector = f"{metric_name}"
+        return self._query_series(selector)
+
+    def _query_series(self, selector: str) -> tuple[bool, dict | None]:
+        """Query Prometheus series API."""
+        url = f"{self.prometheus_url}/api/v1/series"
+        params: dict[str, str | int] = {"match[]": selector, "limit": 1}
+
+        try:
+            with httpx.Client(timeout=self.timeout) as client:
+                response = client.get(url, params=params, auth=self.auth)
+                response.raise_for_status()
+                data = response.json()
+
+                if data.get("status") == "success":
+                    series = data.get("data", [])
+                    if series:
+                        sample = series[0]
+                        labels = {k: v for k, v in sample.items() if k != "__name__"}
+                        return True, labels
+
+                return False, None
+
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 404:
+                return False, None
+            raise
+        except httpx.ConnectError as e:
+            raise ConnectionError(f"Cannot connect to Prometheus at {self.prometheus_url}") from e
+        except httpx.TimeoutException as e:
+            raise TimeoutError(f"Timeout connecting to Prometheus at {self.prometheus_url}") from e
+
+    def test_connection(self) -> bool:
+        """Test connection to Prometheus."""
+        try:
+            url = f"{self.prometheus_url}/api/v1/status/buildinfo"
+            with httpx.Client(timeout=10.0) as client:
+                response = client.get(url, auth=self.auth)
+                return response.status_code == 200
+        except Exception:
+            return False

nthlayer_workers/observe/worker.py

@@ -0,0 +1,332 @@
+"""Observe worker modules — SLO collection, drift detection, topology.
+
+Three modules implementing WorkerModule protocol, each on its own interval:
+- ObserveCollectModule: SLO collection + portfolio (60s)
+- ObserveDriftModule: drift analysis per service+SLO (1800s)
+- ObserveTopologyModule: dependency graph / blast radius (86400s)
+
+Deploy gate is CLI-only (not a worker module).
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+from nthlayer_common.api_client import CoreAPIClient
+from nthlayer_common.manifest.models import SLODefinition
+
+from nthlayer_workers.observe.assessment import create, to_dict
+from nthlayer_workers.observe.portfolio.aggregator import build_portfolio_from_results
+from nthlayer_workers.observe.slo.collector import (
+    SLOMetricCollector,
+    SLOResult,
+    ServiceSLO,
+    results_to_assessments,
+)
+
+logger = structlog.get_logger()
+
+
+# ---------------------------------------------------------------------------
+# ObserveCollectModule — SLO collection + portfolio
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ObserveCollectModule:
+    """SLO collection + portfolio synthesis.
+
+    Each process_cycle():
+    1. Fetches manifests from core's GET /manifests
+    2. Extracts ServiceSLO pairs
+    3. Queries Prometheus for SLI values via SLOMetricCollector
+    4. Converts results to slo_status assessments, submits to core
+    5. Builds portfolio from current-cycle results, submits portfolio_status assessment
+    """
+
+    client: CoreAPIClient
+    prometheus_url: str
+
+    def __post_init__(self):
+        # Constructed once to reuse connections and avoid re-reading
+        # Prometheus auth env vars on every cycle.
+        self._collector = SLOMetricCollector(self.prometheus_url)
+
+    @property
+    def name(self) -> str:
+        return "observe.collect"
+
+    async def restore_state(self, state: dict | None) -> None:
+        pass
+
+    async def process_cycle(self) -> None:
+        manifests = await _fetch_manifests(self.client)
+        if not manifests:
+            return
+
+        service_slos = _extract_service_slos(manifests)
+        if not service_slos:
+            return
+
+        by_service: dict[str, list[ServiceSLO]] = defaultdict(list)
+        for item in service_slos:
+            by_service[item.service].append(item)
+
+        # Collect SLOs and track results + assessment IDs for portfolio
+        all_results: dict[str, list[SLOResult]] = {}
+        slo_assessment_ids: list[str] = []
+        collection_errors = 0
+
+        for service, slos in sorted(by_service.items()):
+            try:
+                results = await self._collector.collect(slos)
+            except Exception:
+                logger.exception("observe_collect_failed", service=service)
+                collection_errors += 1
+                continue
+
+            all_results[service] = results
+            assessments = results_to_assessments(results, service)
+            for assessment in assessments:
+                submit_result = await self.client.submit_assessment(to_dict(assessment))
+                if submit_result.ok:
+                    slo_assessment_ids.append(assessment.id)
+                else:
+                    logger.warning(
+                        "observe_submit_failed",
+                        assessment_id=assessment.id,
+                        error=submit_result.error,
+                    )
+
+        # Portfolio: synthesise current-cycle results
+        if all_results:
+            portfolio = build_portfolio_from_results(all_results)
+            portfolio_assessment = create(
+                "portfolio_status",
+                # "__portfolio__" sentinel: portfolio is an org-level assessment,
+                # not per-service. Uses a reserved name that cannot collide with
+                # real service names (OpenSRM metadata.name cannot start with _).
+                "__portfolio__",
+                {
+                    "total_services": portfolio.total_services,
+                    "healthy_count": portfolio.healthy_count,
+                    "warning_count": portfolio.warning_count,
+                    "critical_count": portfolio.critical_count,
+                    "exhausted_count": portfolio.exhausted_count,
+                    "services": [
+                        {
+                            "service": svc.service,
+                            "overall_status": svc.overall_status,
+                            "slo_count": len(svc.slos),
+                        }
+                        for svc in portfolio.services
+                    ],
+                    "collection_errors": collection_errors,
+                    # parent_ids is best-effort: only includes IDs of
+                    # successfully submitted SLO assessments.
+                    "parent_ids": slo_assessment_ids,
+                },
+            )
+            await self.client.submit_assessment(to_dict(portfolio_assessment))
+
+    async def get_state(self) -> dict:
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# ObserveDriftModule — drift analysis per service+SLO
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ObserveDriftModule:
+    """Drift detection for all SLOs across all services.
+
+    Queries Prometheus range data directly for each service+SLO
+    combination from manifests. Produces drift_signal assessments.
+    """
+
+    client: CoreAPIClient
+    prometheus_url: str
+
+    @property
+    def name(self) -> str:
+        return "observe.drift"
+
+    async def restore_state(self, state: dict | None) -> None:
+        pass
+
+    async def process_cycle(self) -> None:
+        from nthlayer_workers.observe.drift import DriftAnalysisError, DriftAnalyzer
+
+        manifests = await _fetch_manifests(self.client)
+        if not manifests:
+            return
+
+        analyzer = DriftAnalyzer(self.prometheus_url)
+
+        for service, tier, slo_name, window in _drift_targets(manifests):
+            try:
+                result = await analyzer.analyze(service, tier, slo_name, window)
+            except DriftAnalysisError:
+                logger.warning("drift_analysis_failed", service=service, slo=slo_name)
+                continue
+            except Exception:
+                logger.exception("drift_unexpected_error", service=service, slo=slo_name)
+                continue
+
+            assessment = create("drift_signal", service, {
+                "slo_name": result.slo_name,
+                "severity": result.severity.value,
+                "pattern": result.pattern.value,
+                "slope_per_week": result.metrics.slope_per_week,
+                "days_until_exhaustion": result.projection.days_until_exhaustion,
+                "current_budget": result.metrics.current_budget,
+                "summary": result.summary,
+                "recommendation": result.recommendation,
+            })
+            await self.client.submit_assessment(to_dict(assessment))
+
+    async def get_state(self) -> dict:
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# ObserveTopologyModule — dependency graph / blast radius
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class ObserveTopologyModule:
+    """Dependency graph discovery and blast radius computation.
+
+    Queries Prometheus for dependency metrics, builds graph, computes
+    blast radius per service. Produces dependency_graph assessments.
+
+    Use case for v1.5: topology observation for planning purposes.
+    The "incident blast radius" use case lives in respond's correlation,
+    not here. Daily cadence is appropriate.
+
+    Scale note: per-service loop over calculate_blast_radius() builds a
+    single graph then evaluates each service. Fine for v1.5; may need a
+    bulk API if service count grows meaningfully (100+).
+    """
+
+    client: CoreAPIClient
+    prometheus_url: str
+
+    @property
+    def name(self) -> str:
+        return "observe.topology"
+
+    async def restore_state(self, state: dict | None) -> None:
+        pass
+
+    async def process_cycle(self) -> None:
+        from nthlayer_workers.observe.dependencies import DependencyDiscovery
+        from nthlayer_workers.observe.dependencies.providers.prometheus import PrometheusDepProvider
+
+        manifests = await _fetch_manifests(self.client)
+        if not manifests:
+            return
+
+        discovery = DependencyDiscovery()
+        discovery.add_provider(PrometheusDepProvider(url=self.prometheus_url))
+
+        service_names = [m["name"] for m in manifests if m.get("name")]
+        if not service_names:
+            return
+
+        try:
+            graph = await discovery.build_graph(service_names)
+        except Exception:
+            logger.exception("topology_graph_build_failed")
+            return
+
+        for m in manifests:
+            service = m.get("name")
+            if not service:
+                continue
+            try:
+                result = discovery.calculate_blast_radius(service, graph)
+            except Exception:
+                logger.exception("topology_blast_radius_failed", service=service)
+                continue
+
+            assessment = create("dependency_graph", service, {
+                "total_services_affected": result.total_services_affected,
+                "critical_services_affected": result.critical_services_affected,
+                "risk_level": result.risk_level,
+                "direct_downstream_count": len(result.direct_downstream),
+                "recommendation": result.recommendation,
+            })
+            await self.client.submit_assessment(to_dict(assessment))
+
+    async def get_state(self) -> dict:
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# Shared helpers
+# ---------------------------------------------------------------------------
+
+
+async def _fetch_manifests(client: CoreAPIClient) -> list[dict] | None:
+    """Fetch manifests from core API. Returns None on error, [] on empty catalogue."""
+    result = await client.get_manifests()
+    if not result.ok:
+        logger.warning("observe_manifest_fetch_failed", error=result.error)
+        return None
+    return result.data or []
+
+
+def _extract_service_slos(manifest_dicts: list[dict]) -> list[ServiceSLO]:
+    """Extract ServiceSLO pairs from core API manifest response dicts.
+
+    Parallel to spec_loader.load_specs(): that path reads YAML files on disk
+    (CLI use), this path parses manifest dicts from core's GET /manifests
+    (worker use).
+    """
+    results: list[ServiceSLO] = []
+    for m in manifest_dicts:
+        service = m.get("name")
+        if not service:
+            continue
+        for slo_dict in m.get("slos", []):
+            name = slo_dict.get("name")
+            target = slo_dict.get("target")
+            if not name or target is None:
+                continue
+            slo = SLODefinition(
+                name=name,
+                target=target,
+                slo_type=slo_dict.get("slo_type", "availability"),
+                window=slo_dict.get("window", "30d"),
+                indicator_query=slo_dict.get("indicator_query"),
+                total_query=slo_dict.get("total_query"),
+                good_query=slo_dict.get("good_query"),
+                judgment_type=slo_dict.get("judgment_type"),
+            )
+            results.append(ServiceSLO(service=service, slo=slo))
+    return results
+
+
+def _drift_targets(manifest_dicts: list[dict]) -> list[tuple[str, str, str, str | None]]:
+    """Extract (service, tier, slo_name, window) tuples for drift analysis."""
+    targets: list[tuple[str, str, str, str | None]] = []
+    for m in manifest_dicts:
+        service = m.get("name")
+        tier = m.get("tier", "standard")
+        if not service:
+            continue
+        for slo_dict in m.get("slos", []):
+            slo_name = slo_dict.get("name")
+            if not slo_name:
+                continue
+            window = slo_dict.get("window")
+            targets.append((service, tier, slo_name, window))
+    return targets

nthlayer_workers/respond/__init__.py

@@ -0,0 +1,2 @@
+"""Mayday — Multi-agent incident response."""
+__version__ = "0.1.0a1"

nthlayer_workers/respond/__main__.py

@@ -0,0 +1,4 @@
+"""Entry point for python -m nthlayer_respond."""
+from nthlayer_workers.respond.cli import main
+
+main()