nthlayer-workers 1.0.0__py3-none-any.whl

nthlayer_workers/observe/drift/patterns.py

@@ -0,0 +1,88 @@
+"""Pattern detection for drift analysis.
+
+Classifies drift patterns beyond simple linear trends:
+- Gradual decline/improvement
+- Step changes (sudden drops or improvements)
+- Volatile patterns
+- Stable (no significant trend)
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import numpy as np
+
+from nthlayer_workers.observe.drift.models import DriftPattern
+
+
+class PatternDetector:
+    """Detect drift patterns beyond simple linear trends."""
+
+    def __init__(
+        self,
+        step_change_threshold: float = 0.05,
+        volatility_variance_threshold: float = 0.01,
+        volatility_r_squared_threshold: float = 0.3,
+        slope_significance_threshold: float = 0.001,
+    ):
+        self.step_change_threshold = step_change_threshold
+        self.volatility_variance_threshold = volatility_variance_threshold
+        self.volatility_r_squared_threshold = volatility_r_squared_threshold
+        self.slope_significance_threshold = slope_significance_threshold
+
+    def detect(
+        self,
+        data: list[tuple[datetime, float]],
+        slope_per_second: float,
+        r_squared: float,
+    ) -> DriftPattern:
+        """Classify the drift pattern."""
+        if len(data) < 2:
+            return DriftPattern.STABLE
+
+        values = np.array([d[1] for d in data])
+        variance = float(np.var(values))
+
+        step_change = self._detect_step_change(data)
+        if step_change is not None:
+            return step_change
+
+        if (
+            r_squared < self.volatility_r_squared_threshold
+            and variance > self.volatility_variance_threshold
+        ):
+            return DriftPattern.VOLATILE
+
+        seconds_per_week = 7 * 24 * 60 * 60
+        weekly_slope = slope_per_second * seconds_per_week
+
+        if abs(weekly_slope) < self.slope_significance_threshold:
+            return DriftPattern.STABLE
+        elif weekly_slope < 0:
+            return DriftPattern.GRADUAL_DECLINE
+        else:
+            return DriftPattern.GRADUAL_IMPROVEMENT
+
+    def _detect_step_change(
+        self,
+        data: list[tuple[datetime, float]],
+    ) -> DriftPattern | None:
+        """Detect sudden step changes in the data."""
+        if len(data) < 2:
+            return None
+
+        max_time_window = 86400 * 1.5
+
+        for i in range(1, len(data)):
+            time_diff = (data[i][0] - data[i - 1][0]).total_seconds()
+            value_diff = data[i][1] - data[i - 1][1]
+
+            if time_diff < max_time_window:
+                if value_diff < -self.step_change_threshold:
+                    return DriftPattern.STEP_CHANGE_DOWN
+                elif value_diff > self.step_change_threshold:
+                    return DriftPattern.STEP_CHANGE_UP
+
+        return None
+

nthlayer_workers/observe/explanation.py

@@ -0,0 +1,118 @@
+"""ExplanationEngine — build human-readable budget explanations from assessments.
+
+Deterministic. No LLM. Pure arithmetic on assessment data.
+"""
+from __future__ import annotations
+
+from nthlayer_common.explanation import BudgetExplanation
+from nthlayer_workers.observe.assessment import Assessment
+from nthlayer_workers.observe.store import AssessmentFilter, AssessmentStore
+
+# Maps SLO assessment status to explanation severity.
+# ERROR/NO_DATA are data-quality issues (e.g. Prometheus unreachable),
+# not budget concerns — "warning" severity since they need attention
+# but don't indicate budget consumption.
+_STATUS_SEVERITY = {
+    "EXHAUSTED": "critical",
+    "CRITICAL": "critical",
+    "WARNING": "warning",
+    "ERROR": "warning",
+    "HEALTHY": "info",
+    "NO_DATA": "info",
+    "UNKNOWN": "info",
+}
+
+
+class ExplanationEngine:
+    """Build budget explanations from the assessment store."""
+
+    def explain_service(
+        self,
+        service: str,
+        store: AssessmentStore,
+        slo_filter: str | None = None,
+    ) -> list[BudgetExplanation]:
+        """Build explanations for a service from latest slo_status assessments."""
+        assessments = store.query(
+            AssessmentFilter(service=service, kind="slo_status", limit=0)
+        )
+
+        # Deduplicate: keep latest per SLO name (query returns desc by timestamp)
+        seen: set[str] = set()
+        latest: list[Assessment] = []
+        for a in assessments:
+            slo_name = a.data.get("slo_name", "unknown")
+            if slo_name not in seen:
+                seen.add(slo_name)
+                latest.append(a)
+
+        if slo_filter:
+            latest = [a for a in latest if a.data.get("slo_name") == slo_filter]
+
+        return [self._explain_slo(service, a) for a in latest]
+
+    def _explain_slo(self, service: str, assessment: Assessment) -> BudgetExplanation:
+        data = assessment.data
+        slo_name = data.get("slo_name", "unknown")
+        status = data.get("status", "UNKNOWN")
+        pct = data.get("percent_consumed", 0.0) or 0.0
+        burned = data.get("burned_minutes", 0.0) or 0.0
+        total = data.get("total_budget_minutes", 0.0) or 0.0
+        sli = data.get("current_sli", 0.0) or 0.0
+        obj = data.get("objective", 0.0) or 0.0
+        window = data.get("window", "30d")
+        remaining = max(0, total - burned)
+        severity = _STATUS_SEVERITY.get(status, "info")
+
+        # Headline
+        status_desc = {
+            "EXHAUSTED": "budget exhausted",
+            "CRITICAL": "near exhaustion",
+            "WARNING": "approaching threshold",
+        }
+        desc = status_desc.get(status, "within budget")
+        headline = f"{slo_name}: {pct:.0f}% consumed — {desc} ({status})"
+
+        # Body
+        body = (
+            f"Window: {window}. "
+            f"Budget: {total:.0f} min total, {burned:.0f} min consumed, "
+            f"{remaining:.0f} min remaining. "
+            f"Current SLI: {sli:.4f} (target: {obj:.4f})."
+        )
+
+        # Causes
+        causes: list[str] = []
+        if pct > 80:
+            causes.append(
+                "Budget consumption exceeds 80% — sustained error rate above target"
+            )
+        if sli < obj and obj > 0:
+            gap = (obj - sli) * 100
+            causes.append(
+                f"Current SLI ({sli:.4f}) is {gap:.2f}pp below target ({obj:.4f})"
+            )
+
+        # Actions
+        actions: list[str] = []
+        if status == "EXHAUSTED":
+            actions.append(
+                "Deployment gate will block — resolve underlying issue before deploying"
+            )
+        if status in ("CRITICAL", "EXHAUSTED"):
+            actions.append("Investigate root cause of elevated error rate")
+            actions.append("Consider freezing deployments until budget recovers")
+        elif status == "WARNING":
+            actions.append(
+                "Monitor trend — investigate if consumption continues to rise"
+            )
+
+        return BudgetExplanation(
+            service=service,
+            slo_name=slo_name,
+            headline=headline,
+            body=body,
+            causes=causes,
+            recommended_actions=actions,
+            severity=severity,
+        )

nthlayer_workers/observe/gate/__init__.py

@@ -0,0 +1,39 @@
+"""Deployment gate evaluation."""
+
+from nthlayer_workers.observe.gate.conditions import (
+    get_current_context,
+    is_business_hours,
+    is_freeze_period,
+    is_peak_traffic,
+    is_weekday,
+)
+from nthlayer_workers.observe.gate.correlator import (
+    CorrelationInput,
+    CorrelationResult,
+    correlate,
+)
+from nthlayer_workers.observe.gate.evaluator import (
+    GateCheckResult,
+    check_deploy,
+)
+from nthlayer_workers.observe.gate.policies import (
+    ConditionEvaluator,
+    EvaluationResult,
+    PolicyContext,
+)
+
+__all__ = [
+    "ConditionEvaluator",
+    "CorrelationInput",
+    "CorrelationResult",
+    "EvaluationResult",
+    "GateCheckResult",
+    "PolicyContext",
+    "check_deploy",
+    "correlate",
+    "get_current_context",
+    "is_business_hours",
+    "is_freeze_period",
+    "is_peak_traffic",
+    "is_weekday",
+]

nthlayer_workers/observe/gate/conditions.py

@@ -0,0 +1,92 @@
+"""Built-in condition functions for policy evaluation.
+
+Provides time-based, date-based, and service-based conditions.
+All functions are pure — no external data access.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Any
+
+
+def get_current_context(
+    budget_remaining: float = 100.0,
+    budget_consumed: float = 0.0,
+    burn_rate: float = 1.0,
+    tier: str = "standard",
+    environment: str = "prod",
+    downstream_count: int = 0,
+    high_criticality_downstream: int = 0,
+    now: datetime | None = None,
+) -> dict[str, Any]:
+    """Build context dictionary for condition evaluation."""
+    if now is None:
+        now = datetime.now()
+
+    return {
+        "hour": now.hour,
+        "minute": now.minute,
+        "weekday": now.weekday() < 5,
+        "day_of_week": now.weekday(),
+        "date": now.date().isoformat(),
+        "month": now.month,
+        "day": now.day,
+        "year": now.year,
+        "budget_remaining": budget_remaining,
+        "budget_consumed": budget_consumed,
+        "burn_rate": burn_rate,
+        "tier": tier,
+        "environment": environment,
+        "env": environment,
+        "downstream_count": downstream_count,
+        "high_criticality_downstream": high_criticality_downstream,
+    }
+
+
+def is_business_hours(
+    now: datetime | None = None,
+    start_hour: int = 9,
+    end_hour: int = 17,
+) -> bool:
+    """Check if current time is within business hours (Mon-Fri, 9-17)."""
+    if now is None:
+        now = datetime.now()
+    if now.weekday() >= 5:
+        return False
+    return start_hour <= now.hour < end_hour
+
+
+def is_weekday(now: datetime | None = None) -> bool:
+    """Check if current day is a weekday (Mon-Fri)."""
+    if now is None:
+        now = datetime.now()
+    return now.weekday() < 5
+
+
+def is_freeze_period(
+    start_date: str,
+    end_date: str,
+    now: datetime | None = None,
+) -> bool:
+    """Check if current date is within a freeze period."""
+    if now is None:
+        now = datetime.now()
+    try:
+        start = date.fromisoformat(start_date)
+        end = date.fromisoformat(end_date)
+        return start <= now.date() <= end
+    except ValueError:
+        return False
+
+
+def is_peak_traffic(
+    now: datetime | None = None,
+    peak_hours: list[tuple[int, int]] | None = None,
+) -> bool:
+    """Check if current time is during peak traffic hours."""
+    if now is None:
+        now = datetime.now()
+    if peak_hours is None:
+        peak_hours = [(10, 12), (14, 16)]
+    return any(start <= now.hour < end for start, end in peak_hours)

nthlayer_workers/observe/gate/correlator.py

@@ -0,0 +1,154 @@
+"""Deployment correlation engine — 5-factor weighted scoring.
+
+Deterministic heuristic that correlates deployments with error budget burns.
+Adapted from nthlayer.slos.correlator to work with pre-computed inputs
+instead of SLORepository queries.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+# Confidence thresholds
+HIGH_CONFIDENCE = 0.7
+MEDIUM_CONFIDENCE = 0.5
+LOW_CONFIDENCE = 0.3
+BLOCKING_CONFIDENCE = 0.8
+
+# Factor weights
+WEIGHTS = {
+    "burn_rate": 0.35,
+    "proximity": 0.25,
+    "magnitude": 0.15,
+    "dependency": 0.15,
+    "history": 0.10,
+}
+
+
+@dataclass
+class CorrelationInput:
+    """Pre-computed input data for correlation scoring.
+
+    Callers gather this data from assessments or other sources
+    before calling the correlator.
+    """
+
+    deployment_id: str
+    service: str
+    deploy_time: datetime
+    burn_detected_at: datetime  # when burn was first detected (for proximity scoring)
+    burn_rate_before: float  # burn rate per minute before deploy window
+    burn_rate_after: float   # burn rate per minute after deploy window
+    burn_minutes: float      # total burn in after window
+    is_same_service: bool = False   # deployment targets same service as affected SLO
+    is_direct_upstream: bool = False  # deployment targets direct upstream dependency
+    is_transitive_upstream: bool = False  # deployment targets transitive upstream
+    is_yaml_downstream: bool = False  # service listed in YAML downstream_services
+    recent_deploy_count: int = 0   # total deploys for service in history window
+    prior_correlations: int = 0    # prior medium+ correlations in history window
+
+
+@dataclass
+class CorrelationResult:
+    """Result of correlation analysis."""
+
+    deployment_id: str
+    service: str
+    burn_minutes: float
+    confidence: float
+    method: str = "time_window_analysis"
+    details: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def confidence_label(self) -> str:
+        if self.confidence >= HIGH_CONFIDENCE:
+            return "HIGH"
+        elif self.confidence >= MEDIUM_CONFIDENCE:
+            return "MEDIUM"
+        elif self.confidence >= LOW_CONFIDENCE:
+            return "LOW"
+        return "NONE"
+
+
+def correlate(inp: CorrelationInput) -> CorrelationResult:
+    """Correlate a deployment with error budget burns using 5-factor scoring.
+
+    All data is pre-computed in CorrelationInput — no async, no DB queries.
+
+    Factors:
+    - burn_rate (0.35): Spike in burn rate after deploy
+    - proximity (0.25): Time proximity (exponential decay, half-life ~30min)
+    - magnitude (0.15): Absolute burn amount
+    - dependency (0.15): Relationship between deploying and affected service
+    - history (0.10): Historical correlation pattern
+    """
+    burn_rate_score = _calculate_burn_rate_score(inp.burn_rate_before, inp.burn_rate_after)
+    proximity_score = _calculate_proximity_score(inp.deploy_time, inp.burn_detected_at)
+    magnitude_score = _calculate_magnitude_score(inp.burn_minutes)
+    dependency_score = _calculate_dependency_score(inp)
+    history_score = _calculate_history_score(inp.recent_deploy_count, inp.prior_correlations)
+
+    confidence = (
+        WEIGHTS["burn_rate"] * burn_rate_score
+        + WEIGHTS["proximity"] * proximity_score
+        + WEIGHTS["magnitude"] * magnitude_score
+        + WEIGHTS["dependency"] * dependency_score
+        + WEIGHTS["history"] * history_score
+    )
+
+    return CorrelationResult(
+        deployment_id=inp.deployment_id,
+        service=inp.service,
+        burn_minutes=inp.burn_minutes,
+        confidence=confidence,
+        details={
+            "burn_rate_before": inp.burn_rate_before,
+            "burn_rate_after": inp.burn_rate_after,
+            "burn_rate_score": burn_rate_score,
+            "proximity_score": proximity_score,
+            "magnitude_score": magnitude_score,
+            "dependency_score": dependency_score,
+            "history_score": history_score,
+        },
+    )
+
+
+def _calculate_burn_rate_score(before_rate: float, after_rate: float) -> float:
+    """Spike ratio: 5x or more = 1.0. No baseline → absolute rate."""
+    if before_rate == 0:
+        return min(after_rate / 0.1, 1.0)
+    return min((after_rate / before_rate) / 5.0, 1.0)
+
+
+def _calculate_proximity_score(deployed_at: datetime, burn_detected_at: datetime) -> float:
+    """Exponential decay: half-life ~30 minutes."""
+    elapsed = abs((burn_detected_at - deployed_at).total_seconds()) / 60.0
+    return math.exp(-elapsed / 30.0)
+
+
+def _calculate_magnitude_score(burn_minutes: float) -> float:
+    """Absolute burn: 10+ minutes = 1.0."""
+    return min(burn_minutes / 10.0, 1.0)
+
+
+def _calculate_dependency_score(inp: CorrelationInput) -> float:
+    """Relationship score: same=1.0, direct upstream=1.0, transitive=0.4, yaml=0.6, none=0.0."""
+    if inp.is_same_service:
+        return 1.0
+    if inp.is_direct_upstream:
+        return 1.0
+    if inp.is_transitive_upstream:
+        return 0.4
+    if inp.is_yaml_downstream:
+        return 0.6
+    return 0.0
+
+
+def _calculate_history_score(recent_deploy_count: int, prior_correlations: int) -> float:
+    """Repeat offender penalty: fraction of recent deploys with medium+ correlation."""
+    if recent_deploy_count == 0:
+        return 0.0
+    return min(prior_correlations / recent_deploy_count, 1.0)

nthlayer_workers/observe/gate/evaluator.py

@@ -0,0 +1,192 @@
+"""Deployment gate evaluation — decides APPROVED/WARNING/BLOCKED from assessments.
+
+Reads slo_status assessments to determine error budget status,
+then applies tier-based thresholds to make gate decisions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from nthlayer_common.gate_models import GatePolicy, GateResult
+from nthlayer_common.tiers import TIER_CONFIGS
+
+from nthlayer_workers.observe.store import AssessmentFilter, AssessmentStore
+
+
+# Default thresholds from tier config
+THRESHOLDS: dict[str, dict[str, float | None]] = {
+    tier: {
+        "warning": config.error_budget_warning_pct,
+        "blocking": config.error_budget_blocking_pct,
+    }
+    for tier, config in TIER_CONFIGS.items()
+}
+
+
+@dataclass
+class GateCheckResult:
+    """Result of a deployment gate check."""
+
+    service: str
+    tier: str
+    result: GateResult
+    budget_remaining_pct: float
+    warning_threshold: float
+    blocking_threshold: float | None
+    message: str
+    recommendations: list[str]
+    slo_count: int = 0
+
+
+def check_deploy(
+    service: str,
+    tier: str,
+    store: AssessmentStore,
+    policy: GatePolicy | None = None,
+) -> GateCheckResult:
+    """Check if deployment should be allowed based on assessment data.
+
+    Reads recent slo_status assessments for the service, computes
+    aggregate budget consumption, and applies tier thresholds.
+
+    Args:
+        service: Service name
+        tier: Service tier (critical, standard, low)
+        store: Assessment store with slo_status assessments
+        policy: Optional custom GatePolicy to override tier defaults
+
+    Returns:
+        GateCheckResult with APPROVED/WARNING/BLOCKED decision
+    """
+    # Get recent slo_status assessments for this service
+    assessments = store.query(
+        AssessmentFilter(service=service, kind="slo_status")
+    )
+
+    if not assessments:
+        return GateCheckResult(
+            service=service,
+            tier=tier,
+            result=GateResult.APPROVED,
+            budget_remaining_pct=100.0,
+            warning_threshold=_get_warning_threshold(tier, policy),
+            blocking_threshold=_get_blocking_threshold(tier, policy),
+            message="No SLO assessments found — gate approved by default",
+            recommendations=[],
+            slo_count=0,
+        )
+
+    # Compute aggregate budget consumption from latest assessments per SLO
+    latest_by_slo: dict[str, dict[str, Any]] = {}
+    for a in assessments:
+        slo_name = a.data.get("slo_name", "unknown")
+        if slo_name not in latest_by_slo:
+            latest_by_slo[slo_name] = a.data
+
+    consumed_values = [
+        d.get("percent_consumed", 0.0)
+        for d in latest_by_slo.values()
+        if d.get("percent_consumed") is not None
+    ]
+
+    if not consumed_values:
+        avg_consumed = 0.0
+    else:
+        avg_consumed = sum(consumed_values) / len(consumed_values)
+
+    budget_remaining_pct = max(0.0, 100.0 - avg_consumed)
+
+    # Get thresholds
+    warning_threshold = _get_warning_threshold(tier, policy)
+    blocking_threshold = _get_blocking_threshold(tier, policy)
+
+    # Evaluate gate
+    result, message, recommendations = _evaluate_thresholds(
+        budget_remaining_pct, warning_threshold, blocking_threshold, tier, policy
+    )
+
+    return GateCheckResult(
+        service=service,
+        tier=tier,
+        result=result,
+        budget_remaining_pct=budget_remaining_pct,
+        warning_threshold=warning_threshold,
+        blocking_threshold=blocking_threshold,
+        message=message,
+        recommendations=recommendations,
+        slo_count=len(latest_by_slo),
+    )
+
+
+def _get_warning_threshold(tier: str, policy: GatePolicy | None) -> float:
+    """Get warning threshold from policy or tier defaults."""
+    if policy and policy.warning is not None:
+        return policy.warning
+    defaults = THRESHOLDS.get(tier, THRESHOLDS["standard"])
+    return defaults.get("warning") or 20.0
+
+
+def _get_blocking_threshold(tier: str, policy: GatePolicy | None) -> float | None:
+    """Get blocking threshold from policy or tier defaults."""
+    if policy and policy.blocking is not None:
+        return policy.blocking
+    defaults = THRESHOLDS.get(tier, THRESHOLDS["standard"])
+    return defaults.get("blocking")
+
+
+def _evaluate_thresholds(
+    budget_remaining_pct: float,
+    warning_threshold: float,
+    blocking_threshold: float | None,
+    tier: str,
+    policy: GatePolicy | None,
+) -> tuple[GateResult, str, list[str]]:
+    """Apply threshold logic to determine gate result."""
+    recommendations: list[str] = []
+
+    # Check exhaustion
+    if budget_remaining_pct <= 0:
+        if policy and policy.on_exhausted:
+            if "freeze_deploys" in policy.on_exhausted:
+                return (
+                    GateResult.BLOCKED,
+                    "Error budget exhausted (0% remaining). Deployment frozen per policy.",
+                    ["Wait for error budget to recover before deploying"],
+                )
+            if "require_approval" in policy.on_exhausted:
+                return (
+                    GateResult.WARNING,
+                    "Error budget exhausted. Manual approval required per policy.",
+                    ["Get explicit approval before proceeding"],
+                )
+
+    # Check blocking threshold
+    if blocking_threshold is not None and budget_remaining_pct <= blocking_threshold:
+        return (
+            GateResult.BLOCKED,
+            f"Error budget critical: {budget_remaining_pct:.1f}% remaining (blocking threshold: {blocking_threshold}%)",
+            [
+                f"Budget below blocking threshold ({blocking_threshold}%)",
+                "Investigate ongoing issues before deploying",
+                "Consider waiting for budget recovery",
+            ],
+        )
+
+    # Check warning threshold
+    if budget_remaining_pct <= warning_threshold:
+        recommendations.append(f"Budget below warning threshold ({warning_threshold}%)")
+        recommendations.append("Monitor closely after deployment")
+        return (
+            GateResult.WARNING,
+            f"Error budget low: {budget_remaining_pct:.1f}% remaining (warning threshold: {warning_threshold}%)",
+            recommendations,
+        )
+
+    # Approved
+    return (
+        GateResult.APPROVED,
+        f"Error budget healthy: {budget_remaining_pct:.1f}% remaining",
+        [],
+    )