scryml 0.1.0__py3-none-any.whl

scry/__init__.py

@@ -0,0 +1,6 @@
+# Description: Scry predicts infrastructure failure states from a stream of metrics.
+# Description: Top-level package; exposes the package version.
+
+"""Scry: predict infrastructure failure states from a stream of metrics."""
+
+__version__ = "0.1.0"

scry/api/__init__.py

@@ -0,0 +1,25 @@
+# Description: FastAPI prediction API for exposing cluster predictions
+# Description: and recommended remediation actions.
+
+"""FastAPI prediction service for Scry."""
+
+from scry.api.main import app, create_app
+from scry.api.predictor import Predictor
+from scry.api.schemas import (
+    ClusterInfo,
+    HealthResponse,
+    PredictionRequest,
+    PredictionResponse,
+    get_cluster_info,
+)
+
+__all__ = [
+    "app",
+    "create_app",
+    "Predictor",
+    "PredictionRequest",
+    "PredictionResponse",
+    "HealthResponse",
+    "ClusterInfo",
+    "get_cluster_info",
+]

scry/api/forecaster.py

@@ -0,0 +1,110 @@
+# Description: Singleton wrapper around ChronosForecaster for API use.
+# Description: Lazy-loads the Chronos model on first request, reuses across calls.
+
+"""Forecaster service for the API."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import numpy as np
+import torch
+
+log = logging.getLogger(__name__)
+
+
+class Forecaster:
+    """Singleton forecaster service wrapping ChronosForecaster.
+
+    Lazy-loads the Chronos foundation model on first call.
+    Provides a forecast_metrics method that takes multiple metrics
+    and returns per-metric, per-horizon forecasts with quantiles.
+
+    Args:
+        model_id: HuggingFace model ID for Chronos.
+        device: Inference device (cpu, cuda).
+        horizons: Forecast horizons in timesteps.
+        quantile_levels: Quantile levels for prediction intervals.
+    """
+
+    def __init__(
+        self,
+        model_id: str = "amazon/chronos-bolt-tiny",
+        device: str = "cpu",
+        horizons: list[int] | None = None,
+        quantile_levels: list[float] | None = None,
+    ) -> None:
+        self.model_id = model_id
+        self.device = device
+        self.horizons = horizons or [15, 60, 240, 1440]
+        self.quantile_levels = quantile_levels or [0.1, 0.5, 0.9]
+        self._forecaster: Any = None
+
+    def _load(self) -> None:
+        """Lazy-load the ChronosForecaster on first use."""
+        if self._forecaster is not None:
+            return
+
+        from scry.model.forecasting.chronos_wrapper import ChronosForecaster
+
+        log.info(
+            "Loading ChronosForecaster model=%s device=%s horizons=%s",
+            self.model_id,
+            self.device,
+            self.horizons,
+        )
+        self._forecaster = ChronosForecaster(
+            model_id=self.model_id,
+            device=self.device,
+            horizons=self.horizons,
+            quantile_levels=self.quantile_levels,
+        )
+        log.info("ChronosForecaster loaded")
+
+    @property
+    def is_loaded(self) -> bool:
+        """Whether the Chronos model is loaded."""
+        return self._forecaster is not None
+
+    def forecast_metrics(
+        self,
+        metrics: dict[str, list[float]],
+    ) -> list[dict[str, Any]]:
+        """Forecast multiple metrics and extract values at configured horizons.
+
+        Args:
+            metrics: Dict mapping metric names to historical time series values.
+
+        Returns:
+            List of per-metric forecast dicts, each containing:
+                - metric_name: str
+                - horizons: list of {horizon, median, lower, upper} dicts
+        """
+        self._load()
+
+        metric_names = list(metrics.keys())
+        contexts = [
+            torch.tensor(values, dtype=torch.float32)
+            for values in metrics.values()
+        ]
+
+        batch_forecast = self._forecaster.forecast_batch(contexts)
+        at_horizons = self._forecaster.extract_at_horizons(batch_forecast)
+
+        results = []
+        for i, name in enumerate(metric_names):
+            horizon_results = []
+            for j, h in enumerate(self.horizons):
+                horizon_results.append({
+                    "horizon": h,
+                    "median": float(np.round(at_horizons["median"][i, j], 4)),
+                    "lower": float(np.round(at_horizons["lower"][i, j], 4)),
+                    "upper": float(np.round(at_horizons["upper"][i, j], 4)),
+                })
+            results.append({
+                "metric_name": name,
+                "horizons": horizon_results,
+            })
+
+        return results

scry/api/main.py

@@ -0,0 +1,423 @@
+# Description: FastAPI application for the prediction service.
+# Description: Exposes /health, /predict, /predict/lookup, /clusters, /forecast, /drift, /anomaly, /accuracy.
+
+"""FastAPI application for cluster prediction."""
+
+import atexit
+import logging
+import os
+import time
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Query
+
+from scry.api.forecaster import Forecaster
+from scry.api.predictor import Predictor
+from scry.api.schemas import (
+    ClusterInfo,
+    DetailedHealthResponse,
+    ForecastRequest,
+    ForecastResponse,
+    HealthResponse,
+    MetricForecast,
+    PredictionRequest,
+    PredictionResponse,
+    get_cluster_info,
+)
+from scry.utils.config import get_config
+from scry.utils.tracing import get_tracer, setup_tracing, shutdown_tracing
+
+tracer = get_tracer(__name__)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+
+# API version
+VERSION = "0.1.0"
+
+
+def _data_uri() -> str | None:
+    """The configured object-store data URI, if any (env SCRY_DATA_URI / config)."""
+    return getattr(get_config(), "data_uri", None) or os.environ.get("SCRY_DATA_URI")
+
+
+def _datasource_descriptor() -> str | None:
+    """Describe the configured data source for diagnostics, or None if unconfigured.
+
+    Object storage (SCRY_DATA_URI) is the default. The HttpIngest adapter is used
+    only when the ``logicmonitor`` extra is installed.
+    """
+    uri = _data_uri()
+    if uri:
+        return f"object-store: {uri}"
+    try:
+        import scry.data.sources.http_ingest  # noqa: F401
+    except ImportError:
+        return None
+    return f"httpingest: {get_config().httpingest_url}"
+
+
+async def _resource_metrics(resource_id: str, lookback_days: int = 30) -> Any:
+    """Fetch recent metrics for a resource through the configured data source.
+
+    Returns a canonical-schema DataFrame filtered to the resource, or None when no
+    data source is configured. Object storage is the default; the HttpIngest
+    adapter is used only when the ``logicmonitor`` extra is installed.
+    """
+    from scry.data import DataFetcher
+
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(days=lookback_days)
+    profile_name = os.environ.get("SCRY_PROFILE")  # None -> features.yaml default
+
+    uri = _data_uri()
+    if uri:
+        fetcher = DataFetcher.from_object_store(uri)
+        df = await fetcher.get_metrics_dataframe(start, end, profile_name)
+    else:
+        try:
+            from scry.data import HttpIngestClient
+        except ImportError:
+            return None
+        async with HttpIngestClient(base_url=get_config().httpingest_url) as client:
+            fetcher = DataFetcher.from_http_client(client)
+            df = await fetcher.get_metrics_dataframe(start, end, profile_name)
+
+    if df.empty:
+        return df
+
+    needle = resource_id.lower()
+    mask = df["resource_id"].astype(str).str.lower().str.contains(needle, na=False) | df[
+        "host_name"
+    ].astype(str).str.lower().str.contains(needle, na=False)
+    return df[mask]
+
+
+def _split_by_profile(
+    df: Any,
+    profile_name: str | None = None,
+) -> tuple[dict[str, list[float]], dict[str, list[int]]]:
+    """Group canonical metric rows into numerical/categorical series per the active profile."""
+    from scry.config.loader import get_profile
+
+    try:
+        profile = get_profile(profile_name)
+    except (FileNotFoundError, ValueError):
+        return {}, {}
+
+    num_set = set(profile.numerical_features)
+    cat_set = set(profile.categorical_features)
+    numerical: dict[str, list[float]] = {}
+    categorical: dict[str, list[int]] = {}
+
+    ordered = df.sort_values("timestamp")
+    for name, group in ordered.groupby("metric_name"):
+        values = group["value"].dropna().tolist()
+        if not values:
+            continue
+        if name in num_set:
+            numerical[name] = [float(v) for v in values]
+        elif name in cat_set:
+            categorical[name] = [int(v) for v in values]
+    return numerical, categorical
+
+
+def create_app(model_path: str | None = None) -> FastAPI:
+    """Create the FastAPI application.
+
+    Args:
+        model_path: Path to the model file. If None, uses the MODEL_PATH env var.
+
+    Returns:
+        Configured FastAPI application.
+    """
+    app = FastAPI(
+        title="Scry Predictive API",
+        description="Predicts infrastructure failure states from a stream of metrics",
+        version=VERSION,
+    )
+
+    # Optional tracing (no-op unless the 'otel' extra is installed and enabled).
+    setup_tracing(app)
+    atexit.register(shutdown_tracing)
+
+    # Load model
+    if model_path is None:
+        model_path = os.environ.get("MODEL_PATH", "models/xdec_model.pt")
+
+    app.state.started_at = time.monotonic()
+    app.state.model_path = model_path
+    app.state.model_load_time_ms = None
+
+    try:
+        t0 = time.monotonic()
+        predictor = Predictor(model_path=model_path)
+        app.state.model_load_time_ms = (time.monotonic() - t0) * 1000.0
+        app.state.predictor = predictor
+        app.state.model_loaded = True
+    except FileNotFoundError:
+        app.state.predictor = None
+        app.state.model_loaded = False
+
+    # Forecaster is lazy-loaded on first /forecast request, independent of X-DEC.
+    forecast_model_id = os.environ.get("FORECAST_MODEL_ID", "amazon/chronos-bolt-tiny")
+    forecast_device = os.environ.get("FORECAST_DEVICE", "cpu")
+    app.state.forecaster = Forecaster(
+        model_id=forecast_model_id,
+        device=forecast_device,
+    )
+
+    @app.get("/")
+    def root() -> dict:
+        """Root endpoint with API info."""
+        return {"name": "Scry Predictive API", "version": VERSION, "docs": "/docs"}
+
+    @app.get("/health", response_model=HealthResponse)
+    def health() -> HealthResponse:
+        """Liveness/readiness check."""
+        status = "healthy" if app.state.model_loaded else "unhealthy"
+        return HealthResponse(
+            status=status,
+            model_loaded=app.state.model_loaded,
+            version=VERSION,
+        )
+
+    @app.get("/health/detailed", response_model=DetailedHealthResponse)
+    def health_detailed() -> DetailedHealthResponse:
+        """Detailed health check: model metadata, configured data source, uptime."""
+        status = "healthy" if app.state.model_loaded else "unhealthy"
+        uptime = time.monotonic() - app.state.started_at
+
+        model_version = None
+        if app.state.predictor is not None:
+            cfg = app.state.predictor.config
+            model_version = f"xdec-k{cfg['n_clusters']}-d{cfg['latent_dim']}"
+        env_version = os.environ.get("MODEL_VERSION")
+        if env_version:
+            model_version = f"{model_version}-{env_version}" if model_version else env_version
+
+        return DetailedHealthResponse(
+            status=status,
+            model_loaded=app.state.model_loaded,
+            version=VERSION,
+            model_load_time_ms=app.state.model_load_time_ms,
+            model_version=model_version,
+            model_path=app.state.model_path,
+            datasource=_datasource_descriptor(),
+            chronos_loaded=app.state.forecaster.is_loaded,
+            uptime_seconds=round(uptime, 2),
+        )
+
+    @app.post("/predict", response_model=PredictionResponse)
+    def predict(request: PredictionRequest) -> PredictionResponse:
+        """Predict the operational state for metrics supplied in the request body.
+
+        Raises:
+            HTTPException: 503 if the model is not loaded.
+        """
+        if not app.state.model_loaded:
+            raise HTTPException(status_code=503, detail="Model not loaded")
+
+        result = app.state.predictor.predict(
+            numerical_metrics=request.numerical_metrics,
+            categorical_metrics=request.categorical_metrics,
+        )
+
+        return PredictionResponse(
+            resource_id=request.resource_id,
+            cluster_id=result["cluster_id"],
+            cluster_name=result["cluster_name"],
+            confidence=result["confidence"],
+            action=result["action"],
+            priority=result["priority"],
+        )
+
+    @app.get("/clusters", response_model=list[ClusterInfo])
+    def clusters() -> list[ClusterInfo]:
+        """Get all operational state (cluster) definitions."""
+        return get_cluster_info()
+
+    @app.get("/predict/lookup", response_model=PredictionResponse)
+    async def predict_lookup(
+        resource_id: str = Query(..., description="Resource id or hostname to look up"),
+    ) -> PredictionResponse:
+        """Look up a resource's recent metrics through the configured data source and predict.
+
+        Object storage (SCRY_DATA_URI) is the default source; the HttpIngest
+        adapter is used when the ``logicmonitor`` extra is installed.
+
+        Raises:
+            HTTPException: 503 if the model is not loaded or no source is configured,
+                404 if the resource has no usable recent metrics, 502 on a source error.
+        """
+        with tracer.start_as_current_span("predict_lookup") as span:
+            span.set_attribute("resource_id", resource_id)
+
+            if not app.state.model_loaded:
+                raise HTTPException(status_code=503, detail="Model not loaded")
+
+            try:
+                df = await _resource_metrics(resource_id)
+            except Exception as e:
+                logger.error("lookup data fetch failed: %s: %s", type(e).__name__, e)
+                raise HTTPException(status_code=502, detail=f"data source error: {e}") from e
+
+            if df is None:
+                raise HTTPException(
+                    status_code=503,
+                    detail=(
+                        "No data source configured. Set SCRY_DATA_URI, or install the "
+                        "'logicmonitor' extra and set HTTPINGEST_URL."
+                    ),
+                )
+            if df.empty:
+                raise HTTPException(
+                    status_code=404,
+                    detail=f"No recent metrics found for resource '{resource_id}'",
+                )
+
+            numerical, categorical = _split_by_profile(df)
+            if not numerical:
+                raise HTTPException(
+                    status_code=404,
+                    detail=(
+                        f"No usable metrics for resource '{resource_id}' "
+                        "under the active profile"
+                    ),
+                )
+
+            span.set_attribute("numerical_metrics.count", len(numerical))
+            span.set_attribute("categorical_metrics.count", len(categorical))
+            result = app.state.predictor.predict(
+                numerical_metrics=numerical,
+                categorical_metrics=categorical,
+            )
+            return PredictionResponse(
+                resource_id=resource_id,
+                cluster_id=result["cluster_id"],
+                cluster_name=result["cluster_name"],
+                confidence=result["confidence"],
+                action=result["action"],
+                priority=result["priority"],
+            )
+
+    @app.post("/forecast", response_model=ForecastResponse)
+    def forecast(request: ForecastRequest) -> ForecastResponse:
+        """Forecast metric values at the requested horizons using Chronos.
+
+        Independent of the X-DEC cluster model. Requires the ``forecast`` extra.
+        """
+        forecaster: Forecaster = app.state.forecaster
+
+        if request.horizons != forecaster.horizons:
+            forecaster.horizons = request.horizons
+
+        metric_forecasts = forecaster.forecast_metrics(request.metrics)
+
+        return ForecastResponse(
+            resource_id=request.resource_id,
+            forecasts=[MetricForecast(**mf) for mf in metric_forecasts],
+            model_id=forecaster.model_id,
+        )
+
+    @app.get("/drift")
+    def drift_status() -> dict:
+        """Get current drift detection status (PSI feature drift, ADWIN prediction drift)."""
+        if not hasattr(app.state, "drift_detector"):
+            return {
+                "feature_drift": {"has_drift": False, "message": "No reference data configured"},
+                "prediction_drift": {"has_drift": False, "message": "No error history available"},
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+            }
+
+        detector = app.state.drift_detector
+        return detector.get_drift_status(
+            app.state.reference_data,
+            app.state.current_data,
+            app.state.error_stream,
+        )
+
+    @app.get("/anomaly")
+    def anomaly_status() -> dict:
+        """Get current forecast-based anomaly detection status."""
+        if not hasattr(app.state, "anomaly_detector"):
+            return {
+                "is_anomaly": False,
+                "anomaly_score": 0.0,
+                "violated_metrics": [],
+                "severity": "low",
+                "metric_count": 0,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+            }
+
+        detector = app.state.anomaly_detector
+        actuals = app.state.last_actuals
+        forecast = app.state.last_forecast
+
+        result = detector.detect(actuals, forecast)
+
+        return {
+            "is_anomaly": bool(result["is_anomaly"]),
+            "anomaly_score": float(result["anomaly_score"]),
+            "violated_metrics": result["violated_metrics"],
+            "severity": result["severity"],
+            "metric_count": len(detector.metric_names),
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        }
+
+    @app.get("/accuracy")
+    def accuracy_status() -> dict:
+        """Get forecast accuracy and cluster stability metrics as flat key=value pairs."""
+        start_ms = time.monotonic_ns() // 1_000_000
+
+        if not hasattr(app.state, "accuracy_tracker"):
+            elapsed = (time.monotonic_ns() // 1_000_000) - start_ms
+            fallback: dict = {}
+            for metric in ["Picp", "Mae", "Mase", "Mpiw"]:
+                for horizon in ["15m", "1h", "4h", "24h"]:
+                    fallback[f"{metric}{horizon}"] = 0.0
+            fallback["TransitionRate"] = 0.0
+            fallback["ConfidenceStd"] = 0.0
+            fallback["DominantClusterPct"] = 0.0
+            fallback["ObservationCount"] = 0
+            fallback["ApiStatus"] = 1
+            fallback["ApiLatencyMs"] = elapsed
+            fallback["timestamp"] = datetime.now(timezone.utc).isoformat()
+            return fallback
+
+        tracker = app.state.accuracy_tracker
+        metrics = tracker.compute_metrics()
+        elapsed = (time.monotonic_ns() // 1_000_000) - start_ms
+
+        result: dict = {}
+        for metric_key, flat_prefix in [
+            ("picp", "Picp"),
+            ("mae", "Mae"),
+            ("mase", "Mase"),
+            ("mpiw", "Mpiw"),
+        ]:
+            for horizon in metrics["horizons"]:
+                val = metrics["horizons"][horizon][metric_key]
+                if isinstance(val, float) and (val != val):  # NaN check
+                    val = 0.0
+                result[f"{flat_prefix}{horizon}"] = round(val, 4)
+
+        result["TransitionRate"] = round(metrics["stability"]["transition_rate"], 4)
+        result["ConfidenceStd"] = round(metrics["stability"]["confidence_std"], 4)
+        result["DominantClusterPct"] = round(metrics["stability"]["dominant_cluster_pct"], 1)
+        result["ObservationCount"] = metrics["observation_count"]
+        result["ApiStatus"] = 1
+        result["ApiLatencyMs"] = elapsed
+        result["timestamp"] = metrics["timestamp"]
+
+        return result
+
+    return app
+
+
+# Default app instance for uvicorn
+app = create_app()