cloudsealed-jit 0.2.0__py3-none-any.whl

cloudsealed_jit/__init__.py

@@ -0,0 +1,24 @@
+"""
+cloudsealed-jit — cloud billing waste analysis.
+
+Detects structural cost waste in cloud billing exports by modelling a robust
+day-of-week aware baseline and measuring the excess spend above it.
+
+Public API:
+    parse_billing_csv(csv_text) -> BillingSeries
+    analyze(series, analysis_type="waste-audit") -> AnalysisResult
+"""
+
+from .parsing import BillingSeries, ParseError, parse_billing_csv
+from .analysis import AnalysisResult, analyze
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "BillingSeries",
+    "ParseError",
+    "parse_billing_csv",
+    "AnalysisResult",
+    "analyze",
+    "__version__",
+]

cloudsealed_jit/analysis.py

@@ -0,0 +1,423 @@
+"""
+Waste analysis over a daily billing series.
+
+Method
+------
+
+**Baseline.** Expected spend for a day is the product of a level term and a
+weekday term:
+
+    expected[i] = rolling_median(cost, 7)[i] * dow_factor[weekday(i)]
+
+The rolling median tracks growth and step changes without being dragged by
+spikes. The weekday factor is the median ratio of observed spend to the level
+term for that weekday, which captures the weekday/weekend cycle that dominates
+most cloud bills. It is only estimated when at least two full weeks are
+available; below that the factor is 1.0 for every day.
+
+**Anomalies.** Residuals against the baseline are scored with a modified
+z-score (median absolute deviation, see :mod:`cloudsealed_jit.kernels`). Days
+scoring at or above ``Z_THRESHOLD`` are reported. The threshold of 3.5 is the
+value recommended by Iglewicz & Hoaglin for the modified z-score.
+
+**Waste.** Only *positive* excess counts as waste: spending less than expected
+is not an opportunity. Waste percentage is the share of total spend that sits
+above the baseline on anomalous days, which makes it directly convertible to
+currency rather than being a count of unusual days.
+
+**Recommendations.** Every recommendation carries a saving derived from the
+series itself, normalised to 30 days, and states the assumption behind it in
+its description. Estimates that depend on facts the analyser cannot observe --
+whether a workload is production, whether a commitment is acceptable -- are
+labelled as conditional rather than presented as findings.
+"""
+
+from __future__ import annotations
+
+import statistics
+from dataclasses import asdict, dataclass, field
+from datetime import date
+from typing import Literal
+
+import numpy as np
+
+from .kernels import modified_zscores, rolling_median
+from .parsing import BillingSeries
+
+__all__ = [
+    "Anomaly",
+    "Metrics",
+    "Recommendation",
+    "AnalysisResult",
+    "analyze",
+    "Z_THRESHOLD",
+]
+
+AnalysisType = Literal["waste-audit", "cost-forecast", "efficiency"]
+
+#: Modified z-score at which a day is reported as anomalous.
+Z_THRESHOLD = 3.5
+
+#: Minimum days required before the weekday seasonality term is estimated.
+MIN_DAYS_FOR_SEASONALITY = 14
+
+#: Assumed discount for a one-year commitment against the sustained baseline.
+#: Conservative relative to published AWS/GCP/Azure commitment discounts.
+COMMITMENT_DISCOUNT = 0.25
+
+#: A service whose weekend spend is at least this fraction of its weekday
+#: spend is running through the weekend at full cost.
+WEEKEND_RATIO_FLAG = 0.80
+
+# Severity combines statistical strength with financial size. Either alone is
+# misleading: a very tight baseline turns a 5x spike into a moderate z-score,
+# while a noisy series can produce a large z-score over a trivial amount.
+# Each tier is (min |z|, min |deviation %|, label) and matches on either.
+_SEVERITY_TIERS = (
+    (10.0, 200.0, "CRITICAL"),
+    (7.0, 100.0, "HIGH"),
+    (5.0, 25.0, "MEDIUM"),
+)
+
+
+@dataclass
+class Anomaly:
+    date: str
+    expectedCost: float
+    actualCost: float
+    deviation: float  # percent against expected
+    zScore: float
+    severity: str
+    description: str
+
+
+@dataclass
+class Metrics:
+    averageDailyCost: float
+    stdDeviation: float
+    sharpeRatio: float  # spend stability: mean / stddev of daily cost
+    wastePercentage: float
+
+
+@dataclass
+class Recommendation:
+    title: str
+    description: str
+    potentialSavings: float  # per 30 days, in the export's currency
+    effort: str
+
+
+@dataclass
+class AnalysisResult:
+    anomalies: list[Anomaly] = field(default_factory=list)
+    metrics: Metrics = field(
+        default_factory=lambda: Metrics(0.0, 0.0, 0.0, 0.0)
+    )
+    recommendations: list[Recommendation] = field(default_factory=list)
+    summary: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+# --------------------------------------------------------------------------
+# Baseline
+# --------------------------------------------------------------------------
+
+
+def _weekday_factors(costs: np.ndarray, level: np.ndarray,
+                     days: list[date]) -> np.ndarray:
+    """Median ratio of observed spend to the level term, per weekday."""
+    factors = np.ones(7, dtype=np.float64)
+    if len(days) < MIN_DAYS_FOR_SEASONALITY:
+        return factors
+
+    buckets: dict[int, list[float]] = {i: [] for i in range(7)}
+    for i, day in enumerate(days):
+        if level[i] > 0:
+            buckets[day.weekday()].append(costs[i] / level[i])
+
+    for weekday, ratios in buckets.items():
+        # Two observations is not enough to separate a pattern from noise.
+        if len(ratios) >= 2:
+            factors[weekday] = float(statistics.median(ratios))
+
+    # Renormalise so the factors do not shift the overall level.
+    mean_factor = float(np.mean(factors))
+    if mean_factor > 0:
+        factors /= mean_factor
+    return factors
+
+
+def _baseline(costs: np.ndarray, days: list[date]) -> np.ndarray:
+    level = rolling_median(costs, window=7)
+    factors = _weekday_factors(costs, level, days)
+    expected = np.array(
+        [level[i] * factors[day.weekday()] for i, day in enumerate(days)],
+        dtype=np.float64,
+    )
+    return np.maximum(expected, 0.0)
+
+
+def _severity(z: float, deviation_pct: float) -> str:
+    magnitude = abs(z)
+    deviation = abs(deviation_pct)
+    for min_z, min_deviation, label in _SEVERITY_TIERS:
+        if magnitude >= min_z or deviation >= min_deviation:
+            return label
+    return "LOW"
+
+
+# --------------------------------------------------------------------------
+# Recommendations
+# --------------------------------------------------------------------------
+
+
+def _weekend_recommendations(series: BillingSeries,
+                             currency: str) -> list[Recommendation]:
+    """Flag services that cost the same on weekends as on weekdays."""
+    out: list[Recommendation] = []
+    days = series.days
+    if len(days) < MIN_DAYS_FOR_SEASONALITY:
+        return out
+
+    weekend_idx = [i for i, d in enumerate(days) if d.weekday() >= 5]
+    weekday_idx = [i for i, d in enumerate(days) if d.weekday() < 5]
+    if len(weekend_idx) < 4 or len(weekday_idx) < 8:
+        return out
+
+    span = len(days)
+    for name, values in series.by_service.items():
+        weekend = statistics.median(values[i] for i in weekend_idx)
+        weekday = statistics.median(values[i] for i in weekday_idx)
+        if weekday <= 0:
+            continue
+        ratio = weekend / weekday
+        if ratio < WEEKEND_RATIO_FLAG:
+            continue
+
+        service_total = sum(values)
+        if service_total <= 0:
+            continue
+
+        # Upper bound: the weekend spend itself, normalised to 30 days.
+        weekend_spend = sum(values[i] for i in weekend_idx)
+        monthly = weekend_spend / span * 30
+        if monthly < 1.0:
+            continue
+
+        out.append(
+            Recommendation(
+                title=f"Idle weekend spend in {name}",
+                description=(
+                    f"{name} costs {ratio:.0%} as much on weekends as on weekdays "
+                    f"(weekday median {currency} {weekday:,.2f}/day, weekend median "
+                    f"{currency} {weekend:,.2f}/day). If this workload is not "
+                    f"production, stopping it outside business days removes the "
+                    f"weekend spend entirely. The figure is the observed weekend "
+                    f"spend normalised to 30 days and assumes the workload can be "
+                    f"stopped; verify before acting."
+                ),
+                potentialSavings=round(monthly, 2),
+                effort="LOW",
+            )
+        )
+    return out
+
+
+def _commitment_recommendation(costs: np.ndarray,
+                               currency: str) -> Recommendation | None:
+    """Estimate the saving available on the always-on portion of spend."""
+    if costs.size < MIN_DAYS_FOR_SEASONALITY:
+        return None
+    floor = float(np.percentile(costs, 10))
+    if floor <= 0:
+        return None
+
+    monthly = floor * 30 * COMMITMENT_DISCOUNT
+    if monthly < 1.0:
+        return None
+
+    return Recommendation(
+        title="Sustained baseline eligible for commitment pricing",
+        description=(
+            f"Daily spend never fell below {currency} {floor:,.2f} (10th percentile) "
+            f"over the period, so that portion is always-on. Committed-use discounts "
+            f"or savings plans typically price this tier {COMMITMENT_DISCOUNT:.0%} "
+            f"below on-demand. The figure applies that rate to the observed floor "
+            f"over 30 days and assumes a commitment is commercially acceptable."
+        ),
+        potentialSavings=round(monthly, 2),
+        effort="MEDIUM",
+    )
+
+
+def _excess_recommendation(excess_total: float, span: int,
+                           currency: str, count: int) -> Recommendation | None:
+    if excess_total <= 0 or count == 0:
+        return None
+    monthly = excess_total / span * 30
+    if monthly < 1.0:
+        return None
+    return Recommendation(
+        title="Investigate spend above the modelled baseline",
+        description=(
+            f"{count} day(s) spent {currency} {excess_total:,.2f} more than the "
+            f"day-of-week baseline predicted. This is unplanned spend rather than "
+            f"growth: the baseline already tracks trend and weekly seasonality. "
+            f"The figure is that excess normalised to 30 days."
+        ),
+        potentialSavings=round(monthly, 2),
+        effort="MEDIUM",
+    )
+
+
+def _concentration_recommendation(series: BillingSeries,
+                                  currency: str) -> Recommendation | None:
+    total = series.total_cost
+    if total <= 0 or not series.by_service:
+        return None
+    name, values = max(series.by_service.items(), key=lambda kv: sum(kv[1]))
+    share = sum(values) / total
+    if share < 0.40:
+        return None
+    return Recommendation(
+        title=f"Spend concentrated in {name}",
+        description=(
+            f"{name} accounts for {share:.0%} of total spend ({currency} "
+            f"{sum(values):,.2f} of {currency} {total:,.2f}). Optimisation effort "
+            f"applied here has the highest leverage; a 10% reduction on this "
+            f"service alone yields the figure shown."
+        ),
+        potentialSavings=round(sum(values) / max(len(series.days), 1) * 30 * 0.10, 2),
+        effort="MEDIUM",
+    )
+
+
+def _volatility_recommendation(stability: float, mean: float,
+                               std: float, currency: str) -> Recommendation | None:
+    if stability >= 2.0 or mean <= 0:
+        return None
+    return Recommendation(
+        title="Daily spend is highly volatile",
+        description=(
+            f"Daily cost varies with a standard deviation of {currency} {std:,.2f} "
+            f"against a mean of {currency} {mean:,.2f} (stability ratio "
+            f"{stability:.2f}). Spend this irregular cannot be budgeted or alerted "
+            f"on reliably. Establishing budget alerts and per-environment cost "
+            f"attribution is a prerequisite for any further optimisation. No direct "
+            f"saving is claimed."
+        ),
+        potentialSavings=0.0,
+        effort="LOW",
+    )
+
+
+# --------------------------------------------------------------------------
+# Entry point
+# --------------------------------------------------------------------------
+
+
+def analyze(series: BillingSeries,
+            analysis_type: AnalysisType = "waste-audit",
+            *, max_anomalies: int = 25) -> AnalysisResult:
+    """Run waste analysis over a parsed billing series."""
+    currency = series.currency or "USD"
+    span = series.span_days
+
+    if span < 3:
+        return AnalysisResult(
+            summary=(
+                f"Only {span} day(s) of billing data. At least 3 days are needed "
+                f"to model a baseline, and 14 for weekday seasonality."
+            )
+        )
+
+    costs = np.asarray(series.costs, dtype=np.float64)
+    expected = _baseline(costs, series.days)
+    zscores = modified_zscores(costs, expected)
+
+    mean = float(np.mean(costs))
+    std = float(np.std(costs, ddof=1)) if span > 1 else 0.0
+    stability = float(mean / std) if std > 0 else 0.0
+
+    anomalies: list[Anomaly] = []
+    excess_total = 0.0
+    for i in range(span):
+        z = float(zscores[i])
+        if abs(z) < Z_THRESHOLD:
+            continue
+        exp = float(expected[i])
+        act = float(costs[i])
+        deviation = ((act - exp) / exp * 100.0) if exp > 0 else 0.0
+        if act > exp:
+            excess_total += act - exp
+
+        anomalies.append(
+            Anomaly(
+                date=series.days[i].isoformat(),
+                expectedCost=round(exp, 2),
+                actualCost=round(act, 2),
+                deviation=round(deviation, 2),
+                zScore=round(z, 2),
+                severity=_severity(z, deviation),
+                description=(
+                    f"Spend {'above' if act > exp else 'below'} the day-of-week "
+                    f"baseline by {currency} {abs(act - exp):,.2f} "
+                    f"({abs(deviation):.1f}%)."
+                ),
+            )
+        )
+
+    total = series.total_cost
+    waste_pct = (excess_total / total * 100.0) if total > 0 else 0.0
+
+    metrics = Metrics(
+        averageDailyCost=round(mean, 2),
+        stdDeviation=round(std, 2),
+        sharpeRatio=round(stability, 2),
+        wastePercentage=round(waste_pct, 2),
+    )
+
+    recommendations: list[Recommendation] = []
+    if analysis_type in ("waste-audit", "cost-forecast"):
+        over = sum(1 for a in anomalies if a.actualCost > a.expectedCost)
+        excess_rec = _excess_recommendation(excess_total, span, currency, over)
+        if excess_rec:
+            recommendations.append(excess_rec)
+        recommendations.extend(_weekend_recommendations(series, currency))
+        commitment = _commitment_recommendation(costs, currency)
+        if commitment:
+            recommendations.append(commitment)
+        concentration = _concentration_recommendation(series, currency)
+        if concentration:
+            recommendations.append(concentration)
+
+    volatility = _volatility_recommendation(stability, mean, std, currency)
+    if volatility:
+        recommendations.append(volatility)
+
+    recommendations.sort(key=lambda r: r.potentialSavings, reverse=True)
+
+    identified = sum(r.potentialSavings for r in recommendations)
+    summary = (
+        f"Analysed {span} day(s) of billing ({series.rows_parsed:,} line items) "
+        f"totalling {currency} {total:,.2f}. "
+        f"Mean daily spend {currency} {mean:,.2f}, stability ratio {stability:.2f}. "
+        f"{len(anomalies)} anomalous day(s) detected; {waste_pct:.1f}% of total "
+        f"spend sits above the modelled baseline. "
+        f"Identified up to {currency} {identified:,.2f} in addressable monthly spend."
+    )
+    if series.rows_skipped:
+        summary += f" {series.rows_skipped:,} row(s) could not be parsed."
+    if analysis_type == "cost-forecast":
+        summary += f" Projected 30-day spend at current run rate: {currency} {mean * 30:,.2f}."
+
+    anomalies.sort(key=lambda a: abs(a.zScore), reverse=True)
+
+    return AnalysisResult(
+        anomalies=anomalies[:max_anomalies],
+        metrics=metrics,
+        recommendations=recommendations,
+        summary=summary,
+    )

cloudsealed_jit/api.py

@@ -0,0 +1,165 @@
+"""
+HTTP surface for the waste analyser.
+
+The request and response schemas are the contract consumed by the CloudSealed
+Framework4D assessment engine (``framework4d-jit-client.ts``). Field names and
+types are part of that contract and must not change without a version bump.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from typing import Literal, Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from pydantic import BaseModel, Field
+
+from . import __version__
+from .analysis import analyze
+from .kernels import JIT_ENABLED
+from .parsing import ParseError, parse_billing_csv
+
+logger = logging.getLogger("cloudsealed_jit.api")
+
+#: Reject payloads above this size before parsing. Billing exports are large,
+#: but an unbounded body is a denial-of-service surface.
+MAX_CSV_BYTES = int(os.getenv("JIT_MAX_CSV_BYTES", str(64 * 1024 * 1024)))
+
+#: When set, every request must present a matching ``X-Api-Key`` header.
+API_KEY = os.getenv("JIT_OPTIMIZATION_API_KEY", "")
+
+
+# --------------------------------------------------------------------------
+# Contract
+# --------------------------------------------------------------------------
+
+
+class AnalyzeBillingRequest(BaseModel):
+    companyName: str = Field(min_length=1, max_length=200)
+    csvContent: str = Field(description="Raw contents of the billing export.")
+    analysisType: Optional[Literal["waste-audit", "cost-forecast", "efficiency"]] = (
+        "waste-audit"
+    )
+
+
+class CostAnomaly(BaseModel):
+    date: str
+    expectedCost: float
+    actualCost: float
+    deviation: float
+    zScore: float
+    severity: Literal["LOW", "MEDIUM", "HIGH", "CRITICAL"]
+    description: str
+
+
+class Metrics(BaseModel):
+    averageDailyCost: float
+    stdDeviation: float
+    sharpeRatio: float
+    wastePercentage: float
+
+
+class Recommendation(BaseModel):
+    title: str
+    description: str
+    potentialSavings: float
+    effort: Literal["LOW", "MEDIUM", "HIGH"]
+
+
+class AnalyzeBillingResponse(BaseModel):
+    anomalies: list[CostAnomaly]
+    metrics: Metrics
+    recommendations: list[Recommendation]
+    summary: str
+
+
+# --------------------------------------------------------------------------
+# Application
+# --------------------------------------------------------------------------
+
+app = FastAPI(
+    title="CloudSealed JIT Optimization Engine",
+    version=__version__,
+    description=(
+        "Detects structural waste in cloud billing exports using a robust, "
+        "day-of-week aware baseline and modified z-scores."
+    ),
+)
+
+
+async def require_api_key(x_api_key: str = Header(default="")) -> None:
+    """Enforce ``X-Api-Key`` when an API key is configured."""
+    if API_KEY and x_api_key != API_KEY:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or missing X-Api-Key.",
+        )
+
+
+@app.get("/health")
+def health() -> dict:
+    return {
+        "status": "ok",
+        "version": __version__,
+        "jitEnabled": JIT_ENABLED,
+    }
+
+
+@app.post(
+    "/v1/analyze-billing",
+    response_model=AnalyzeBillingResponse,
+    dependencies=[Depends(require_api_key)],
+)
+def analyze_billing(payload: AnalyzeBillingRequest) -> AnalyzeBillingResponse:
+    """Analyse a cloud billing export and return waste findings.
+
+    Returns 422 when the export cannot be interpreted, so callers can tell a
+    malformed file from an internal failure.
+    """
+    size = len(payload.csvContent.encode("utf-8", errors="ignore"))
+    if size > MAX_CSV_BYTES:
+        raise HTTPException(
+            status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+            detail=f"Billing export is {size:,} bytes; limit is {MAX_CSV_BYTES:,}.",
+        )
+
+    started = time.perf_counter()
+    try:
+        series = parse_billing_csv(payload.csvContent)
+    except ParseError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)
+        ) from exc
+
+    try:
+        result = analyze(series, payload.analysisType or "waste-audit")
+    except Exception:  # pragma: no cover - defensive
+        logger.exception("Analysis failed for %s", payload.companyName)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Analysis failed.",
+        )
+
+    elapsed_ms = (time.perf_counter() - started) * 1000
+    logger.info(
+        "analyzed company=%s days=%d rows=%d anomalies=%d elapsed_ms=%.1f",
+        payload.companyName, series.span_days, series.rows_parsed,
+        len(result.anomalies), elapsed_ms,
+    )
+    return AnalyzeBillingResponse(**result.to_dict())
+
+
+def main() -> None:  # pragma: no cover - entry point
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host=os.getenv("JIT_HOST", "0.0.0.0"),
+        port=int(os.getenv("JIT_PORT", "8091")),
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

cloudsealed_jit/cli.py

@@ -0,0 +1,57 @@
+"""Command-line entry point: analyse a billing export without running a server."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .analysis import analyze
+from .parsing import ParseError, parse_billing_csv
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="cloudsealed-jit",
+        description="Detect structural waste in a cloud billing export.",
+    )
+    parser.add_argument("csv", help="path to the billing export, or - for stdin")
+    parser.add_argument(
+        "--type",
+        default="waste-audit",
+        choices=("waste-audit", "cost-forecast", "efficiency"),
+        help="analysis mode (default: waste-audit)",
+    )
+    parser.add_argument("--json", action="store_true", help="emit raw JSON")
+    args = parser.parse_args(argv)
+
+    text = sys.stdin.read() if args.csv == "-" else open(args.csv, encoding="utf-8").read()
+
+    try:
+        result = analyze(parse_billing_csv(text), args.type)
+    except ParseError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    if args.json:
+        print(json.dumps(result.to_dict(), indent=2))
+        return 0
+
+    print(result.summary)
+    if result.anomalies:
+        print("\nAnomalies")
+        for a in result.anomalies:
+            print(
+                f"  {a.date}  {a.severity:<8} expected {a.expectedCost:>12,.2f}  "
+                f"actual {a.actualCost:>12,.2f}  ({a.deviation:+.1f}%, z={a.zScore})"
+            )
+    if result.recommendations:
+        print("\nRecommendations")
+        for r in result.recommendations:
+            print(f"  [{r.effort:<6}] {r.title} — {r.potentialSavings:,.2f}/30d")
+            print(f"           {r.description}")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())