modeldiet 0.1.0__py3-none-any.whl

modeldiet/__init__.py

@@ -0,0 +1,6 @@
+"""Model Diet public API."""
+
+from modeldiet.api import profile
+from modeldiet.result import ProfileResult
+
+__all__ = ["profile", "ProfileResult"]

modeldiet/api.py

@@ -0,0 +1,44 @@
+"""Public Model Diet API."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Literal
+
+from modeldiet.profiling.measurements import BaseMeasurement, LatencyMeasurement
+from modeldiet.profiling.orchestrator import run_measurements
+from modeldiet.result import ProfileResult
+
+
+def profile(
+    model: Any,
+    X: Any,
+    *,
+    measurements: Sequence[BaseMeasurement] | None = None,
+    on_error: Literal["raise", "record"] = "raise",
+    include_environment: bool = False,
+) -> ProfileResult:
+    """Profile a fitted estimator using the selected measurement strategies."""
+    if on_error not in ("raise", "record"):
+        raise ValueError("on_error must be 'raise' or 'record'")
+    if not callable(getattr(model, "predict", None)):
+        raise TypeError("model must provide a callable predict(X) method")
+    if X is None:
+        raise ValueError("X must contain representative tabular data")
+    selected: list[BaseMeasurement]
+    if measurements is None:
+        selected = [LatencyMeasurement()]
+    else:
+        selected = list(measurements)
+    if not selected:
+        raise ValueError("measurements must contain at least one measurement")
+    if any(not isinstance(m, BaseMeasurement) for m in selected):
+        raise TypeError("each measurement must be a BaseMeasurement instance")
+    result = run_measurements(model, X, selected, on_error=on_error)
+    if include_environment:
+        from dataclasses import replace
+
+        from modeldiet.profiling.environment import collect_environment
+
+        return replace(result, environment=collect_environment())
+    return result

modeldiet/benchmark/__init__.py

@@ -0,0 +1,5 @@
+"""Optional repeated benchmark tooling for bundled demonstration models."""
+
+from modeldiet.benchmark.runner import run_repeated_benchmark
+
+__all__ = ["run_repeated_benchmark"]

modeldiet/benchmark/model.py

@@ -0,0 +1,143 @@
+"""Model wrapper that delegates profiling to independent measurements."""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any
+
+from modeldiet import profile
+from modeldiet.profiling.measurements import (
+    BaseMeasurement,
+    LatencyMeasurement,
+    ProcessRSSMeasurement,
+    SerializationMeasurement,
+    TracedAllocationMeasurement,
+)
+
+
+class _NamedMeasurement(BaseMeasurement):
+    """Give a legacy strategy slot a stable name for the core profile API."""
+
+    def __init__(self, name: str, measurement: BaseMeasurement) -> None:
+        self.name = name
+        self.measurement = measurement
+
+    def measure(self, model: Any, X: Any = None) -> dict[str, Any]:
+        return self.measurement.measure(model, X)
+
+
+class Model:
+    """Compatibility wrapper around a trained estimator.
+
+    Estimator-specific subclasses provide ``clf``. Measurement strategies are
+    independent objects that can be reused, injected, or subclassed by callers.
+    """
+
+    def __init__(
+        self,
+        loading_time: float = 0.0,
+        mean_inference_latency: float = 0.0,
+        p50_latency: float = 0.0,
+        p90_latency: float = 0.0,
+        p95_latency: float = 0.0,
+        p99_latency: float = 0.0,
+        traced_peak_memory_usage: float = 0.0,
+        processed_rss: float = 0.0,
+        serialized_model_size: float = 0.0,
+        latency_measurement: BaseMeasurement | None = None,
+        allocation_measurement: BaseMeasurement | None = None,
+        rss_measurement: BaseMeasurement | None = None,
+        serialization_measurement: BaseMeasurement | None = None,
+    ) -> None:
+        warnings.warn(
+            f"{type(self).__name__} is a deprecated benchmark compatibility "
+            "interface. Use an ordinary fitted estimator with "
+            "modeldiet.profile(estimator, X) instead. The legacy model "
+            "interfaces will remain available through the 0.x series and "
+            "will not be removed before 1.0.0.",
+            DeprecationWarning,
+            stacklevel=3 if type(self) is not Model else 2,
+        )
+        self.loading_time = loading_time
+        self.mean_inference_latency = mean_inference_latency
+        self.p50_latency = p50_latency
+        self.p90_latency = p90_latency
+        self.p95_latency = p95_latency
+        self.p99_latency = p99_latency
+        self.traced_peak_inference_allocation_mb = traced_peak_memory_usage
+        self.rss_before_inference_mb = processed_rss
+        self.rss_after_inference_mb = processed_rss
+        self.rss_delta_mb = 0.0
+        self.traced_peak_memory_usage = traced_peak_memory_usage  # compatibility
+        self.processed_rss = processed_rss  # compatibility
+        self.serialized_model_size = serialized_model_size
+
+        self.clf: Any = None
+        self.latency_measurement = (
+            latency_measurement
+            if latency_measurement is not None
+            else LatencyMeasurement()
+        )
+        self.allocation_measurement = (
+            allocation_measurement
+            if allocation_measurement is not None
+            else TracedAllocationMeasurement()
+        )
+        self.rss_measurement = (
+            rss_measurement if rss_measurement is not None else ProcessRSSMeasurement()
+        )
+        self.serialization_measurement = (
+            serialization_measurement
+            if serialization_measurement is not None
+            else SerializationMeasurement()
+        )
+
+    def profile_execution(
+        self, X_test: Any, load_start_time: float | None = None
+    ) -> dict[str, dict[str, Any]]:
+        """Run each measurement through the core profiling API.
+
+        ``load_start_time`` remains accepted for backwards compatibility. An
+        already-loaded model has no measurable historical load duration.
+        """
+        del load_start_time
+        named_measurements = [
+            _NamedMeasurement("latency", self.latency_measurement),
+            _NamedMeasurement("python_traced_allocation", self.allocation_measurement),
+            _NamedMeasurement("process_rss", self.rss_measurement),
+            _NamedMeasurement("pickle_size", self.serialization_measurement),
+        ]
+        result = profile(self.clf, X_test, measurements=named_measurements)
+
+        def values(name: str) -> dict[str, Any]:
+            return {
+                key: metric.value
+                for key, metric in result.measurements[name].metrics.items()
+            }
+
+        latency = values("latency")
+        allocation = values("python_traced_allocation")
+        process_memory = values("process_rss")
+        serialization = values("pickle_size")
+
+        self.mean_inference_latency = latency["mean_ms"]
+        self.p50_latency = latency["p50_ms"]
+        self.p90_latency = latency["p90_ms"]
+        self.p95_latency = latency["p95_ms"]
+        self.p99_latency = latency["p99_ms"]
+        self.traced_peak_inference_allocation_mb = allocation[
+            "traced_peak_inference_allocation_mb"
+        ]
+        self.rss_before_inference_mb = process_memory["rss_before_inference_mb"]
+        self.rss_after_inference_mb = process_memory["rss_after_inference_mb"]
+        self.rss_delta_mb = process_memory["rss_delta_mb"]
+        self.traced_peak_memory_usage = self.traced_peak_inference_allocation_mb
+        self.processed_rss = self.rss_after_inference_mb
+        self.serialized_model_size = serialization["serialized_model_size_mb"]
+
+        return {
+            "latency": latency,
+            "allocation": allocation,
+            "process_memory": process_memory,
+            "serialization": serialization,
+        }

modeldiet/benchmark/models/__init__.py

@@ -0,0 +1 @@
+"""Demonstration estimators used by the optional benchmark worker."""

modeldiet/benchmark/models/lr.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from sklearn.linear_model import LogisticRegression
+
+from modeldiet.benchmark.model import Model
+
+
+class LogisticRegressionModel(Model):
+    def __init__(
+        self,
+        C: float = 1.0,
+        max_iter: int = 100,
+        solver: str = "lbfgs",
+        **kwargs: Any,
+    ) -> None:
+        # Passes all telemetry tracking arguments automatically to the base class
+        super().__init__(**kwargs)
+
+        self.C = C
+        self.max_iter = max_iter
+        self.solver = solver
+        self.clf = LogisticRegression(C=C, max_iter=max_iter, solver=solver)

modeldiet/benchmark/models/rf.py

@@ -0,0 +1,22 @@
+from typing import Any
+
+from sklearn.ensemble import RandomForestClassifier
+
+from modeldiet.benchmark.model import Model
+
+
+class RandomForestModel(Model):
+    def __init__(
+        self,
+        n_estimators: int = 100,
+        max_depth: int | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+
+        # specific class configurations
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.clf = RandomForestClassifier(
+            n_estimators=self.n_estimators, max_depth=self.max_depth
+        )

modeldiet/benchmark/models/xgb.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+from xgboost import XGBClassifier
+
+from modeldiet.benchmark.model import Model
+
+
+class XGBoostModel(Model):
+    def __init__(
+        self,
+        n_estimators: int = 100,
+        max_depth: int = 6,
+        learning_rate: float = 0.3,
+        subsample: float = 1.0,
+        **kwargs: Any,
+    ) -> None:
+        # Passes all telemetry tracking metrics straight to the base class
+        super().__init__(**kwargs)
+
+        self.n_estimators = n_estimators
+        self.max_depth = max_depth
+        self.learning_rate = learning_rate
+        self.subsample = subsample
+
+        self.clf = XGBClassifier(
+            n_estimators=n_estimators,
+            max_depth=max_depth,
+            learning_rate=learning_rate,
+            subsample=subsample,
+        )

modeldiet/benchmark/report.py

@@ -0,0 +1,60 @@
+import warnings
+from typing import Any
+
+from sklearn.metrics import accuracy_score
+
+from modeldiet.benchmark.model import Model
+
+
+class Report:
+    def __init__(self, model: Model, X_test: Any, y_test: Any) -> None:
+        warnings.warn(
+            "Report is a deprecated benchmark compatibility interface. Use "
+            "run_repeated_benchmark() for demonstration benchmark output; use "
+            "profile() and ProfileResult exports for general-purpose profiling. "
+            "Report will remain available through the 0.x series and will not "
+            "be removed before 1.0.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.model = model
+        self.X_test = X_test
+        self.y_test = y_test
+
+    def print_report(self) -> None:
+        y_pred = self.model.clf.predict(self.X_test)
+        accuracy = accuracy_score(self.y_test, y_pred)
+        model_name = self.model.__class__.__name__
+
+        print("=" * 60)
+        print(f" BENCHMARK REPORT: {model_name.upper()} ".center(60, "="))
+        print("=" * 60)
+        print("\n [1] MODEL METADATA & ACCURACY")
+        print(f" • Model Architecture       : {model_name}")
+        print(f" • Model Deserialization    : {self.model.loading_time:.4f} sec")
+        print(f" • Test Dataset Accuracy    : {accuracy * 100:.2f}%")
+
+        print("\n [2] INFERENCE LATENCY PROFILE")
+        print(
+            f" • Mean Inference Latency   : {self.model.mean_inference_latency:.4f} ms"
+        )
+        print(f" • P50 Latency              : {self.model.p50_latency:.4f} ms")
+        print(f" • P90 Latency              : {self.model.p90_latency:.4f} ms")
+        print(f" • P95 Latency              : {self.model.p95_latency:.4f} ms")
+        print(f" • P99 Latency              : {self.model.p99_latency:.4f} ms")
+
+        print("\n [3] MEMORY & STORAGE FOOTPRINT PROFILE")
+        print(
+            " • Traced Peak Inference Allocation: "
+            f"{self.model.traced_peak_inference_allocation_mb:.2f} MB"
+        )
+        print(
+            " • Process RSS Before / After     : "
+            f"{self.model.rss_before_inference_mb:.2f} / "
+            f"{self.model.rss_after_inference_mb:.2f} MB"
+        )
+        print(f" • Process RSS Delta               : {self.model.rss_delta_mb:.4f} MB")
+        print(
+            f" • Pickle Serialized Model Size   : {self.model.serialized_model_size:.4f} MB"
+        )
+        print("\n" + "=" * 60 + "\n")

modeldiet/benchmark/runner.py

@@ -0,0 +1,75 @@
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from modeldiet.benchmark.statistics import summarize_runs
+
+
+def run_repeated_benchmark(
+    model_name: str,
+    repetitions: int = 10,
+    output_dir: str | Path | None = None,
+) -> dict[str, dict[str, float]]:
+    """Run repeated benchmarks, writing results under output_dir or ./results."""
+    if repetitions < 1:
+        raise ValueError("repetitions must be at least one")
+
+    result_dir = Path(output_dir) if output_dir is not None else Path.cwd() / "results"
+    raw_dir = result_dir / "raw_runs"
+    summary_dir = result_dir / "summaries"
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    summary_dir.mkdir(parents=True, exist_ok=True)
+    runs: list[dict[str, Any]] = []
+
+    print("\n" + "=" * 60)
+    print(f"REPEATED BENCHMARK: {model_name.upper()}")
+    print("=" * 60)
+
+    for iteration in range(1, repetitions + 1):
+        print(f"\nRunning {model_name.upper()} - Iteration {iteration}/{repetitions}")
+        output_file = raw_dir / f"{model_name}_run_{iteration}.json"
+        command = [
+            sys.executable,
+            "-m",
+            "modeldiet.benchmark.worker",
+            "--model",
+            model_name,
+            "--output",
+            str(output_file),
+        ]
+        result = subprocess.run(
+            command,
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0:
+            print("Run failed!")
+            print(result.stderr)
+            continue
+
+        print(result.stdout)
+        with output_file.open(encoding="utf-8") as result_file:
+            runs.append(json.load(result_file))
+
+    if not runs:
+        raise RuntimeError(f"All benchmark runs failed for {model_name}")
+
+    summary = summarize_runs(runs)
+    summary_file = summary_dir / f"{model_name}_summary.json"
+    with summary_file.open("w", encoding="utf-8") as summary_stream:
+        json.dump(summary, summary_stream, indent=4)
+
+    print("\nSTATISTICAL SUMMARY")
+    print("=" * 60)
+    for metric, stats in summary.items():
+        print(f"\n{metric}")
+        print(f"Mean : {stats['mean']:.4f}")
+        print(f"Std  : {stats['std_dev']:.4f}")
+        print(f"CV   : {stats['coefficient_of_variation']:.2f}%")
+        print(f"P95  : {stats['p95']:.4f}")
+
+    print("\nSaved:", summary_file)
+    return summary

modeldiet/benchmark/statistics.py

@@ -0,0 +1,63 @@
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+import numpy as np
+
+
+def calculate_statistics(values: Sequence[float]) -> dict[str, float]:
+    """Calculate statistics across independent benchmark runs."""
+    measurements = np.asarray(values, dtype=float)
+    if len(measurements) == 0:
+        raise ValueError("No benchmark measurements provided")
+
+    mean = float(np.mean(measurements))
+    std = float(np.std(measurements, ddof=1)) if len(measurements) > 1 else 0.0
+    cv = (std / mean * 100) if mean != 0 else 0.0
+    return {
+        "mean": mean,
+        "median": float(np.median(measurements)),
+        "std_dev": std,
+        "coefficient_of_variation": float(cv),
+        "min": float(np.min(measurements)),
+        "max": float(np.max(measurements)),
+        "p90": float(np.percentile(measurements, 90)),
+        "p95": float(np.percentile(measurements, 95)),
+        "p99": float(np.percentile(measurements, 99)),
+    }
+
+
+def summarize_runs(
+    runs: Sequence[Mapping[str, Any]],
+) -> dict[str, dict[str, float]]:
+    """Aggregate measurements from multiple independent benchmark runs."""
+    metrics = [
+        "training_time",
+        "loading_time",
+        "mean_inference_latency",
+        "p50_latency",
+        "p90_latency",
+        "p95_latency",
+        "p99_latency",
+        "traced_peak_inference_allocation_mb",
+        "rss_before_inference_mb",
+        "rss_after_inference_mb",
+        "rss_delta_mb",
+        "serialized_model_size",
+    ]
+    legacy_names = {
+        "traced_peak_inference_allocation_mb": "traced_peak_memory_usage",
+        "rss_after_inference_mb": "processed_rss",
+    }
+    summary: dict[str, dict[str, float]] = {}
+    for metric in metrics:
+        legacy_name = legacy_names.get(metric)
+        values = []
+        for run in runs:
+            if metric in run:
+                values.append(run[metric])
+            elif legacy_name is not None and legacy_name in run:
+                values.append(run[legacy_name])
+        values = [value for value in values if value is not None]
+        if values:
+            summary[metric] = calculate_statistics(values)
+    return summary