expops 0.1.3__py3-none-any.whl

mlops/storage/interfaces/kv_store.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from typing import Any, Optional, Protocol
+
+
+class KeyValueEventStore(Protocol):
+    """Key/value store + events interface used by the platform."""
+
+    # Cache indices (strict-hash match)
+    def set_step_cache_record(
+        self,
+        process_name: str,
+        step_name: str,
+        input_hash: str,
+        config_hash: str,
+        function_hash: str | None,
+        record: dict[str, Any],
+        ttl_seconds: int | None = None,
+    ) -> None: ...
+
+    def get_step_cache_path(
+        self,
+        process_name: str,
+        step_name: str,
+        input_hash: str | None,
+        config_hash: str | None,
+        function_hash: str | None,
+    ) -> str | None: ...
+
+    def get_step_cache_record(
+        self,
+        process_name: str,
+        step_name: str,
+        input_hash: str | None,
+        config_hash: str | None,
+        function_hash: str | None,
+    ) -> dict[str, Any] | None: ...
+
+    def set_process_cache_record(
+        self,
+        process_name: str,
+        input_hash: str,
+        config_hash: str,
+        function_hash: str | None,
+        record: dict[str, Any],
+        ttl_seconds: int | None = None,
+    ) -> None: ...
+
+    def get_process_cache_path(
+        self,
+        process_name: str,
+        input_hash: str | None,
+        config_hash: str | None,
+        function_hash: str | None,
+    ) -> str | None: ...
+
+    def get_process_cache_record(
+        self,
+        process_name: str,
+        input_hash: str | None,
+        config_hash: str | None,
+        function_hash: str | None,
+    ) -> dict[str, Any] | None: ...
+
+    # Optional: batched cache lookups (implement when backend supports efficient multi-get)
+    def get_process_cache_paths_batch(
+        self,
+        lookups: list[tuple[str, str | None, str | None, str | None]],
+    ) -> dict[str, str | None]: ...
+
+    # Run lifecycle + metrics
+    def mark_pipeline_started(self, run_id: str) -> None: ...
+    def mark_pipeline_completed(self, run_id: str, success: bool) -> None: ...
+    def get_run_status(self, run_id: str) -> str | None: ...
+
+    # Events
+    def publish_event(self, event: dict[str, Any]) -> None: ...
+
+    # Per-run step bookkeeping (for resume/get_step_results)
+    def record_run_step(self, run_id: str, process_name: str, step_name: str, record: dict[str, Any]) -> None: ...
+    def list_run_steps(self, run_id: str) -> dict[str, dict[str, Any]]: ...
+
+    # Stats
+    def increment_stat(self, run_id: str, name: str, amount: int = 1) -> None: ...
+    def get_pipeline_stats(self, run_id: str) -> dict[str, Any]: ...
+
+    # Charts/artifacts index per run (optional but recommended)
+    def record_run_chart_artifacts(self, run_id: str, chart_name: str, artifacts: list[dict[str, Any]]) -> None: ...
+    def list_run_charts(self, run_id: str) -> dict[str, Any]: ...
+    def copy_run_chart_artifacts(self, from_run_id: str, to_run_id: str, chart_name: str) -> bool: ...
+
+    # Run listing for UI (optional)
+    def list_runs(self, limit: int = 100) -> list[str]: ...
+
+    # Probe metrics (keyed by probe_path)
+    def save_probe_metrics_by_path(self, run_id: str, probe_path: str, metrics: dict[str, Any]) -> None: ...
+    def get_probe_metrics_by_path(self, run_id: str, probe_path: str) -> dict[str, Any]: ...
+
+
+
+
+# -------------------- Object storage protocol --------------------
+class ObjectStore(Protocol):
+    """Abstraction for binary/object storage backends (e.g., GCS/S3).
+
+    Implementations operate on opaque URIs (e.g., gs://bucket/prefix/key.pkl).
+    """
+
+    def put_bytes(self, uri: str, data: bytes, content_type: str | None = None) -> None: ...
+
+    def put_file(self, uri: str, file_path: str, content_type: str | None = None) -> None: ...
+
+    def get_bytes(self, uri: str) -> bytes: ...
+
+    def exists(self, uri: str) -> bool: ...
+
+    def build_uri(self, *parts: str) -> str: ...
+

mlops/storage/path_utils.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import base64
+
+
+def encode_probe_path(probe_path: str) -> str:
+    """Encode a logical probe_path into a Firestore/Redis-safe identifier.
+
+    Uses URL-safe base64 without padding and a small prefix to avoid pure-numeric IDs.
+    """
+    raw = str(probe_path).encode("utf-8")
+    enc = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
+    return f"p_{enc}"
+
+
+def decode_probe_path(encoded_id: str) -> str:
+    """Decode an encoded probe_path identifier back to the logical path.
+
+    Raises ValueError if the identifier cannot be decoded.
+    """
+    payload = str(encoded_id)
+    if payload.startswith("p_"):
+        payload = payload[2:]
+    # Restore base64 padding
+    pad = "=" * (-len(payload) % 4)
+    try:
+        raw = base64.urlsafe_b64decode(payload + pad)
+        return raw.decode("utf-8")
+    except Exception as e:
+        raise ValueError(f"Invalid encoded probe path id: {encoded_id}") from e
+
+
+__all__ = [
+    "encode_probe_path",
+    "decode_probe_path",
+]
+
+

mlops/templates/premier-league/charts/plot_metrics.js

@@ -0,0 +1,70 @@
+import { chart } from '/mlops-charts.js';
+
+// Dynamic chart: NN A vs NN B vs NN C losses on same chart
+chart('nn_losses', (probePaths, ctx, listener) => {
+  const canvas = document.createElement('canvas');
+  ctx.containerElement.innerHTML = '';
+  ctx.containerElement.appendChild(canvas);
+
+  const colors = [
+    'rgb(75, 192, 192)',
+    'rgb(255, 99, 132)',
+    'rgb(54, 162, 235)'
+  ];
+
+  const chartData = {
+    labels: [],
+    datasets: []
+  };
+
+  let colorIndex = 0;
+  const keys = Object.keys(probePaths);
+  keys.forEach((k) => {
+    chartData.datasets.push({
+      label: k.replace(/_/g, ' ').toUpperCase(),
+      data: [],
+      borderColor: colors[colorIndex % colors.length],
+      backgroundColor: colors[colorIndex % colors.length] + '33',
+      tension: 0.1,
+      fill: false
+    });
+    colorIndex++;
+  });
+
+  const chartInstance = new Chart(canvas, {
+    type: 'line',
+    data: chartData,
+    options: {
+      responsive: true,
+      maintainAspectRatio: true,
+      scales: {
+        x: { title: { display: true, text: 'Epoch' } },
+        y: { title: { display: true, text: 'Loss' }, beginAtZero: false }
+      },
+      plugins: {
+        title: { display: true, text: 'NN Training Loss (A/B)' },
+        legend: { display: true }
+      },
+      animation: { duration: 200 }
+    }
+  });
+  ctx.setChartInstance(chartInstance);
+
+  listener.subscribeAll(probePaths, (allMetrics) => {
+    let maxLength = 0;
+
+    chartData.datasets.forEach((dataset, idx) => {
+      const probeKey = keys[idx];
+      const metrics = allMetrics[probeKey] || {};
+      const lossSeries = ctx.toSeries(metrics.train_loss || {});
+      dataset.data = lossSeries;
+      maxLength = Math.max(maxLength, lossSeries.length);
+    });
+
+    chartData.labels = Array.from({ length: maxLength }, (_, i) => i + 1);
+
+    chartInstance.update();
+  });
+});
+
+

mlops/templates/premier-league/charts/plot_metrics.py

@@ -0,0 +1,145 @@
+from typing import Dict, Any
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import numpy as np
+
+from mlops.reporting import chart, ChartContext
+
+
+@chart()
+def pca_scree(metrics: Dict[str, Any], ctx: ChartContext) -> None:
+    """
+    Static chart showing PCA explained variance ratio and cumulative variance.
+    Expects metrics from feature_engineering/encode_and_pca step containing:
+      - pca_explained_variance_ratio: list[float]
+      - pca_cumulative_variance: list[float]
+    """
+
+    evr = metrics.get('feat', {}).get('pca_explained_variance_ratio', [])
+    cum = metrics.get('feat', {}).get('pca_cumulative_variance', [])
+
+    if not isinstance(evr, (list, tuple)) or len(evr) == 0:
+        return
+
+    xs = np.arange(1, len(evr) + 1)
+    fig, ax1 = plt.subplots(figsize=(10, 4))
+    ax1.bar(xs, evr, color='steelblue', alpha=0.7, label='Explained Variance Ratio')
+    ax1.set_xlabel('Principal Component')
+    ax1.set_ylabel('Explained Variance Ratio')
+    ax1.grid(True, axis='y', alpha=0.3)
+
+    ax2 = ax1.twinx()
+    if isinstance(cum, (list, tuple)) and len(cum) == len(evr):
+        ax2.plot(xs, cum, color='coral', marker='o', label='Cumulative Variance')
+        ax2.set_ylabel('Cumulative Variance')
+
+    fig.tight_layout()
+    ctx.savefig('pca_scree.png', dpi=150, fig=fig)
+    plt.close(fig)
+
+
+@chart()
+def goals_distribution(metrics: Dict[str, Any], ctx: ChartContext) -> None:
+    """
+    Static chart showing histograms of home and away goals.
+    Expects metrics from feature_engineering/encode_and_pca step containing:
+      - goals_hist_home: dict[str, int]
+      - goals_hist_away: dict[str, int]
+    """
+
+    feat_metrics = metrics.get('feat', {})
+    g_home = feat_metrics.get('goals_hist_home', {}) or {}
+    g_away = feat_metrics.get('goals_hist_away', {}) or {}
+
+    if not g_home and not g_away:
+        return
+
+    keys = sorted(set([int(k) for k in g_home.keys()] + [int(k) for k in g_away.keys()]))
+    vals_home = [int(g_home.get(str(k), 0)) for k in keys]
+    vals_away = [int(g_away.get(str(k), 0)) for k in keys]
+
+    x = np.arange(len(keys))
+    width = 0.4
+    fig, ax = plt.subplots(figsize=(10, 4))
+    ax.bar(x - width/2, vals_home, width, label='Home Goals', color='slateblue')
+    ax.bar(x + width/2, vals_away, width, label='Away Goals', color='seagreen')
+    ax.set_xticks(x, [str(k) for k in keys])
+    ax.set_xlabel('Goals')
+    ax.set_ylabel('Count')
+    ax.set_title('Goals Distribution (Home vs Away)')
+    ax.legend()
+    ax.grid(True, axis='y', alpha=0.3)
+    fig.tight_layout()
+    ctx.savefig('goals_distribution.png', dpi=150, fig=fig)
+    plt.close(fig)
+
+
+@chart()
+def test_metrics_comparison(metrics: Dict[str, Any], ctx: ChartContext) -> None:
+    """
+    Static chart comparing classification metrics across baseline/best models and ensemble.
+    Expected keys: linear, nn_best, xgb_best, ensemble; each with test_accuracy, test_precision, test_f1.
+    """
+
+    def get_value(data):
+        if isinstance(data, dict) and data:
+            items = sorted(data.items(), key=lambda x: int(x[0]) if str(x[0]).isdigit() else 0)
+            return float(items[-1][1]) if items else None
+        if isinstance(data, (int, float)):
+            return float(data)
+        return None
+
+    groups = {
+        'Linear': metrics.get('linear', {}),
+        'NN (Best)': metrics.get('nn_best', {}),
+        'XGB (Best)': metrics.get('xgb_best', {}),
+        'Ensemble': metrics.get('ensemble', {}),
+    }
+
+    labels = []
+    accs = []
+    precs = []
+    f1s = []
+    for label, m in groups.items():
+        acc = get_value(m.get('test_accuracy'))
+        prec = get_value(m.get('test_precision'))
+        f1 = get_value(m.get('test_f1'))
+        if all(v is None for v in (acc, prec, f1)):
+            continue
+        labels.append(label)
+        accs.append(acc if acc is not None else 0.0)
+        precs.append(prec if prec is not None else 0.0)
+        f1s.append(f1 if f1 is not None else 0.0)
+
+    if not labels:
+        return
+
+    x = np.arange(len(labels))
+    width = 0.25
+    fig, ax = plt.subplots(figsize=(12, 4))
+    r1 = ax.bar(x - width, accs, width, label='Accuracy', color='steelblue')
+    r2 = ax.bar(x, precs, width, label='Precision (macro)', color='mediumseagreen')
+    r3 = ax.bar(x + width, f1s, width, label='F1 (macro)', color='coral')
+
+    ax.set_ylabel('Score')
+    ax.set_xticks(x, labels)
+    ax.set_ylim(0.0, 1.0)
+    ax.grid(True, axis='y', alpha=0.3)
+    ax.legend(loc='upper left')
+
+    # Annotate bars
+    for rect in list(r1) + list(r2) + list(r3):
+        height = rect.get_height()
+        ax.annotate(f'{height:.3f}',
+                    xy=(rect.get_x() + rect.get_width() / 2, height),
+                    xytext=(0, 3),
+                    textcoords="offset points",
+                    ha='center', va='bottom', fontsize=8)
+
+    fig.tight_layout()
+    ctx.savefig('test_metrics_comparison.png', dpi=150, fig=fig)
+    plt.close(fig)
+
+

mlops/templates/premier-league/charts/requirements.txt

@@ -0,0 +1,6 @@
+matplotlib>=3.8.0
+numpy>=1.24.0
+
+
+
+

mlops/templates/premier-league/configs/cluster_config.yaml

@@ -0,0 +1,13 @@
+provider: slurm
+
+num_workers: 7
+
+options:
+  worker_cores: 1
+  worker_memory: "3GB"
+  worker_processes: 1
+  queue: null
+  walltime: "01:00:00"
+  scheduler_address: ""
+
+

mlops/templates/premier-league/configs/project_config.yaml

@@ -0,0 +1,207 @@
+metadata:
+  name: "premier-league"
+  description: "Premier League 3-class classification with generic FE, NN A/B, XGB A/B, logistic baseline, and ensemble"
+  version: "1.0.0"
+
+environment:
+  venv:
+    name: "premier-league-env"
+    requirements_file: "projects/premier-league/requirements.txt"
+    reporting:
+      name: "premier-league-env-reporting"
+      requirements_file: "projects/premier-league/charts/requirements.txt"
+
+reproducibility:
+  random_seed: 43
+
+model:
+  framework: "custom"
+  language: "python"
+  name: "premier_league_model"
+  version: "1.0.0"
+  parameters:
+    custom_script_path: "projects/premier-league/models/premier_league_model.py"
+    cache:
+      backend:
+        type: gcp
+        gcp_project: mlops-platform-470017
+        credentials_json: keys/firestore.json
+      object_store:
+        type: gcs
+        bucket: mlops-platform123
+        prefix: projects/premier-league/cache/steps
+    executor:
+      n_workers: 4
+    hyperparameters:
+      test_size: 0.2
+      pca_components: 21
+
+    pipeline:
+      process_adjlist: |
+        feature_engineering_generic preprocess_linear_nn
+        feature_engineering_generic preprocess_xgb
+        feature_engineering_generic pca_scree
+        feature_engineering_generic goals_distribution
+        preprocess_linear_nn linear_training
+        preprocess_linear_nn nn_training_a
+        preprocess_linear_nn nn_training_b
+        preprocess_xgb xgb_training_a
+        preprocess_xgb xgb_training_b
+        linear_training linear_inference
+        nn_training_a nn_inference_a
+        nn_training_b nn_inference_b
+        xgb_training_a xgb_inference_a
+        xgb_training_b xgb_inference_b
+        nn_inference_a nn_best_selection
+        nn_inference_b nn_best_selection
+        xgb_inference_a xgb_best_selection
+        xgb_inference_b xgb_best_selection
+        nn_best_selection nn_best_inference
+        xgb_best_selection xgb_best_inference
+        linear_training ensemble_inference
+        nn_best_selection ensemble_inference
+        xgb_best_selection ensemble_inference
+        linear_inference test_metrics_comparison
+        nn_best_inference test_metrics_comparison
+        xgb_best_inference test_metrics_comparison
+        ensemble_inference test_metrics_comparison
+
+      processes:
+        - name: "feature_engineering_generic"
+          description: "Generic FE: parse dates, derive labels (H/D/A), stratified indices"
+          code_function: "define_feature_engineering_generic_process"
+
+        - name: "preprocess_linear_nn"
+          description: "Preprocess for linear/nn: OHE + StandardScaler"
+          code_function: "define_preprocess_linear_nn_process"
+
+        - name: "preprocess_xgb"
+          description: "Preprocess for xgb: OHE only"
+          code_function: "define_preprocess_xgb_process"
+
+        - name: "linear_training"
+          description: "Train a multinomial Logistic Regression classifier"
+          code_function: "define_linear_training_process"
+
+        - name: "nn_training_a"
+          description: "Train NN classifier branch A"
+          code_function: "define_nn_training_process"
+          hyperparameters:
+            nn_params:
+              hidden_layers: [128, 64]
+              learning_rate: 0.001
+              epochs: 50
+
+        - name: "nn_training_b"
+          description: "Train NN classifier branch B"
+          code_function: "define_nn_training_process"
+          hyperparameters:
+            nn_params:
+              hidden_layers: [256, 128, 64]
+              learning_rate: 0.0008
+              epochs: 50
+
+        - name: "xgb_training_a"
+          description: "Train an XGBoost classifier (branch A)"
+          code_function: "define_xgb_training_process"
+          hyperparameters:
+            xgb_params:
+              n_estimators: 350
+              max_depth: 4
+              learning_rate: 0.1
+              subsample: 0.8
+              colsample_bytree: 0.9
+
+        - name: "xgb_training_b"
+          description: "Train an XGBoost classifier (branch B) with different hyperparameters"
+          code_function: "define_xgb_training_process"
+          hyperparameters:
+            xgb_params:
+              n_estimators: 500
+              max_depth: 8
+              learning_rate: 0.05
+              subsample: 0.8
+              colsample_bytree: 0.8
+
+        - name: "nn_inference_a"
+          description: "Run test inference for NN branch A and compute classification metrics"
+          code_function: "define_nn_inference_process"
+          hyperparameters:
+            train_key: nn_training_a
+
+        - name: "nn_inference_b"
+          description: "Run test inference for NN branch B and compute classification metrics"
+          code_function: "define_nn_inference_process"
+          hyperparameters:
+            train_key: nn_training_b
+
+        - name: "xgb_inference_a"
+          description: "Run test inference for XGBoost branch A and compute classification metrics"
+          code_function: "define_xgb_inference_process"
+          hyperparameters:
+            train_key: xgb_training_a
+
+        - name: "xgb_inference_b"
+          description: "Run test inference for XGBoost branch B and compute classification metrics"
+          code_function: "define_xgb_inference_process"
+          hyperparameters:
+            train_key: xgb_training_b
+
+        - name: "linear_inference"
+          description: "Run test inference for Logistic Regression and compute classification metrics"
+          code_function: "define_linear_inference_process"
+
+        - name: "nn_best_selection"
+          description: "Select best NN by F1 from A/B"
+          code_function: "define_select_best_nn_process"
+
+        - name: "xgb_best_selection"
+          description: "Select best XGB by F1 from A/B"
+          code_function: "define_select_best_xgb_process"
+
+        - name: "nn_best_inference"
+          description: "Stable best NN inference for charts"
+          code_function: "define_nn_best_inference_process"
+
+        - name: "xgb_best_inference"
+          description: "Stable best XGB inference for charts"
+          code_function: "define_xgb_best_inference_process"
+
+        - name: "ensemble_inference"
+          description: "Weighted soft-vote ensemble across Logistic, best NN, best XGB"
+          code_function: "define_ensemble_inference_process"
+
+        - name: "pca_scree"
+          type: chart
+          description: "Render PCA scree plot"
+
+        - name: "goals_distribution"
+          type: chart
+          description: "Render goals distribution plot"
+
+        - name: "test_metrics_comparison"
+          type: chart
+          description: "Compare classification metrics across models and ensemble"
+
+
+reporting:
+  static_entrypoint: "projects/premier-league/charts/plot_metrics.py"
+  dynamic_entrypoint: "projects/premier-league/charts/plot_metrics.js"
+  charts:
+    - name: "pca_scree"
+      probe_paths:
+        feat: "feature_engineering_generic/feature_analysis"
+    - name: "goals_distribution"
+      probe_paths:
+        feat: "feature_engineering_generic/derive_labels_and_indices"
+    - name: "test_metrics_comparison"
+      probe_paths:
+        linear: "linear_inference/test_inference_classification"
+        nn_best: "nn_best_inference/test_inference_classification"
+        xgb_best: "xgb_best_inference/test_inference_classification"
+        ensemble: "ensemble_inference"
+    - name: "nn_losses"
+      type: dynamic
+      probe_paths:
+        nn_a: "nn_training_a/train_and_evaluate_nn_classifier"
+        nn_b: "nn_training_b/train_and_evaluate_nn_classifier"