openrunner-sdk 2.3.0__tar.gz → 2.4.1__tar.gz

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.3.0
+Version: 2.4.1
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner
@@ -82,6 +82,8 @@ Requires-Dist: numpy>=1.24; extra == 'tts'
 Provides-Extra: ultralytics
 Requires-Dist: ultralytics>=8.0; extra == 'ultralytics'
 Provides-Extra: voice-agent
+Provides-Extra: wer
+Requires-Dist: num2words2>=0.1; extra == 'wer'
 Provides-Extra: whisper
 Requires-Dist: openai-whisper>=20231117; extra == 'whisper'
 Provides-Extra: xgboost

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/openrunner/__init__.py

@@ -97,6 +97,7 @@ from openrunner.settings import SDKSettings
 from openrunner.summary import Summary
 from openrunner.sweep import agent, sweep
 from openrunner.evaluation import EvaluationLogger, Scorer, evaluate, scorer
+from openrunner.wer import WERScorer, compute_wer, compute_wer_batch
 from openrunner.guardrails import (
     GuardrailCheckResult,
     GuardrailResult,

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/openrunner/api_client.py

@@ -490,6 +490,86 @@ class APIClient:
             logger.warning("use_artifact failed: %s", e)
             return None
 
+    def download_artifact(
+        self,
+        run_id: str,
+        artifact_name: str,
+        dest_dir: str = ".",
+        version: int | None = None,
+        alias: str | None = None,
+    ) -> str | None:
+        """Download all files from an artifact version to a local directory.
+
+        Args:
+            run_id: Run ID that used/created the artifact
+            artifact_name: Artifact name (or "name:alias")
+            dest_dir: Local directory to save files (created if needed)
+            version: Specific version number (optional)
+            alias: Alias name like "latest", "best" (optional)
+
+        Returns:
+            Path to the download directory, or None on failure.
+
+        Example:
+            path = client.download_artifact(run_id, "model-checkpoint", "./models")
+            # Files saved to ./models/model-checkpoint/v3/...
+        """
+        import os
+        from pathlib import Path
+
+        info = self.use_artifact(run_id, artifact_name, version=version, alias=alias)
+        if not info:
+            logger.warning("download_artifact: use_artifact returned no data")
+            return None
+
+        # Extract version info and files
+        ver = info.get("version", version or 1)
+        files = info.get("files", [])
+        if not files:
+            logger.warning("download_artifact: no files in artifact")
+            return None
+
+        # Create destination directory
+        name_clean = artifact_name.split(":")[0].replace("/", "_")
+        out_dir = Path(dest_dir) / name_clean / f"v{ver}"
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        downloaded = 0
+        for f in files:
+            url = f.get("presigned_url") or f.get("url") or f.get("download_url")
+            fname = f.get("name") or f.get("path") or f"file_{downloaded}"
+            if not url:
+                continue
+
+            # Try presigned URL first, fall back to proxy
+            data = self.download_file_from_presigned_url(url)
+            if data is None and "localhost" in url:
+                # Try proxy
+                key = f.get("storage_key") or f.get("key")
+                if key:
+                    try:
+                        resp = self._request("GET", f"/storage/download?key={key}")
+                        if resp.status_code == 200:
+                            data = resp.content
+                    except Exception:
+                        pass
+
+            if data:
+                file_path = out_dir / fname
+                file_path.parent.mkdir(parents=True, exist_ok=True)
+                file_path.write_bytes(data)
+                downloaded += 1
+                logger.info("downloaded: %s (%d bytes)", fname, len(data))
+            else:
+                logger.warning("failed to download: %s", fname)
+
+        if downloaded == 0:
+            logger.warning("download_artifact: no files downloaded")
+            return None
+
+        logger.info("artifact downloaded: %d files → %s", downloaded, out_dir)
+        return str(out_dir)
+
     def set_alias(
         self,
         artifact_id: str,

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/openrunner/run.py

@@ -940,10 +940,11 @@ class Run:
             artifact_dir.mkdir(parents=True, exist_ok=True)
 
             for file_info in result.get("files", []):
-                content_hash = file_info.get("content_hash", "")
-                cached = self._artifact_cache.get(content_hash)
+                content_hash = file_info.get("content_hash") or ""
+                file_path = file_info.get("name") or file_info.get("path") or "file"
+                cached = self._artifact_cache.get(content_hash) if content_hash else None
 
-                dest = artifact_dir / file_info["path"]
+                dest = artifact_dir / file_path
                 dest.parent.mkdir(parents=True, exist_ok=True)
 
                 if cached:
@@ -954,11 +955,21 @@ class Run:
                         shutil.copy2(str(cached), str(dest))
                 else:
                     # Download and cache
-                    data = self._client.download_file_from_presigned_url(
-                        file_info["presigned_url"]
-                    )
+                    url = file_info.get("download_url") or file_info.get("presigned_url", "")
+                    data = None
+                    if url.startswith("/"):
+                        # Relative proxy URL — use authenticated client
+                        try:
+                            resp = self._client._request("GET", url)
+                            if resp.status_code == 200:
+                                data = resp.content
+                        except Exception:
+                            pass
+                    else:
+                        data = self._client.download_file_from_presigned_url(url)
                     if data:
-                        self._artifact_cache.put(content_hash, data)
+                        if content_hash:
+                            self._artifact_cache.put(content_hash, data)
                         dest.write_bytes(data)
 
             return artifact_dir
@@ -1154,18 +1165,28 @@ class Run:
         name: str,
         version: int | None = None,
         alias: str | None = None,
-    ):
-        """Convenience: download a model artifact.
+        dest_dir: str = "./artifacts",
+    ) -> str | None:
+        """Download a model artifact to disk.
 
         Args:
             name: Model artifact name (supports "name:alias" syntax).
             version: Specific version number, or None for latest.
             alias: Alias name to resolve.
+            dest_dir: Local directory for downloaded files.
 
         Returns:
             Path to the local artifact directory, or None on failure.
         """
-        return self.use_artifact(name, version=version, alias=alias)
+        if not self._client:
+            return None
+        return self._client.download_artifact(
+            run_id=self._run_id,
+            artifact_name=name,
+            dest_dir=dest_dir,
+            version=version,
+            alias=alias,
+        )
 
     def link_model(
         self,

openrunner_sdk-2.4.1/openrunner/wer.py

@@ -0,0 +1,232 @@
+"""Word Error Rate (WER) computation with num2words2 normalization.
+
+Normalizes numbers to words before comparison so "50" vs "fifty" aren't
+counted as substitution errors. Uses num2words2 (modern fork optimized
+for LLM/AI/speech applications).
+
+Install: pip install num2words2
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _normalize_text(text: str, language: str = "en") -> str:
+    """Normalize text for WER: lowercase, expand numbers, strip punctuation."""
+    text = text.lower().strip()
+
+    # Expand numbers to words using num2words2
+    try:
+        from num2words2 import num2words
+
+        def _expand_number(match: re.Match) -> str:
+            num_str = match.group(0)
+            try:
+                # Handle decimals
+                if "." in num_str:
+                    return num2words(float(num_str), lang=language)
+                # Handle integers
+                return num2words(int(num_str), lang=language)
+            except (ValueError, OverflowError):
+                return num_str
+
+        # Match numbers (integers, decimals, negatives)
+        text = re.sub(r"-?\d+\.?\d*", _expand_number, text)
+
+    except ImportError:
+        pass  # num2words2 not installed — skip normalization
+
+    # Expand common currency symbols
+    text = re.sub(r"\$\s*", "dollars ", text)
+    text = re.sub(r"€\s*", "euros ", text)
+    text = re.sub(r"£\s*", "pounds ", text)
+    text = re.sub(r"%", " percent", text)
+
+    # Strip punctuation (keep hyphens inside words for compound words)
+    text = re.sub(r"[^\w\s-]", " ", text)
+    # Collapse whitespace
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text
+
+
+def compute_wer(
+    reference: str,
+    hypothesis: str,
+    normalize: bool = True,
+    language: str = "en",
+) -> dict[str, Any]:
+    """Compute Word Error Rate between reference and hypothesis.
+
+    Args:
+        reference: Ground truth transcription
+        hypothesis: Model prediction
+        normalize: If True, expand numbers with num2words2 before comparing
+        language: Language code for num2words2 (en, es, fr, de, etc.)
+
+    Returns:
+        Dict with: wer, substitutions, insertions, deletions, ref_words, hyp_words
+    """
+    if normalize:
+        ref = _normalize_text(reference, language)
+        hyp = _normalize_text(hypothesis, language)
+    else:
+        ref = reference.lower().strip()
+        hyp = hypothesis.lower().strip()
+
+    ref_words = ref.split()
+    hyp_words = hyp.split()
+
+    # Edit distance DP
+    m, n = len(ref_words), len(hyp_words)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    for i in range(m + 1):
+        dp[i][0] = i
+    for j in range(n + 1):
+        dp[0][j] = j
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            if ref_words[i - 1] == hyp_words[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1]
+            else:
+                dp[i][j] = 1 + min(
+                    dp[i - 1][j - 1],  # substitution
+                    dp[i - 1][j],      # deletion
+                    dp[i][j - 1],      # insertion
+                )
+
+    # Backtrace for error counts
+    subs, dels, ins = 0, 0, 0
+    i, j = m, n
+    while i > 0 or j > 0:
+        if i > 0 and j > 0 and ref_words[i - 1] == hyp_words[j - 1]:
+            i -= 1
+            j -= 1
+        elif i > 0 and j > 0 and dp[i][j] == dp[i - 1][j - 1] + 1:
+            subs += 1
+            i -= 1
+            j -= 1
+        elif i > 0 and dp[i][j] == dp[i - 1][j] + 1:
+            dels += 1
+            i -= 1
+        else:
+            ins += 1
+            j -= 1
+
+    total_errors = subs + dels + ins
+    wer_score = total_errors / max(len(ref_words), 1)
+
+    return {
+        "wer": round(wer_score, 4),
+        "substitutions": subs,
+        "insertions": ins,
+        "deletions": dels,
+        "errors": total_errors,
+        "ref_words": len(ref_words),
+        "hyp_words": len(hyp_words),
+    }
+
+
+def compute_wer_batch(
+    references: list[str],
+    hypotheses: list[str],
+    normalize: bool = True,
+    language: str = "en",
+) -> dict[str, Any]:
+    """Compute WER across a batch of reference/hypothesis pairs.
+
+    Returns aggregate metrics + per-example breakdown.
+    """
+    total_errors = 0
+    total_ref_words = 0
+    total_subs = 0
+    total_ins = 0
+    total_dels = 0
+    examples = []
+
+    for ref, hyp in zip(references, hypotheses):
+        result = compute_wer(ref, hyp, normalize=normalize, language=language)
+        total_errors += result["errors"]
+        total_ref_words += result["ref_words"]
+        total_subs += result["substitutions"]
+        total_ins += result["insertions"]
+        total_dels += result["deletions"]
+        examples.append(result)
+
+    wer_score = total_errors / max(total_ref_words, 1)
+
+    return {
+        "wer": round(wer_score, 4),
+        "substitutions": total_subs,
+        "insertions": total_ins,
+        "deletions": total_dels,
+        "errors": total_errors,
+        "ref_words": total_ref_words,
+        "n_examples": len(references),
+        "examples": examples,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Scorer integration for openrunner.evaluate()
+# ---------------------------------------------------------------------------
+
+from openrunner.evaluation import Scorer
+
+
+class WERScorer(Scorer):
+    """Word Error Rate scorer for evaluation framework.
+
+    Uses num2words2 to normalize numbers before comparison.
+
+    Args:
+        normalize: Expand numbers to words (default True)
+        language: Language for number expansion (default "en")
+        ref_key: Key in the example dict for ground truth (default "expected")
+        hyp_key: Key in the output for hypothesis (default: uses output directly)
+
+    Example:
+        results = openrunner.evaluate(
+            model_fn=my_asr_model,
+            dataset=[{"input": audio, "expected": "fifty dollars"}],
+            scorers=[WERScorer(language="en")],
+        )
+    """
+
+    def __init__(
+        self,
+        normalize: bool = True,
+        language: str = "en",
+    ):
+        self.normalize = normalize
+        self.language = language
+
+    def score(self, output: Any, expected: Any, **kwargs) -> dict:
+        ref = str(expected) if expected else ""
+        hyp = str(output) if output else ""
+        result = compute_wer(ref, hyp, normalize=self.normalize, language=self.language)
+        return {
+            "wer": result["wer"],
+            "substitutions": result["substitutions"],
+            "insertions": result["insertions"],
+            "deletions": result["deletions"],
+        }
+
+    def summarize(self, scores: list[dict]) -> dict:
+        """Aggregate WER across all examples (corpus-level)."""
+        total_errors = sum(s["substitutions"] + s["insertions"] + s["deletions"] for s in scores)
+        # Approximate ref_words from individual WERs
+        total_ref = sum(
+            round((s["substitutions"] + s["insertions"] + s["deletions"]) / max(s["wer"], 1e-9))
+            if s["wer"] > 0 else 10
+            for s in scores
+        )
+        return {
+            "wer": round(total_errors / max(total_ref, 1), 4),
+            "mean_wer": round(sum(s["wer"] for s in scores) / max(len(scores), 1), 4),
+            "total_substitutions": sum(s["substitutions"] for s in scores),
+            "total_insertions": sum(s["insertions"] for s in scores),
+            "total_deletions": sum(s["deletions"] for s in scores),
+        }

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.3.0"
+version = "2.4.1"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}
@@ -30,6 +30,7 @@ Issues = "https://github.com/jqueguiner/openrunner/issues"
 openrunner = "openrunner.cli:main"
 
 [project.optional-dependencies]
+wer = ["num2words2>=0.1"]
 gpu = ["nvidia-ml-py>=12.0"]
 pytorch = ["torch>=2.0"]
 huggingface = ["transformers>=4.30"]

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/tests/test_alert.py

@@ -192,7 +192,7 @@ class TestModuleLevelAlert:
             result = openrunner.alert("Title", text="Body", level="WARN")
             assert result == {"id": "x"}
             mock_run.alert.assert_called_once_with(
-                title="Title", text="Body", level="WARN"
+                title="Title", text="Body", level="WARN", wait_duration=None
             )
         finally:
             openrunner._active_run = original

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/tests/test_integration_sklearn.py

@@ -70,15 +70,15 @@ class TestLogModel:
         from openrunner.integration.sklearn import log_model
         import openrunner
 
-        mock_log = MagicMock()
-        monkeypatch.setattr("openrunner.log", mock_log)
+        mock_config = MagicMock()
+        monkeypatch.setattr("openrunner.config", mock_config)
         monkeypatch.setattr("openrunner._active_run", MagicMock())
 
         model = _make_model(params={"n_estimators": 100, "max_depth": 5, "random_state": 42})
         log_model(model)
 
-        mock_log.assert_called_once()
-        logged = mock_log.call_args[0][0]
+        mock_config.update.assert_called_once()
+        logged = mock_config.update.call_args[0][0]
         assert logged["model/n_estimators"] == 100
         assert logged["model/max_depth"] == 5
         assert logged["model/random_state"] == 42
@@ -88,12 +88,12 @@ class TestLogModel:
         from openrunner.integration.sklearn import log_model
         import openrunner
 
-        mock_log = MagicMock()
-        monkeypatch.setattr("openrunner.log", mock_log)
+        mock_config = MagicMock()
+        monkeypatch.setattr("openrunner.config", mock_config)
         monkeypatch.setattr("openrunner._active_run", None)
 
         log_model(_make_model())
-        mock_log.assert_not_called()
+        mock_config.update.assert_not_called()
 
 
 class TestLogClassificationReport:

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/tests/test_media.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import pytest
 import io
 import json
 import struct
@@ -586,16 +587,18 @@ class TestHtml:
     """Tests for the Html class."""
 
     def test_html_basic(self) -> None:
-        """Html with raw string serializes correctly."""
+        """Html with raw string serializes correctly (inject=True prepends style)."""
         html = Html("<div>Hello World</div>")
         result = html._serialize()
-        assert result == {"html": "<div>Hello World</div>"}
+        assert result["html"].endswith("<div>Hello World</div>")
+        assert "<style>" in result["html"]
 
     def test_html_with_caption(self) -> None:
         """Html with caption includes it in serialized output."""
         html = Html("<p>Report</p>", caption="Training Report")
         result = html._serialize()
-        assert result == {"html": "<p>Report</p>", "caption": "Training Report"}
+        assert result["html"].endswith("<p>Report</p>")
+        assert result["caption"] == "Training Report"
         assert html.caption == "Training Report"
 
     def test_html_no_caption(self) -> None:
@@ -616,7 +619,7 @@ class TestHtml:
         )
         html = Html(content, caption="complex")
         result = html._serialize()
-        assert result["html"] == content
+        assert result["html"].endswith(content)
         assert result["caption"] == "complex"
 
 
@@ -627,6 +630,11 @@ class TestHtml:
 class TestMatplotlibFigure:
     """Tests for the MatplotlibFigure class."""
 
+    pytestmark = pytest.mark.skipif(
+        not __import__("importlib").util.find_spec("matplotlib"),
+        reason="matplotlib not installed",
+    )
+
     def test_matplotlib_explicit_figure(self) -> None:
         """MatplotlibFigure from explicit figure serializes to PNG bytes."""
         import matplotlib

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/tests/test_plot.py

@@ -208,6 +208,11 @@ class TestLineSeries:
 # pr_curve()
 # ---------------------------------------------------------------------------
 
+import importlib.util
+_has_sklearn = importlib.util.find_spec("sklearn") is not None
+
+
+@pytest.mark.skipif(not _has_sklearn, reason="scikit-learn not installed")
 class TestPRCurve:
     """Tests for plot.pr_curve()."""
 
@@ -272,6 +277,7 @@ class TestPRCurve:
 # roc_curve()
 # ---------------------------------------------------------------------------
 
+@pytest.mark.skipif(not _has_sklearn, reason="scikit-learn not installed")
 class TestROCCurve:
     """Tests for plot.roc_curve()."""
 
@@ -336,6 +342,7 @@ class TestROCCurve:
 # confusion_matrix()
 # ---------------------------------------------------------------------------
 
+@pytest.mark.skipif(not _has_sklearn, reason="scikit-learn not installed")
 class TestConfusionMatrix:
     """Tests for plot.confusion_matrix()."""
 

{openrunner_sdk-2.3.0 → openrunner_sdk-2.4.1}/tests/test_system_metrics.py

@@ -82,8 +82,8 @@ class TestGPUMetrics:
             assert result["system.gpu.0.gpu"] == 75.0
             assert "system.gpu.0.memory" in result
             assert result["system.gpu.0.memory"] == 50.0
-            assert "system.gpu.0.memoryAllocatedBytes" in result
-            assert result["system.gpu.0.memoryAllocatedBytes"] == 4_000_000_000.0
+            assert "system.gpu.0.memoryAllocatedMB" in result
+            assert result["system.gpu.0.memoryAllocatedMB"] == 4_000_000_000.0 / (1024 * 1024)
 
 
 # ---------------------------------------------------------------------------