openrunner-sdk 2.4.0__tar.gz → 2.4.1__tar.gz

{openrunner_sdk-2.4.0 → openrunner_sdk-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.4.0
+Version: 2.4.1
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner
@@ -82,6 +82,8 @@ Requires-Dist: numpy>=1.24; extra == 'tts'
 Provides-Extra: ultralytics
 Requires-Dist: ultralytics>=8.0; extra == 'ultralytics'
 Provides-Extra: voice-agent
+Provides-Extra: wer
+Requires-Dist: num2words2>=0.1; extra == 'wer'
 Provides-Extra: whisper
 Requires-Dist: openai-whisper>=20231117; extra == 'whisper'
 Provides-Extra: xgboost

{openrunner_sdk-2.4.0 → openrunner_sdk-2.4.1}/openrunner/__init__.py

@@ -97,6 +97,7 @@ from openrunner.settings import SDKSettings
 from openrunner.summary import Summary
 from openrunner.sweep import agent, sweep
 from openrunner.evaluation import EvaluationLogger, Scorer, evaluate, scorer
+from openrunner.wer import WERScorer, compute_wer, compute_wer_batch
 from openrunner.guardrails import (
     GuardrailCheckResult,
     GuardrailResult,

{openrunner_sdk-2.4.0 → openrunner_sdk-2.4.1}/openrunner/run.py

@@ -940,10 +940,11 @@ class Run:
             artifact_dir.mkdir(parents=True, exist_ok=True)
 
             for file_info in result.get("files", []):
-                content_hash = file_info.get("content_hash", "")
-                cached = self._artifact_cache.get(content_hash)
+                content_hash = file_info.get("content_hash") or ""
+                file_path = file_info.get("name") or file_info.get("path") or "file"
+                cached = self._artifact_cache.get(content_hash) if content_hash else None
 
-                dest = artifact_dir / file_info["path"]
+                dest = artifact_dir / file_path
                 dest.parent.mkdir(parents=True, exist_ok=True)
 
                 if cached:
@@ -954,11 +955,21 @@ class Run:
                         shutil.copy2(str(cached), str(dest))
                 else:
                     # Download and cache
-                    data = self._client.download_file_from_presigned_url(
-                        file_info["presigned_url"]
-                    )
+                    url = file_info.get("download_url") or file_info.get("presigned_url", "")
+                    data = None
+                    if url.startswith("/"):
+                        # Relative proxy URL — use authenticated client
+                        try:
+                            resp = self._client._request("GET", url)
+                            if resp.status_code == 200:
+                                data = resp.content
+                        except Exception:
+                            pass
+                    else:
+                        data = self._client.download_file_from_presigned_url(url)
                     if data:
-                        self._artifact_cache.put(content_hash, data)
+                        if content_hash:
+                            self._artifact_cache.put(content_hash, data)
                         dest.write_bytes(data)
 
             return artifact_dir

openrunner_sdk-2.4.1/openrunner/wer.py

@@ -0,0 +1,232 @@
+"""Word Error Rate (WER) computation with num2words2 normalization.
+
+Normalizes numbers to words before comparison so "50" vs "fifty" aren't
+counted as substitution errors. Uses num2words2 (modern fork optimized
+for LLM/AI/speech applications).
+
+Install: pip install num2words2
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _normalize_text(text: str, language: str = "en") -> str:
+    """Normalize text for WER: lowercase, expand numbers, strip punctuation."""
+    text = text.lower().strip()
+
+    # Expand numbers to words using num2words2
+    try:
+        from num2words2 import num2words
+
+        def _expand_number(match: re.Match) -> str:
+            num_str = match.group(0)
+            try:
+                # Handle decimals
+                if "." in num_str:
+                    return num2words(float(num_str), lang=language)
+                # Handle integers
+                return num2words(int(num_str), lang=language)
+            except (ValueError, OverflowError):
+                return num_str
+
+        # Match numbers (integers, decimals, negatives)
+        text = re.sub(r"-?\d+\.?\d*", _expand_number, text)
+
+    except ImportError:
+        pass  # num2words2 not installed — skip normalization
+
+    # Expand common currency symbols
+    text = re.sub(r"\$\s*", "dollars ", text)
+    text = re.sub(r"€\s*", "euros ", text)
+    text = re.sub(r"£\s*", "pounds ", text)
+    text = re.sub(r"%", " percent", text)
+
+    # Strip punctuation (keep hyphens inside words for compound words)
+    text = re.sub(r"[^\w\s-]", " ", text)
+    # Collapse whitespace
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text
+
+
+def compute_wer(
+    reference: str,
+    hypothesis: str,
+    normalize: bool = True,
+    language: str = "en",
+) -> dict[str, Any]:
+    """Compute Word Error Rate between reference and hypothesis.
+
+    Args:
+        reference: Ground truth transcription
+        hypothesis: Model prediction
+        normalize: If True, expand numbers with num2words2 before comparing
+        language: Language code for num2words2 (en, es, fr, de, etc.)
+
+    Returns:
+        Dict with: wer, substitutions, insertions, deletions, ref_words, hyp_words
+    """
+    if normalize:
+        ref = _normalize_text(reference, language)
+        hyp = _normalize_text(hypothesis, language)
+    else:
+        ref = reference.lower().strip()
+        hyp = hypothesis.lower().strip()
+
+    ref_words = ref.split()
+    hyp_words = hyp.split()
+
+    # Edit distance DP
+    m, n = len(ref_words), len(hyp_words)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    for i in range(m + 1):
+        dp[i][0] = i
+    for j in range(n + 1):
+        dp[0][j] = j
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            if ref_words[i - 1] == hyp_words[j - 1]:
+                dp[i][j] = dp[i - 1][j - 1]
+            else:
+                dp[i][j] = 1 + min(
+                    dp[i - 1][j - 1],  # substitution
+                    dp[i - 1][j],      # deletion
+                    dp[i][j - 1],      # insertion
+                )
+
+    # Backtrace for error counts
+    subs, dels, ins = 0, 0, 0
+    i, j = m, n
+    while i > 0 or j > 0:
+        if i > 0 and j > 0 and ref_words[i - 1] == hyp_words[j - 1]:
+            i -= 1
+            j -= 1
+        elif i > 0 and j > 0 and dp[i][j] == dp[i - 1][j - 1] + 1:
+            subs += 1
+            i -= 1
+            j -= 1
+        elif i > 0 and dp[i][j] == dp[i - 1][j] + 1:
+            dels += 1
+            i -= 1
+        else:
+            ins += 1
+            j -= 1
+
+    total_errors = subs + dels + ins
+    wer_score = total_errors / max(len(ref_words), 1)
+
+    return {
+        "wer": round(wer_score, 4),
+        "substitutions": subs,
+        "insertions": ins,
+        "deletions": dels,
+        "errors": total_errors,
+        "ref_words": len(ref_words),
+        "hyp_words": len(hyp_words),
+    }
+
+
+def compute_wer_batch(
+    references: list[str],
+    hypotheses: list[str],
+    normalize: bool = True,
+    language: str = "en",
+) -> dict[str, Any]:
+    """Compute WER across a batch of reference/hypothesis pairs.
+
+    Returns aggregate metrics + per-example breakdown.
+    """
+    total_errors = 0
+    total_ref_words = 0
+    total_subs = 0
+    total_ins = 0
+    total_dels = 0
+    examples = []
+
+    for ref, hyp in zip(references, hypotheses):
+        result = compute_wer(ref, hyp, normalize=normalize, language=language)
+        total_errors += result["errors"]
+        total_ref_words += result["ref_words"]
+        total_subs += result["substitutions"]
+        total_ins += result["insertions"]
+        total_dels += result["deletions"]
+        examples.append(result)
+
+    wer_score = total_errors / max(total_ref_words, 1)
+
+    return {
+        "wer": round(wer_score, 4),
+        "substitutions": total_subs,
+        "insertions": total_ins,
+        "deletions": total_dels,
+        "errors": total_errors,
+        "ref_words": total_ref_words,
+        "n_examples": len(references),
+        "examples": examples,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Scorer integration for openrunner.evaluate()
+# ---------------------------------------------------------------------------
+
+from openrunner.evaluation import Scorer
+
+
+class WERScorer(Scorer):
+    """Word Error Rate scorer for evaluation framework.
+
+    Uses num2words2 to normalize numbers before comparison.
+
+    Args:
+        normalize: Expand numbers to words (default True)
+        language: Language for number expansion (default "en")
+        ref_key: Key in the example dict for ground truth (default "expected")
+        hyp_key: Key in the output for hypothesis (default: uses output directly)
+
+    Example:
+        results = openrunner.evaluate(
+            model_fn=my_asr_model,
+            dataset=[{"input": audio, "expected": "fifty dollars"}],
+            scorers=[WERScorer(language="en")],
+        )
+    """
+
+    def __init__(
+        self,
+        normalize: bool = True,
+        language: str = "en",
+    ):
+        self.normalize = normalize
+        self.language = language
+
+    def score(self, output: Any, expected: Any, **kwargs) -> dict:
+        ref = str(expected) if expected else ""
+        hyp = str(output) if output else ""
+        result = compute_wer(ref, hyp, normalize=self.normalize, language=self.language)
+        return {
+            "wer": result["wer"],
+            "substitutions": result["substitutions"],
+            "insertions": result["insertions"],
+            "deletions": result["deletions"],
+        }
+
+    def summarize(self, scores: list[dict]) -> dict:
+        """Aggregate WER across all examples (corpus-level)."""
+        total_errors = sum(s["substitutions"] + s["insertions"] + s["deletions"] for s in scores)
+        # Approximate ref_words from individual WERs
+        total_ref = sum(
+            round((s["substitutions"] + s["insertions"] + s["deletions"]) / max(s["wer"], 1e-9))
+            if s["wer"] > 0 else 10
+            for s in scores
+        )
+        return {
+            "wer": round(total_errors / max(total_ref, 1), 4),
+            "mean_wer": round(sum(s["wer"] for s in scores) / max(len(scores), 1), 4),
+            "total_substitutions": sum(s["substitutions"] for s in scores),
+            "total_insertions": sum(s["insertions"] for s in scores),
+            "total_deletions": sum(s["deletions"] for s in scores),
+        }

{openrunner_sdk-2.4.0 → openrunner_sdk-2.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.4.0"
+version = "2.4.1"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}
@@ -30,6 +30,7 @@ Issues = "https://github.com/jqueguiner/openrunner/issues"
 openrunner = "openrunner.cli:main"
 
 [project.optional-dependencies]
+wer = ["num2words2>=0.1"]
 gpu = ["nvidia-ml-py>=12.0"]
 pytorch = ["torch>=2.0"]
 huggingface = ["transformers>=4.30"]