scorebook 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

scorebook/score/_sync/score.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Union, cast
 
+from scorebook.dashboard.upload_results import upload_result
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.score.score_helpers import (
     calculate_metric_scores,
@@ -8,9 +9,9 @@ from scorebook.score.score_helpers import (
     resolve_metrics,
     validate_items,
 )
-from scorebook.trismik.upload_results import upload_result
 from scorebook.types import Metrics
-from scorebook.utils import resolve_show_progress, resolve_upload_results, scoring_progress_context
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import scoring_progress_context
 
 logger = logging.getLogger(__name__)
 

scorebook/score/score_helpers.py

@@ -7,7 +7,7 @@ from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.metrics.metric_base import MetricBase
 from scorebook.metrics.metric_registry import MetricRegistry
 from scorebook.types import MetricScore
-from scorebook.utils import is_awaitable
+from scorebook.utils.async_utils import is_awaitable
 
 logger = logging.getLogger(__name__)
 

scorebook/types.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.metrics.metric_base import MetricBase
 
 # Type alias for metrics parameter
@@ -17,6 +17,7 @@ class AdaptiveEvalDataset:
     """Represents a dataset configured for adaptive evaluation."""
 
     name: str
+    split: Optional[str] = None
 
 
 @dataclass
@@ -50,6 +51,7 @@ class AdaptiveEvalRunSpec:
     hyperparameters_index: int
     experiment_id: str
     project_id: str
+    split: Optional[str] = None
     metadata: Optional[Dict[str, Any]] = None
 
 

scorebook/utils/__init__.py

@@ -1,23 +1 @@
 """Utility functions and common helpers for the Scorebook framework."""
-
-from contextlib import nullcontext
-
-from scorebook.utils.async_utils import async_nullcontext, is_awaitable
-from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
-from scorebook.utils.io_helpers import validate_path
-from scorebook.utils.progress_bars import evaluation_progress_context, scoring_progress_context
-from scorebook.utils.render_template import render_template
-from scorebook.utils.transform_helpers import expand_dict
-
-__all__ = [
-    "async_nullcontext",
-    "nullcontext",
-    "is_awaitable",
-    "resolve_show_progress",
-    "resolve_upload_results",
-    "validate_path",
-    "expand_dict",
-    "evaluation_progress_context",
-    "scoring_progress_context",
-    "render_template",
-]

scorebook/utils/common_helpers.py

@@ -17,7 +17,7 @@ def resolve_upload_results(upload_results: Union[Literal["auto"], bool]) -> bool
         bool: Whether to upload results to Trismik
     """
     if upload_results == "auto":
-        from scorebook.trismik.credentials import get_token
+        from scorebook.dashboard.credentials import get_token
 
         upload_results = get_token() is not None
         logger.debug("Auto upload results resolved to: %s", upload_results)

scorebook/utils/mock_llm/__init__.py

@@ -0,0 +1,41 @@
+"""Mock LLM utilities for testing and demonstrations."""
+
+import json
+import random
+from pathlib import Path
+from typing import Any, List
+
+# Load the mock data once at module initialization
+_DATA_PATH = Path(__file__).parent / "data" / "mock_llm_data.json"
+with open(_DATA_PATH, "r", encoding="utf-8") as f:
+    _MOCK_DATA = json.load(f)
+
+
+def mock_llm(inputs: List[Any], **hyperparameters: Any) -> List[str]:
+    """Mock LLM that returns answers based on pre-recorded accuracy data."""
+
+    results = []
+    all_choices = ["A", "B", "C", "D", "E"]
+
+    for item in inputs:
+        item_id = item["id"]
+
+        # Look up the item in our mock data
+        if item_id not in _MOCK_DATA:
+            # If item not found, return random answer
+            results.append(random.choice(all_choices))
+            continue
+
+        item_data = _MOCK_DATA[item_id]
+        correct_answer = item_data["answer"]
+        was_accurate = item_data["accuracy"]
+
+        if was_accurate:
+            # Return the correct answer
+            results.append(correct_answer)
+        else:
+            # Return a random incorrect answer
+            incorrect_choices = [choice for choice in all_choices if choice != correct_answer]
+            results.append(random.choice(incorrect_choices))
+
+    return results