scorebook 0.0.13__py3-none-any.whl → 0.0.14__py3-none-any.whl

scorebook/__init__.py

@@ -9,12 +9,15 @@ import importlib.metadata
 # get version from pyproject.toml
 __version__ = importlib.metadata.version(__package__ or __name__)
 
-from scorebook.eval_datasets import EvalDataset
-from scorebook.evaluate import evaluate, evaluate_async
+from scorebook.dashboard.create_project import create_project, create_project_async
+from scorebook.dashboard.credentials import login, logout, whoami
+from scorebook.dashboard.upload_results import upload_result, upload_result_async
+from scorebook.eval_datasets.eval_dataset import EvalDataset
+from scorebook.evaluate._async.evaluate_async import evaluate_async
+from scorebook.evaluate._sync.evaluate import evaluate
 from scorebook.inference.inference_pipeline import InferencePipeline
-from scorebook.score import score, score_async
-from scorebook.trismik.credentials import login, logout, whoami
-from scorebook.trismik.upload_results import upload_result, upload_result_async
+from scorebook.score._async.score_async import score_async
+from scorebook.score._sync.score import score
 from scorebook.utils.render_template import render_template
 
 __all__ = [
@@ -28,6 +31,8 @@ __all__ = [
     "logout",
     "whoami",
     "InferencePipeline",
+    "create_project",
+    "create_project_async",
     "upload_result",
     "upload_result_async",
 ]

scorebook/cli/auth.py

@@ -4,7 +4,7 @@ import argparse
 import getpass
 import sys
 
-from scorebook.trismik.credentials import get_stored_token, get_token_path, login, logout, whoami
+from scorebook.dashboard.credentials import get_stored_token, get_token_path, login, logout, whoami
 
 
 def auth_command(args: argparse.Namespace) -> int:

scorebook/dashboard/__init__.py

@@ -0,0 +1 @@
+"""Trismik authentication and API integration."""

scorebook/dashboard/create_project.py

@@ -0,0 +1,91 @@
+"""Create projects in Trismik's experimentation platform."""
+
+import logging
+from typing import Optional
+
+from trismik.types import TrismikProject
+
+from scorebook.evaluate.evaluate_helpers import (
+    create_trismik_async_client,
+    create_trismik_sync_client,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def create_project(
+    name: str,
+    team_id: Optional[str] = None,
+    description: Optional[str] = None,
+) -> TrismikProject:
+    """Create a new project in Trismik's experimentation platform (synchronous).
+
+    This function creates a new project that can be used to organize experiments
+    and evaluation runs in the Trismik platform.
+
+    Args:
+        name: Name of the project
+        team_id: Optional ID of the team to create the project in. If not provided,
+            the project will be created in the user's default team.
+        description: Optional description of the project
+
+    Returns:
+        TrismikProject: Created project object containing project details including
+            id, name, description, accountId, createdAt, and updatedAt fields
+
+    Raises:
+        TrismikValidationError: If the request fails validation
+        TrismikApiError: If the API request fails
+    """
+    # Create Trismik client
+    trismik_client = create_trismik_sync_client()
+
+    # Create project via Trismik API
+    project = trismik_client.create_project(
+        name=name,
+        team_id=team_id,
+        description=description,
+    )
+
+    logger.info(f"Project '{name}' created successfully with ID: {project.id}")
+
+    return project
+
+
+async def create_project_async(
+    name: str,
+    team_id: Optional[str] = None,
+    description: Optional[str] = None,
+) -> TrismikProject:
+    """Create a new project in Trismik's experimentation platform (asynchronous).
+
+    This function creates a new project that can be used to organize experiments
+    and evaluation runs in the Trismik platform.
+
+    Args:
+        name: Name of the project
+        team_id: Optional ID of the team to create the project in. If not provided,
+            the project will be created in the user's default team.
+        description: Optional description of the project
+
+    Returns:
+        TrismikProject: Created project object containing project details including
+            id, name, description, accountId, createdAt, and updatedAt fields
+
+    Raises:
+        TrismikValidationError: If the request fails validation
+        TrismikApiError: If the API request fails
+    """
+    # Create Trismik async client
+    trismik_client = create_trismik_async_client()
+
+    # Create project via Trismik API (async)
+    project = await trismik_client.create_project(
+        name=name,
+        team_id=team_id,
+        description=description,
+    )
+
+    logger.info(f"Project '{name}' created successfully with ID: {project.id}")
+
+    return project

scorebook/{trismik → dashboard}/credentials.py

@@ -5,6 +5,10 @@ import os
 import pathlib
 from typing import Optional
 
+from trismik import TrismikClient
+
+from scorebook.settings import TRISMIK_SERVICE_URL
+
 logger = logging.getLogger(__name__)
 
 
@@ -66,15 +70,26 @@ def get_token() -> Optional[str]:
 
 
 def validate_token(token: str) -> bool:
-    """Validate the token by making a test API call to trismik."""
-    # TODO: Implement actual API validation once you have an endpoint
-    # This would typically make a request to something like:
-    # response = requests.get("https://api.trismik.com/whoami",
-    #                        headers={"Authorization": f"Bearer {token}"})
-    # return response.status_code == 200
-
-    # For now, just check it's not empty
-    return bool(token and token.strip())
+    """Validate the token by making a test API call to trismik.
+
+    Args:
+        token: The API token to validate.
+
+    Returns:
+        bool: True if the token is valid, False otherwise.
+    """
+    if not token or not token.strip():
+        return False
+
+    try:
+        # Create a client with the token and verify it works
+        client = TrismikClient(service_url=TRISMIK_SERVICE_URL, api_key=token)
+        client.me()
+        client.close()
+        return True
+    except Exception as e:
+        logger.debug(f"Token validation failed: {e}")
+        return False
 
 
 def login(trismik_api_key: str) -> None:

scorebook/{trismik → dashboard}/upload_results.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 
 # Known fields that are not metrics or hyperparameters
 KNOWN_AGGREGATE_FIELDS = {"dataset", "run_id", "run_completed"}
-KNOWN_ITEM_FIELDS = {"id", "dataset_name", "input", "output", "label", "run_id"}
+KNOWN_ITEM_FIELDS = {"id", "dataset", "input", "output", "label", "run_id"}
 
 
 def upload_result(

scorebook/eval_datasets/__init__.py

@@ -1,5 +1 @@
 """Dataset utilities for scorebook."""
-
-from scorebook.eval_datasets.eval_dataset import EvalDataset
-
-__all__ = ["EvalDataset"]

scorebook/eval_datasets/eval_dataset.py

@@ -18,8 +18,10 @@ from scorebook.exceptions import (
     DatasetSampleError,
     MissingFieldError,
 )
-from scorebook.metrics import MetricBase, MetricRegistry
-from scorebook.utils import render_template, validate_path
+from scorebook.metrics.metric_base import MetricBase
+from scorebook.metrics.metric_registry import MetricRegistry
+from scorebook.utils.io_helpers import validate_path
+from scorebook.utils.render_template import render_template
 
 
 class EvalDataset:

scorebook/evaluate/__init__.py

@@ -1,15 +1 @@
-"""
-Evaluation module for Scorebook.
-
-This module provides both synchronous and asynchronous evaluation functions.
-The async version serves as the source of truth, with the sync version
-automatically generated using unasync.
-"""
-
-# Import from async module
-from ._async.evaluate_async import evaluate_async
-
-# Import from generated sync module
-from ._sync.evaluate import evaluate
-
-__all__ = ["evaluate", "evaluate_async"]
+"""Evaluation module for Scorebook."""

scorebook/evaluate/_async/evaluate_async.py

@@ -6,7 +6,7 @@ from trismik import TrismikAsyncClient, TrismikClient
 from trismik.settings import evaluation_settings
 from trismik.types import TrismikRunMetadata
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.evaluate.evaluate_helpers import (
     build_eval_run_specs,
     create_trismik_async_client,
@@ -28,12 +28,9 @@ from scorebook.types import (
     EvalResult,
     EvalRunSpec,
 )
-from scorebook.utils import (
-    async_nullcontext,
-    evaluation_progress_context,
-    resolve_show_progress,
-    resolve_upload_results,
-)
+from scorebook.utils.async_utils import async_nullcontext
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import evaluation_progress_context
 
 logger = logging.getLogger(__name__)
 
@@ -392,6 +389,10 @@ async def run_adaptive_evaluation(
         available_splits=available_splits,
     )
 
+    # Create inference function with bound hyperparameters
+    async def inference_with_hyperparams(items: Any) -> Any:
+        return await inference(items, **adaptive_run_spec.hyperparameter_config)
+
     trismik_results = await trismik_client.run(
         test_id=adaptive_run_spec.dataset,
         split=resolved_split,
@@ -402,7 +403,7 @@ async def run_adaptive_evaluation(
             test_configuration={},
             inference_setup={},
         ),
-        item_processor=make_trismik_inference(inference),
+        item_processor=make_trismik_inference(inference_with_hyperparams),
         return_dict=False,
     )
 

scorebook/evaluate/_sync/evaluate.py

@@ -5,7 +5,7 @@ from trismik import TrismikAsyncClient, TrismikClient
 from trismik.settings import evaluation_settings
 from trismik.types import TrismikRunMetadata
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.evaluate.evaluate_helpers import (
     build_eval_run_specs,
     create_trismik_sync_client,
@@ -27,12 +27,9 @@ from scorebook.types import (
     EvalResult,
     EvalRunSpec,
 )
-from scorebook.utils import (
-    nullcontext,
-    evaluation_progress_context,
-    resolve_show_progress,
-    resolve_upload_results,
-)
+from contextlib import nullcontext
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import evaluation_progress_context
 
 logger = logging.getLogger(__name__)
 
@@ -391,6 +388,10 @@ def run_adaptive_evaluation(
         available_splits=available_splits,
     )
 
+    # Create inference function with bound hyperparameters
+    def inference_with_hyperparams(items: Any) -> Any:
+        return inference(items, **adaptive_run_spec.hyperparameter_config)
+
     trismik_results = trismik_client.run(
         test_id=adaptive_run_spec.dataset,
         split=resolved_split,
@@ -401,7 +402,7 @@ def run_adaptive_evaluation(
             test_configuration={},
             inference_setup={},
         ),
-        item_processor=make_trismik_inference(inference),
+        item_processor=make_trismik_inference(inference_with_hyperparams),
         return_dict=False,
     )
 

scorebook/evaluate/evaluate_helpers.py

@@ -9,7 +9,8 @@ from trismik._async.client import TrismikAsyncClient
 from trismik._sync.client import TrismikClient
 from trismik.types import TrismikMultipleChoiceTextItem
 
-from scorebook import EvalDataset
+from scorebook.dashboard.credentials import get_token
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.exceptions import (
     DataMismatchError,
     MetricComputationError,
@@ -17,9 +18,9 @@ from scorebook.exceptions import (
     ScoreBookError,
 )
 from scorebook.settings import TRISMIK_SERVICE_URL
-from scorebook.trismik.credentials import get_token
 from scorebook.types import AdaptiveEvalDataset, AdaptiveEvalRunSpec, EvalResult, EvalRunSpec
-from scorebook.utils import expand_dict, is_awaitable
+from scorebook.utils.async_utils import is_awaitable
+from scorebook.utils.transform_helpers import expand_dict
 
 logger = logging.getLogger(__name__)
 

scorebook/inference/__init__.py

@@ -1,11 +1 @@
-"""
-Inference module for model execution and predictions.
-
-This module provides functionality for running inference with various models
-and processing their responses. It includes utilities for both single and
-batch inference operations.
-"""
-
-from scorebook.inference.inference_pipeline import InferencePipeline
-
-__all__ = ["InferencePipeline"]
+"""Inference module for model execution and predictions."""

scorebook/inference/clients/__init__.py

@@ -1,8 +1 @@
-"""
-Inference clients for various LLM providers.
-
-This module provides client implementations for different LLM providers including
-OpenAI, AWS Bedrock, Google Vertex AI, and Portkey.
-"""
-
-__all__ = ["bedrock", "openai", "portkey", "vertex"]
+"""Inference clients for various LLM providers."""

scorebook/inference/inference_pipeline.py

@@ -9,7 +9,7 @@ configurable way.
 import asyncio
 from typing import Any, Callable, Dict, List, Optional, cast
 
-from scorebook.utils import is_awaitable
+from scorebook.utils.async_utils import is_awaitable
 
 
 class InferencePipeline:

scorebook/metrics/__init__.py

@@ -1,18 +1 @@
-"""
-Metrics for evaluating model predictions.
-
-This module provides a collection of evaluation metrics for comparing model outputs
-against ground truth labels. Available metrics include standard classification and
-generation metrics like accuracy, precision, recall, F1-score, etc.
-
-Metrics can be accessed by name through the `get_metrics()` function or used
-directly by instantiating specific metric classes. All metrics implement a
-common interface for scoring predictions against references.
-"""
-
-from scorebook.metrics.accuracy import Accuracy
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
-from scorebook.metrics.precision import Precision
-
-__all__ = ["MetricBase", "Precision", "Accuracy", "MetricRegistry"]
+"""Metrics for evaluating model predictions."""

scorebook/metrics/metric_registry.py

@@ -85,8 +85,10 @@ class MetricRegistry:
         # If input is a string, look up the class in the registry
         if isinstance(name_or_class, str):
             key = name_or_class.lower()
+
             if key not in cls._registry:
                 raise ValueError(f"Metric '{name_or_class}' not registered.")
+
             return cls._registry[key](**kwargs)
 
         raise ValueError(

scorebook/score/__init__.py

@@ -1,6 +1 @@
 """Score module for computing metrics on pre-computed outputs."""
-
-from scorebook.score._async.score_async import score_async
-from scorebook.score._sync.score import score
-
-__all__ = ["score", "score_async"]

scorebook/score/_async/score_async.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Union, cast
 
+from scorebook.dashboard.upload_results import upload_result_async
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.score.score_helpers import (
     calculate_metric_scores_async,
@@ -8,9 +9,9 @@ from scorebook.score.score_helpers import (
     resolve_metrics,
     validate_items,
 )
-from scorebook.trismik.upload_results import upload_result_async
 from scorebook.types import Metrics
-from scorebook.utils import resolve_show_progress, resolve_upload_results, scoring_progress_context
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import scoring_progress_context
 
 logger = logging.getLogger(__name__)
 

scorebook/score/_sync/score.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Union, cast
 
+from scorebook.dashboard.upload_results import upload_result
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.score.score_helpers import (
     calculate_metric_scores,
@@ -8,9 +9,9 @@ from scorebook.score.score_helpers import (
     resolve_metrics,
     validate_items,
 )
-from scorebook.trismik.upload_results import upload_result
 from scorebook.types import Metrics
-from scorebook.utils import resolve_show_progress, resolve_upload_results, scoring_progress_context
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import scoring_progress_context
 
 logger = logging.getLogger(__name__)
 

scorebook/score/score_helpers.py

@@ -7,7 +7,7 @@ from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.metrics.metric_base import MetricBase
 from scorebook.metrics.metric_registry import MetricRegistry
 from scorebook.types import MetricScore
-from scorebook.utils import is_awaitable
+from scorebook.utils.async_utils import is_awaitable
 
 logger = logging.getLogger(__name__)
 

scorebook/types.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.metrics.metric_base import MetricBase
 
 # Type alias for metrics parameter

scorebook/utils/__init__.py

@@ -1,23 +1 @@
 """Utility functions and common helpers for the Scorebook framework."""
-
-from contextlib import nullcontext
-
-from scorebook.utils.async_utils import async_nullcontext, is_awaitable
-from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
-from scorebook.utils.io_helpers import validate_path
-from scorebook.utils.progress_bars import evaluation_progress_context, scoring_progress_context
-from scorebook.utils.render_template import render_template
-from scorebook.utils.transform_helpers import expand_dict
-
-__all__ = [
-    "async_nullcontext",
-    "nullcontext",
-    "is_awaitable",
-    "resolve_show_progress",
-    "resolve_upload_results",
-    "validate_path",
-    "expand_dict",
-    "evaluation_progress_context",
-    "scoring_progress_context",
-    "render_template",
-]

scorebook/utils/common_helpers.py

@@ -17,7 +17,7 @@ def resolve_upload_results(upload_results: Union[Literal["auto"], bool]) -> bool
         bool: Whether to upload results to Trismik
     """
     if upload_results == "auto":
-        from scorebook.trismik.credentials import get_token
+        from scorebook.dashboard.credentials import get_token
 
         upload_results = get_token() is not None
         logger.debug("Auto upload results resolved to: %s", upload_results)

scorebook/utils/mock_llm/__init__.py

@@ -0,0 +1,41 @@
+"""Mock LLM utilities for testing and demonstrations."""
+
+import json
+import random
+from pathlib import Path
+from typing import Any, List
+
+# Load the mock data once at module initialization
+_DATA_PATH = Path(__file__).parent / "data" / "mock_llm_data.json"
+with open(_DATA_PATH, "r", encoding="utf-8") as f:
+    _MOCK_DATA = json.load(f)
+
+
+def mock_llm(inputs: List[Any], **hyperparameters: Any) -> List[str]:
+    """Mock LLM that returns answers based on pre-recorded accuracy data."""
+
+    results = []
+    all_choices = ["A", "B", "C", "D", "E"]
+
+    for item in inputs:
+        item_id = item["id"]
+
+        # Look up the item in our mock data
+        if item_id not in _MOCK_DATA:
+            # If item not found, return random answer
+            results.append(random.choice(all_choices))
+            continue
+
+        item_data = _MOCK_DATA[item_id]
+        correct_answer = item_data["answer"]
+        was_accurate = item_data["accuracy"]
+
+        if was_accurate:
+            # Return the correct answer
+            results.append(correct_answer)
+        else:
+            # Return a random incorrect answer
+            incorrect_choices = [choice for choice in all_choices if choice != correct_answer]
+            results.append(random.choice(incorrect_choices))
+
+    return results