scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

scorebook/__init__.py

@@ -9,12 +9,16 @@ import importlib.metadata
 # get version from pyproject.toml
 __version__ = importlib.metadata.version(__package__ or __name__)
 
-from scorebook.eval_datasets import EvalDataset
-from scorebook.evaluate import evaluate, evaluate_async
+from scorebook.dashboard.create_project import create_project, create_project_async
+from scorebook.dashboard.credentials import login, logout, whoami
+from scorebook.dashboard.upload_results import upload_result, upload_result_async
+from scorebook.eval_datasets.eval_dataset import EvalDataset
+from scorebook.evaluate._async.evaluate_async import evaluate_async
+from scorebook.evaluate._sync.evaluate import evaluate
 from scorebook.inference.inference_pipeline import InferencePipeline
-from scorebook.score import score, score_async
-from scorebook.trismik.credentials import login, logout, whoami
-from scorebook.trismik.upload_results import upload_result, upload_result_async
+from scorebook.metrics.core.metric_registry import scorebook_metric
+from scorebook.score._async.score_async import score_async
+from scorebook.score._sync.score import score
 from scorebook.utils.render_template import render_template
 
 __all__ = [
@@ -28,6 +32,9 @@ __all__ = [
     "logout",
     "whoami",
     "InferencePipeline",
+    "create_project",
+    "create_project_async",
     "upload_result",
     "upload_result_async",
+    "scorebook_metric",
 ]

scorebook/cli/auth.py

@@ -4,7 +4,7 @@ import argparse
 import getpass
 import sys
 
-from scorebook.trismik.credentials import get_stored_token, get_token_path, login, logout, whoami
+from scorebook.dashboard.credentials import get_stored_token, get_token_path, login, logout, whoami
 
 
 def auth_command(args: argparse.Namespace) -> int:

scorebook/dashboard/__init__.py

@@ -0,0 +1 @@
+"""Trismik authentication and API integration."""

scorebook/dashboard/create_project.py

@@ -0,0 +1,91 @@
+"""Create projects in Trismik's experimentation platform."""
+
+import logging
+from typing import Optional
+
+from trismik.types import TrismikProject
+
+from scorebook.evaluate.evaluate_helpers import (
+    create_trismik_async_client,
+    create_trismik_sync_client,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def create_project(
+    name: str,
+    team_id: Optional[str] = None,
+    description: Optional[str] = None,
+) -> TrismikProject:
+    """Create a new project in Trismik's experimentation platform (synchronous).
+
+    This function creates a new project that can be used to organize experiments
+    and evaluation runs in the Trismik platform.
+
+    Args:
+        name: Name of the project
+        team_id: Optional ID of the team to create the project in. If not provided,
+            the project will be created in the user's default team.
+        description: Optional description of the project
+
+    Returns:
+        TrismikProject: Created project object containing project details including
+            id, name, description, accountId, createdAt, and updatedAt fields
+
+    Raises:
+        TrismikValidationError: If the request fails validation
+        TrismikApiError: If the API request fails
+    """
+    # Create Trismik client
+    trismik_client = create_trismik_sync_client()
+
+    # Create project via Trismik API
+    project = trismik_client.create_project(
+        name=name,
+        team_id=team_id,
+        description=description,
+    )
+
+    logger.info(f"Project '{name}' created successfully with ID: {project.id}")
+
+    return project
+
+
+async def create_project_async(
+    name: str,
+    team_id: Optional[str] = None,
+    description: Optional[str] = None,
+) -> TrismikProject:
+    """Create a new project in Trismik's experimentation platform (asynchronous).
+
+    This function creates a new project that can be used to organize experiments
+    and evaluation runs in the Trismik platform.
+
+    Args:
+        name: Name of the project
+        team_id: Optional ID of the team to create the project in. If not provided,
+            the project will be created in the user's default team.
+        description: Optional description of the project
+
+    Returns:
+        TrismikProject: Created project object containing project details including
+            id, name, description, accountId, createdAt, and updatedAt fields
+
+    Raises:
+        TrismikValidationError: If the request fails validation
+        TrismikApiError: If the API request fails
+    """
+    # Create Trismik async client
+    trismik_client = create_trismik_async_client()
+
+    # Create project via Trismik API (async)
+    project = await trismik_client.create_project(
+        name=name,
+        team_id=team_id,
+        description=description,
+    )
+
+    logger.info(f"Project '{name}' created successfully with ID: {project.id}")
+
+    return project

scorebook/{trismik → dashboard}/credentials.py

@@ -3,8 +3,14 @@
 import logging
 import os
 import pathlib
+import warnings
 from typing import Optional
 
+from dotenv import load_dotenv
+from trismik import TrismikClient
+
+from scorebook.settings import TRISMIK_SERVICE_URL
+
 logger = logging.getLogger(__name__)
 
 
@@ -66,27 +72,66 @@ def get_token() -> Optional[str]:
 
 
 def validate_token(token: str) -> bool:
-    """Validate the token by making a test API call to trismik."""
-    # TODO: Implement actual API validation once you have an endpoint
-    # This would typically make a request to something like:
-    # response = requests.get("https://api.trismik.com/whoami",
-    #                        headers={"Authorization": f"Bearer {token}"})
-    # return response.status_code == 200
+    """Validate the token by making a test API call to trismik.
+
+    Args:
+        token: The API token to validate.
+
+    Returns:
+        bool: True if the token is valid, False otherwise.
+    """
+    if not token or not token.strip():
+        return False
 
-    # For now, just check it's not empty
-    return bool(token and token.strip())
+    try:
+        # Create a client with the token and verify it works
+        client = TrismikClient(service_url=TRISMIK_SERVICE_URL, api_key=token)
+        client.me()
+        client.close()
+        return True
+    except Exception as e:
+        logger.debug(f"Token validation failed: {e}")
+        return False
 
 
-def login(trismik_api_key: str) -> None:
+def login(trismik_api_key: Optional[str] = None) -> None:
     """Login to trismik by saving API key locally.
 
+    If no API key is provided, the function will attempt to read it from the
+    TRISMIK_API_KEY environment variable or .env file (using python-dotenv).
+    Environment variables take precedence over .env file values.
+
     Args:
-        trismik_api_key: The API key to use.
+        trismik_api_key: The API key to use. If not provided, reads from
+            environment or .env file.
     Raises:
-        ValueError: If API key is empty or invalid.
+        ValueError: If API key is empty, not found, or invalid.
+
+    Warns:
+        UserWarning: If an explicit API key is passed but TRISMIK_API_KEY
+            environment variable is also set.
     """
+    # Warn if user passes explicit key but env var is also set
+    if trismik_api_key is not None and os.environ.get("TRISMIK_API_KEY"):
+        warnings.warn(
+            "TRISMIK_API_KEY environment variable is set. The environment variable "
+            "takes precedence over the stored token when calling evaluate(). "
+            "To use the explicitly provided key, unset the TRISMIK_API_KEY "
+            "environment variable.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    if trismik_api_key is None:
+        # Load from .env file if TRISMIK_API_KEY is not already set in environment
+        load_dotenv()
+        trismik_api_key = os.environ.get("TRISMIK_API_KEY")
+
     if not trismik_api_key:
-        raise ValueError("API key cannot be empty")
+        raise ValueError(
+            "API key cannot be empty. Either pass it as a parameter or "
+            "set the TRISMIK_API_KEY environment variable or .env file."
+        )
 
     # Validate token
     if not validate_token(trismik_api_key):

scorebook/{trismik → dashboard}/upload_results.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 
 # Known fields that are not metrics or hyperparameters
 KNOWN_AGGREGATE_FIELDS = {"dataset", "run_id", "run_completed"}
-KNOWN_ITEM_FIELDS = {"id", "dataset_name", "input", "output", "label", "run_id"}
+KNOWN_ITEM_FIELDS = {"id", "dataset", "input", "output", "label", "run_id"}
 
 
 def upload_result(

scorebook/eval_datasets/__init__.py

@@ -1,5 +1 @@
 """Dataset utilities for scorebook."""
-
-from scorebook.eval_datasets.eval_dataset import EvalDataset
-
-__all__ = ["EvalDataset"]

scorebook/eval_datasets/eval_dataset.py

@@ -18,8 +18,10 @@ from scorebook.exceptions import (
     DatasetSampleError,
     MissingFieldError,
 )
-from scorebook.metrics import MetricBase, MetricRegistry
-from scorebook.utils import render_template, validate_path
+from scorebook.metrics.core.metric_base import MetricBase
+from scorebook.metrics.core.metric_registry import MetricRegistry
+from scorebook.utils.io_helpers import validate_path
+from scorebook.utils.render_template import render_template
 
 
 class EvalDataset:

scorebook/evaluate/__init__.py

@@ -1,15 +1 @@
-"""
-Evaluation module for Scorebook.
-
-This module provides both synchronous and asynchronous evaluation functions.
-The async version serves as the source of truth, with the sync version
-automatically generated using unasync.
-"""
-
-# Import from async module
-from ._async.evaluate_async import evaluate_async
-
-# Import from generated sync module
-from ._sync.evaluate import evaluate
-
-__all__ = ["evaluate", "evaluate_async"]
+"""Evaluation module for Scorebook."""

scorebook/evaluate/_async/evaluate_async.py

@@ -6,7 +6,7 @@ from trismik import TrismikAsyncClient, TrismikClient
 from trismik.settings import evaluation_settings
 from trismik.types import TrismikRunMetadata
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.evaluate.evaluate_helpers import (
     build_eval_run_specs,
     create_trismik_async_client,
@@ -28,12 +28,9 @@ from scorebook.types import (
     EvalResult,
     EvalRunSpec,
 )
-from scorebook.utils import (
-    async_nullcontext,
-    evaluation_progress_context,
-    resolve_show_progress,
-    resolve_upload_results,
-)
+from scorebook.utils.async_utils import async_nullcontext
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import evaluation_progress_context
 
 logger = logging.getLogger(__name__)
 
@@ -116,8 +113,6 @@ async def evaluate_async(
         with evaluation_progress_context(
             total_eval_runs=len(eval_run_specs),
             total_items=total_items,
-            dataset_count=len(datasets),
-            hyperparam_count=len(hyperparameter_configs),
             model_display=model_display,
             enabled=show_progress_bars,
         ) as progress_bars:
@@ -154,19 +149,31 @@ async def execute_runs(
     async def worker(
         run: Union[EvalRunSpec, AdaptiveEvalRunSpec]
     ) -> Union[ClassicEvalRunResult, AdaptiveEvalRunResult]:
+        # Create progress callback for adaptive evals
+        on_progress: Optional[Callable[[int, int], None]] = None
+        if progress_bars is not None and isinstance(run, AdaptiveEvalRunSpec):
+
+            def _on_progress(current: int, total: int) -> None:
+                progress_bars.on_item_progress(current, total)
+
+            on_progress = _on_progress
+
         # Execute run (score_async handles upload internally for classic evals)
         run_result = await execute_run(
-            inference, run, upload_results, experiment_id, project_id, metadata, trismik_client
+            inference,
+            run,
+            upload_results,
+            experiment_id,
+            project_id,
+            metadata,
+            trismik_client,
+            on_progress,
         )
 
         # Update progress bars with items processed and success status
         if progress_bars is not None:
-            # Classic evals have .items; adaptive evals use max_iterations
-            items_processed = (
-                len(run.dataset.items)
-                if isinstance(run, EvalRunSpec)
-                else evaluation_settings["max_iterations"]
-            )
+            # Classic evals: update items count; Adaptive evals: items already tracked via callback
+            items_processed = len(run.dataset.items) if isinstance(run, EvalRunSpec) else 0
             progress_bars.on_run_completed(items_processed, run_result.run_completed)
 
         # Update upload progress for classic evals
@@ -198,11 +205,12 @@ async def execute_runs(
 async def execute_run(
     inference: Callable,
     run: Union[EvalRunSpec, AdaptiveEvalRunSpec],
-    upload_results: bool,  # NEW PARAMETER
+    upload_results: bool,
     experiment_id: Optional[str] = None,
     project_id: Optional[str] = None,
     metadata: Optional[Dict[str, Any]] = None,
     trismik_client: Optional[Union[TrismikClient, TrismikAsyncClient]] = None,
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> Union[ClassicEvalRunResult, AdaptiveEvalRunResult]:
     """Execute a single evaluation run."""
 
@@ -221,6 +229,7 @@ async def execute_run(
             resolved_project_id,
             metadata,
             trismik_client,
+            on_progress,
         )
 
     else:
@@ -341,6 +350,7 @@ async def execute_adaptive_eval_run(
     project_id: str,
     metadata: Optional[Dict[str, Any]] = None,
     trismik_client: Optional[Union[TrismikClient, TrismikAsyncClient]] = None,
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> AdaptiveEvalRunResult:
     """Execute an adaptive evaluation run."""
     logger.debug("Executing adaptive run for %s", run)
@@ -350,7 +360,7 @@ async def execute_adaptive_eval_run(
             raise ScoreBookError("Trismik client is required for adaptive evaluation")
 
         adaptive_eval_run_result = await run_adaptive_evaluation(
-            inference, run, experiment_id, project_id, metadata, trismik_client
+            inference, run, experiment_id, project_id, metadata, trismik_client, on_progress
         )
         logger.debug("Adaptive evaluation completed for run %s", adaptive_eval_run_result)
 
@@ -368,6 +378,7 @@ async def run_adaptive_evaluation(
     project_id: str,
     metadata: Any,
     trismik_client: Union[TrismikClient, TrismikAsyncClient],
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> AdaptiveEvalRunResult:
     """Run an adaptive evaluation using the Trismik API.
 
@@ -378,6 +389,7 @@ async def run_adaptive_evaluation(
         project_id: Trismik project ID
         metadata: Additional metadata
         trismik_client: Trismik client instance
+        on_progress: Optional callback for progress updates (current, total)
     Returns:
         Results from the adaptive evaluation
     """
@@ -392,6 +404,10 @@ async def run_adaptive_evaluation(
         available_splits=available_splits,
     )
 
+    # Create inference function with bound hyperparameters
+    async def inference_with_hyperparams(items: Any) -> Any:
+        return await inference(items, **adaptive_run_spec.hyperparameter_config)
+
     trismik_results = await trismik_client.run(
         test_id=adaptive_run_spec.dataset,
         split=resolved_split,
@@ -402,7 +418,8 @@ async def run_adaptive_evaluation(
             test_configuration={},
             inference_setup={},
         ),
-        item_processor=make_trismik_inference(inference),
+        item_processor=make_trismik_inference(inference_with_hyperparams),
+        on_progress=on_progress,
         return_dict=False,
     )
 

scorebook/evaluate/_sync/evaluate.py

@@ -5,7 +5,7 @@ from trismik import TrismikAsyncClient, TrismikClient
 from trismik.settings import evaluation_settings
 from trismik.types import TrismikRunMetadata
 
-from scorebook.eval_datasets import EvalDataset
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.evaluate.evaluate_helpers import (
     build_eval_run_specs,
     create_trismik_sync_client,
@@ -27,12 +27,9 @@ from scorebook.types import (
     EvalResult,
     EvalRunSpec,
 )
-from scorebook.utils import (
-    nullcontext,
-    evaluation_progress_context,
-    resolve_show_progress,
-    resolve_upload_results,
-)
+from contextlib import nullcontext
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import evaluation_progress_context
 
 logger = logging.getLogger(__name__)
 
@@ -115,8 +112,6 @@ def evaluate(
         with evaluation_progress_context(
             total_eval_runs=len(eval_run_specs),
             total_items=total_items,
-            dataset_count=len(datasets),
-            hyperparam_count=len(hyperparameter_configs),
             model_display=model_display,
             enabled=show_progress_bars,
         ) as progress_bars:
@@ -153,19 +148,31 @@ def execute_runs(
     def worker(
         run: Union[EvalRunSpec, AdaptiveEvalRunSpec]
     ) -> Union[ClassicEvalRunResult, AdaptiveEvalRunResult]:
+        # Create progress callback for adaptive evals
+        on_progress: Optional[Callable[[int, int], None]] = None
+        if progress_bars is not None and isinstance(run, AdaptiveEvalRunSpec):
+
+            def _on_progress(current: int, total: int) -> None:
+                progress_bars.on_item_progress(current, total)
+
+            on_progress = _on_progress
+
         # Execute run (score_async handles upload internally for classic evals)
         run_result = execute_run(
-            inference, run, upload_results, experiment_id, project_id, metadata, trismik_client
+            inference,
+            run,
+            upload_results,
+            experiment_id,
+            project_id,
+            metadata,
+            trismik_client,
+            on_progress,
         )
 
         # Update progress bars with items processed and success status
         if progress_bars is not None:
-            # Classic evals have .items; adaptive evals use max_iterations
-            items_processed = (
-                len(run.dataset.items)
-                if isinstance(run, EvalRunSpec)
-                else evaluation_settings["max_iterations"]
-            )
+            # Classic evals: update items count; Adaptive evals: items already tracked via callback
+            items_processed = len(run.dataset.items) if isinstance(run, EvalRunSpec) else 0
             progress_bars.on_run_completed(items_processed, run_result.run_completed)
 
         # Update upload progress for classic evals
@@ -197,11 +204,12 @@ def execute_runs(
 def execute_run(
     inference: Callable,
     run: Union[EvalRunSpec, AdaptiveEvalRunSpec],
-    upload_results: bool,  # NEW PARAMETER
+    upload_results: bool,
     experiment_id: Optional[str] = None,
     project_id: Optional[str] = None,
     metadata: Optional[Dict[str, Any]] = None,
     trismik_client: Optional[Union[TrismikClient, TrismikAsyncClient]] = None,
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> Union[ClassicEvalRunResult, AdaptiveEvalRunResult]:
     """Execute a single evaluation run."""
 
@@ -220,6 +228,7 @@ def execute_run(
             resolved_project_id,
             metadata,
             trismik_client,
+            on_progress,
         )
 
     else:
@@ -340,6 +349,7 @@ def execute_adaptive_eval_run(
     project_id: str,
     metadata: Optional[Dict[str, Any]] = None,
     trismik_client: Optional[Union[TrismikClient, TrismikAsyncClient]] = None,
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> AdaptiveEvalRunResult:
     """Execute an adaptive evaluation run."""
     logger.debug("Executing adaptive run for %s", run)
@@ -349,7 +359,7 @@ def execute_adaptive_eval_run(
             raise ScoreBookError("Trismik client is required for adaptive evaluation")
 
         adaptive_eval_run_result = run_adaptive_evaluation(
-            inference, run, experiment_id, project_id, metadata, trismik_client
+            inference, run, experiment_id, project_id, metadata, trismik_client, on_progress
         )
         logger.debug("Adaptive evaluation completed for run %s", adaptive_eval_run_result)
 
@@ -367,6 +377,7 @@ def run_adaptive_evaluation(
     project_id: str,
     metadata: Any,
     trismik_client: Union[TrismikClient, TrismikAsyncClient],
+    on_progress: Optional[Callable[[int, int], None]] = None,
 ) -> AdaptiveEvalRunResult:
     """Run an adaptive evaluation using the Trismik API.
 
@@ -377,6 +388,7 @@ def run_adaptive_evaluation(
         project_id: Trismik project ID
         metadata: Additional metadata
         trismik_client: Trismik client instance
+        on_progress: Optional callback for progress updates (current, total)
     Returns:
         Results from the adaptive evaluation
     """
@@ -391,6 +403,10 @@ def run_adaptive_evaluation(
         available_splits=available_splits,
     )
 
+    # Create inference function with bound hyperparameters
+    def inference_with_hyperparams(items: Any) -> Any:
+        return inference(items, **adaptive_run_spec.hyperparameter_config)
+
     trismik_results = trismik_client.run(
         test_id=adaptive_run_spec.dataset,
         split=resolved_split,
@@ -401,7 +417,8 @@ def run_adaptive_evaluation(
             test_configuration={},
             inference_setup={},
         ),
-        item_processor=make_trismik_inference(inference),
+        item_processor=make_trismik_inference(inference_with_hyperparams),
+        on_progress=on_progress,
         return_dict=False,
     )
 

scorebook/evaluate/evaluate_helpers.py

@@ -9,7 +9,8 @@ from trismik._async.client import TrismikAsyncClient
 from trismik._sync.client import TrismikClient
 from trismik.types import TrismikMultipleChoiceTextItem
 
-from scorebook import EvalDataset
+from scorebook.dashboard.credentials import get_token
+from scorebook.eval_datasets.eval_dataset import EvalDataset
 from scorebook.exceptions import (
     DataMismatchError,
     MetricComputationError,
@@ -17,9 +18,9 @@ from scorebook.exceptions import (
     ScoreBookError,
 )
 from scorebook.settings import TRISMIK_SERVICE_URL
-from scorebook.trismik.credentials import get_token
 from scorebook.types import AdaptiveEvalDataset, AdaptiveEvalRunSpec, EvalResult, EvalRunSpec
-from scorebook.utils import expand_dict, is_awaitable
+from scorebook.utils.async_utils import is_awaitable
+from scorebook.utils.transform_helpers import expand_dict
 
 logger = logging.getLogger(__name__)
 

scorebook/inference/__init__.py

@@ -1,11 +1 @@
-"""
-Inference module for model execution and predictions.
-
-This module provides functionality for running inference with various models
-and processing their responses. It includes utilities for both single and
-batch inference operations.
-"""
-
-from scorebook.inference.inference_pipeline import InferencePipeline
-
-__all__ = ["InferencePipeline"]
+"""Inference module for model execution and predictions."""

scorebook/inference/clients/__init__.py

@@ -1,8 +1 @@
-"""
-Inference clients for various LLM providers.
-
-This module provides client implementations for different LLM providers including
-OpenAI, AWS Bedrock, Google Vertex AI, and Portkey.
-"""
-
-__all__ = ["bedrock", "openai", "portkey", "vertex"]
+"""Inference clients for various LLM providers."""

scorebook/inference/inference_pipeline.py

@@ -9,7 +9,7 @@ configurable way.
 import asyncio
 from typing import Any, Callable, Dict, List, Optional, cast
 
-from scorebook.utils import is_awaitable
+from scorebook.utils.async_utils import is_awaitable
 
 
 class InferencePipeline: