scorebook 0.0.9__py3-none-any.whl → 0.0.10__py3-none-any.whl

scorebook/trismik_services/adaptive_testing_service.py

@@ -1,141 +0,0 @@
-"""Trismik adaptive testing service integration."""
-
-import asyncio
-import dataclasses
-import inspect
-import logging
-from typing import Any, Callable, Iterable, Mapping
-
-from trismik.adaptive_test import AdaptiveTest
-from trismik.client_async import TrismikAsyncClient
-from trismik.types import TrismikMultipleChoiceTextItem, TrismikRunMetadata
-
-from scorebook.types import AdaptiveEvalRunResult, AdaptiveEvalRunSpec
-
-from .login import get_token
-
-logger = logging.getLogger(__name__)
-
-
-async def run_adaptive_evaluation(
-    inference: Callable,
-    adaptive_run_spec: AdaptiveEvalRunSpec,
-    experiment_id: str,
-    project_id: str,
-    metadata: Any,
-) -> AdaptiveEvalRunResult:
-    """Run an adaptive evaluation using the Trismik API.
-
-    Args:
-        inference: Function to run inference
-        adaptive_run_spec: Specification for the adaptive evaluation
-        experiment_id: Experiment identifier
-        project_id: Trismik project ID
-        metadata: Additional metadata
-    Returns:
-        Results from the adaptive evaluation
-    """
-    runner = AdaptiveTest(
-        make_trismik_inference(inference),
-        client=TrismikAsyncClient(
-            service_url="https://api.trismik.com/adaptive-testing", api_key=get_token()
-        ),
-    )
-
-    logger.debug(
-        "test_id: %s, project_id: %s, experiment: %s ",
-        adaptive_run_spec.dataset,
-        project_id,
-        experiment_id,
-    )
-    trismik_results = runner.run(
-        adaptive_run_spec.dataset,
-        project_id,
-        experiment_id,
-        run_metadata=TrismikRunMetadata(
-            model_metadata=TrismikRunMetadata.ModelMetadata(name="unknown"),
-            test_configuration={},
-            inference_setup={},
-        ),
-        return_dict=False,
-    )
-
-    # Convert TrismikRunResults to AdaptiveEvalRunResult
-    # Extract scores from the Trismik results
-    scores = {}
-    if hasattr(trismik_results, "scores") and trismik_results.scores:
-        scores = trismik_results.scores
-    elif hasattr(trismik_results, "__dict__"):
-        # If scores aren't directly available, include all attributes as scores
-        scores = {k: v for k, v in trismik_results.__dict__.items() if not k.startswith("_")}
-
-    # Convert AdaptiveTestScore objects to JSON-serializable dictionaries
-    def make_json_serializable(obj: Any) -> Any:
-        if hasattr(obj, "theta") and hasattr(obj, "std_error"):
-            # This is likely an AdaptiveTestScore object
-            return {"theta": obj.theta, "std_error": obj.std_error}
-        elif isinstance(obj, dict):
-            return {k: make_json_serializable(v) for k, v in obj.items()}
-        elif isinstance(obj, (list, tuple)):
-            return [make_json_serializable(item) for item in obj]
-        else:
-            return obj
-
-    # Make scores JSON serializable
-    scores = make_json_serializable(scores)
-
-    return AdaptiveEvalRunResult(run_spec=adaptive_run_spec, scores=scores)
-
-
-def make_trismik_inference(
-    inference_function: Callable,
-    return_list: bool = False,
-) -> Callable[[Any], Any]:
-    """Wrap an inference function for flexible input handling.
-
-    Takes a function expecting list[dict] and makes it accept single dict
-    or TrismikMultipleChoiceTextItem.
-    """
-
-    # Check if the inference function is async
-    is_async = inspect.iscoroutinefunction(inference_function) or (
-        hasattr(inference_function, "__call__")
-        and inspect.iscoroutinefunction(inference_function.__call__)
-    )
-
-    def sync_trismik_inference_function(eval_items: Any, **kwargs: Any) -> Any:
-        # Single TrismikMultipleChoiceTextItem dataclass
-        if isinstance(eval_items, TrismikMultipleChoiceTextItem):
-            eval_item_dict = dataclasses.asdict(eval_items)
-            results = inference_function([eval_item_dict], **kwargs)
-            if is_async:
-                results = asyncio.run(results)
-            return results if return_list else results[0]
-
-        # Single item (a mapping)
-        if isinstance(eval_items, Mapping):
-            results = inference_function([eval_items], **kwargs)
-            if is_async:
-                results = asyncio.run(results)
-            return results if return_list else results[0]
-
-        # Iterable of items (but not a string/bytes)
-        if isinstance(eval_items, Iterable) and not isinstance(eval_items, (str, bytes)):
-            # Convert any TrismikMultipleChoiceTextItem instances to dicts
-            converted_items = []
-            for item in eval_items:
-                if isinstance(item, TrismikMultipleChoiceTextItem):
-                    converted_items.append(dataclasses.asdict(item))
-                else:
-                    converted_items.append(item)
-            results = inference_function(converted_items, **kwargs)
-            if is_async:
-                results = asyncio.run(results)
-            return results
-
-        raise TypeError(
-            "Expected a single item (Mapping[str, Any] or TrismikMultipleChoiceTextItem) "
-            "or an iterable of such items."
-        )
-
-    return sync_trismik_inference_function

scorebook/trismik_services/upload_classic_eval_run.py

@@ -1,102 +0,0 @@
-"""Upload classic evaluation run results to Trismik platform."""
-
-import logging
-from typing import Any, Dict, List, Optional
-
-from trismik.adaptive_test import AdaptiveTest
-from trismik.client_async import TrismikAsyncClient
-from trismik.types import (
-    TrismikClassicEvalItem,
-    TrismikClassicEvalMetric,
-    TrismikClassicEvalRequest,
-    TrismikClassicEvalResponse,
-)
-
-from scorebook.trismik_services.login import get_token
-from scorebook.types import ClassicEvalRunResult
-
-logger = logging.getLogger(__name__)
-
-
-async def upload_classic_eval_run(
-    run: ClassicEvalRunResult,
-    experiment_id: str,
-    project_id: str,
-    model: str,
-    metadata: Optional[Dict[str, Any]],
-) -> TrismikClassicEvalResponse:
-    """Upload a classic evaluation run result to Trismik platform.
-
-    Args:
-        run: The evaluation run result to upload
-        experiment_id: Trismik experiment identifier
-        project_id: Trismik project identifier
-        model: Model name used for evaluation
-        metadata: Optional metadata dictionary
-
-    Returns:
-        Response from Trismik API containing the upload result
-    """
-    runner = AdaptiveTest(
-        lambda x: None,
-        client=TrismikAsyncClient(
-            service_url="https://api.trismik.com/adaptive-testing", api_key=get_token()
-        ),
-    )
-
-    # Create eval items from run_spec items, outputs, and labels
-    items: List[TrismikClassicEvalItem] = []
-    for idx, (item, output) in enumerate(zip(run.run_spec.items, run.outputs)):
-        label = run.run_spec.labels[idx] if idx < len(run.run_spec.labels) else ""
-
-        # Calculate item-level metrics for this item
-        item_metrics: Dict[str, Any] = {}
-        for metric_name, metric_data in run.scores.items():
-            if isinstance(metric_data, dict) and "item_scores" in metric_data:
-                if idx < len(metric_data["item_scores"]):
-                    item_metrics[metric_name] = metric_data["item_scores"][idx]
-            else:
-                # If scores is just a single value, use it for all items
-                item_metrics[metric_name] = metric_data
-
-        eval_item = TrismikClassicEvalItem(
-            datasetItemId=str(idx),
-            modelInput=str(item),
-            modelOutput=str(output),
-            goldOutput=str(label),
-            metrics=item_metrics,
-        )
-        items.append(eval_item)
-
-    # Create eval metrics from run aggregate scores
-    metrics: List[TrismikClassicEvalMetric] = []
-    for metric_name, metric_data in run.scores.items():
-        if isinstance(metric_data, dict) and "aggregate_scores" in metric_data:
-            # Handle structured metric data with aggregate scores
-            for agg_name, agg_value in metric_data["aggregate_scores"].items():
-                metric_id = f"{metric_name}_{agg_name}" if agg_name != metric_name else metric_name
-                metric = TrismikClassicEvalMetric(metricId=metric_id, value=agg_value)
-                metrics.append(metric)
-        else:
-            # Handle simple metric data (single value)
-            metric = TrismikClassicEvalMetric(metricId=metric_name, value=metric_data)
-            metrics.append(metric)
-
-    classic_eval_request = TrismikClassicEvalRequest(
-        project_id,
-        experiment_id,
-        run.run_spec.dataset.name,
-        model,
-        run.run_spec.hyperparameter_config,
-        items,
-        metrics,
-    )
-
-    response: TrismikClassicEvalResponse = await runner.submit_classic_eval_async(
-        classic_eval_request
-    )
-
-    run_id: str = response.id
-    logger.info(f"Classic eval run uploaded successfully with run_id: {run_id}")
-
-    return response

scorebook-0.0.9.dist-info/RECORD

@@ -1,36 +0,0 @@
-scorebook/__init__.py,sha256=30kyXG8sVbThtCt6cjPtkx7hiaUEukRQC-RsgunlkL4,557
-scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
-scorebook/cli/auth.py,sha256=bv3imsgmY_t52wFoMJt9iu-cKPwvKYkVqZ7nE8EVc6E,2931
-scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
-scorebook/eval_dataset.py,sha256=LSTyxUkT06iEAVYCnjIDFxFgZzRejwiS5CZA-jvy1ns,15098
-scorebook/evaluate.py,sha256=OOBTZmx84ZAuZKbIe1pp9L3201cX2gjPAkw_llYNnfE,21899
-scorebook/exceptions.py,sha256=emq2QY-4mW6VXlq1dxunPjt-xZpLQIxo8Ck_gYxz1VE,1827
-scorebook/inference/__init__.py,sha256=tqSXSyVurc_YRfPypYed8iTH7Fwt7iFCXMxBXnqY-9I,242
-scorebook/inference/bedrock.py,sha256=wllq0ysNFQKWJDEqoN-k96Jx43BHCAvfxm14zMRCf90,10074
-scorebook/inference/openai.py,sha256=iJVWp0HT9skyM4KXf21yaEjLafdETT5kK3HKl7MZ1hg,8292
-scorebook/inference/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
-scorebook/inference/vertex.py,sha256=jv_Nbt1NJQ6mMUyEuW_idxhj_3fugBojshtpGP9fMeY,9874
-scorebook/inference_pipeline.py,sha256=-HcGGbwM34fGJ_FlXcyqj_pV6DjWIXRGgICiN_63UsU,3251
-scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
-scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
-scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
-scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
-scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
-scorebook/trismik_services/__init__.py,sha256=CiGl1u4GcfYhWmB_fGOlsJPwYeKXtIr-uCXoOv4O8yg,284
-scorebook/trismik_services/adaptive_testing_service.py,sha256=4FVW8g7EvJmHYpQp68y0U3xzOw_qJ9nkhEPiMnzTb4s,5103
-scorebook/trismik_services/login.py,sha256=WtJLaNmBMwCi6gT1Bgp4J9x2tq5HDrDI9U074r08TnU,3275
-scorebook/trismik_services/upload_classic_eval_run.py,sha256=t6tvaKfP2lFBKmZ_bM6oe_mUOjA_UKu35GbgL26QW3A,3658
-scorebook/types.py,sha256=zt8sGfbRjXatx1WtttWZDVIoiS-yhh_1lP0K4VHYvAM,5797
-scorebook/utils/__init__.py,sha256=l_bfi9lAMz1oyGnuyKuzYasQKt2DJwffqsbfSl4-GIQ,452
-scorebook/utils/async_utils.py,sha256=OeNvMrOT9P4rIyaCf5IbR3ZIFMtEzXgoAArNbINRtMU,728
-scorebook/utils/build_prompt.py,sha256=L_Y84a1ewm3GvwnSSuUXfPO_M0QL1Dl8UgOS_l_zvh4,1617
-scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
-scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
-scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
-scorebook/utils/progress_bars.py,sha256=TBz41w3yFujsO9n8vUjeubgOrmdiAMI2P2SSVqTJzAA,5269
-scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
-scorebook-0.0.9.dist-info/METADATA,sha256=-VyogmnwuCfPBaj6BS_gOGKxgAkxcnAx6k2GFaI1jGg,11516
-scorebook-0.0.9.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-scorebook-0.0.9.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
-scorebook-0.0.9.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
-scorebook-0.0.9.dist-info/RECORD,,