osmosis-ai 0.1.9__py3-none-any.whl → 0.2.4__py3-none-any.whl

osmosis_ai/utils.py

@@ -1,7 +1,8 @@
 
 import functools
 import inspect
-from typing import Callable
+import types
+from typing import Any, Callable, Mapping, Union, get_args, get_origin, get_type_hints
 
 
 def osmosis_reward(func: Callable) -> Callable:
@@ -27,9 +28,8 @@ def osmosis_reward(func: Callable) -> Callable:
     sig = inspect.signature(func)
     params = list(sig.parameters.values())
 
-    # Check parameter count
-    if len(params) < 2 or len(params) > 3:
-        raise TypeError(f"Function {func.__name__} must have 2-3 parameters, got {len(params)}")
+    if len(params) < 3:
+        raise TypeError(f"Function {func.__name__} must have at least 3 parameters, got {len(params)}")
 
     # Check first parameter: solution_str: str
     if params[0].name != 'solution_str':
@@ -44,7 +44,7 @@ def osmosis_reward(func: Callable) -> Callable:
         raise TypeError(f"Second parameter 'ground_truth' must be annotated as str, got {params[1].annotation}")
 
     # Check third parameter if present: extra_info: dict = None
-    if len(params) == 3:
+    if len(params) >= 3:
         if params[2].name != 'extra_info':
             raise TypeError(f"Third parameter must be named 'extra_info', got '{params[2].name}'")
         if params[2].annotation != dict:
@@ -54,6 +54,259 @@ def osmosis_reward(func: Callable) -> Callable:
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
+        kwargs.pop("data_source", None)
+        result = func(*args, **kwargs)
+        if not isinstance(result, float):
+            raise TypeError(f"Function {func.__name__} must return a float, got {type(result).__name__}")
+        return result
+
+    return wrapper
+
+
+ALLOWED_ROLES = {"user", "system", "assistant", "developer", "tool", "function"}
+
+_UNION_TYPES = {Union}
+_types_union_type = getattr(types, "UnionType", None)
+if _types_union_type is not None:
+    _UNION_TYPES.add(_types_union_type)
+
+
+def _is_str_annotation(annotation: Any) -> bool:
+    if annotation is inspect.Parameter.empty:
+        return False
+    if annotation is str:
+        return True
+    if isinstance(annotation, str):
+        return annotation in {"str", "builtins.str"}
+    if isinstance(annotation, type):
+        try:
+            return issubclass(annotation, str)
+        except TypeError:
+            return False
+    forward_arg = getattr(annotation, "__forward_arg__", None)
+    if isinstance(forward_arg, str):
+        return forward_arg in {"str", "builtins.str"}
+    return False
+
+
+def _is_optional_str(annotation: Any) -> bool:
+    if _is_str_annotation(annotation):
+        return True
+    if isinstance(annotation, str):
+        normalized = annotation.replace(" ", "")
+        if normalized in {
+            "Optional[str]",
+            "typing.Optional[str]",
+            "Str|None",
+            "str|None",
+            "builtins.str|None",
+            "None|str",
+            "None|builtins.str",
+        }:
+            return True
+    origin = get_origin(annotation)
+    if origin in _UNION_TYPES:
+        args = tuple(arg for arg in get_args(annotation) if arg is not type(None))  # noqa: E721
+        return len(args) == 1 and _is_str_annotation(args[0])
+    return False
+
+
+def _is_list_annotation(annotation: Any) -> bool:
+    if annotation is list:
+        return True
+    if isinstance(annotation, str):
+        normalized = annotation.replace(" ", "")
+        return (
+            normalized in {"list", "builtins.list", "typing.List", "List"}
+            or normalized.startswith("list[")
+            or normalized.startswith("builtins.list[")
+            or normalized.startswith("typing.List[")
+            or normalized.startswith("List[")
+        )
+    origin = get_origin(annotation)
+    return origin is list
+
+
+def _is_float_annotation(annotation: Any) -> bool:
+    if annotation in {inspect.Parameter.empty, float}:
+        return True
+    if isinstance(annotation, str):
+        return annotation in {"float", "builtins.float"}
+    origin = get_origin(annotation)
+    return origin is float
+
+
+def _is_numeric(value: Any) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def _is_dict_annotation(annotation: Any) -> bool:
+    if annotation in {dict, Mapping}:
+        return True
+    origin = get_origin(annotation)
+    if origin in {dict, Mapping}:
+        return True
+    if isinstance(annotation, type):
+        try:
+            return issubclass(annotation, dict)
+        except TypeError:
+            return False
+    if isinstance(annotation, str):
+        normalized = annotation.replace(" ", "")
+        return (
+            normalized in {"dict", "builtins.dict", "typing.Mapping", "collections.abc.Mapping", "Mapping"}
+            or normalized.startswith("dict[")
+            or normalized.startswith("builtins.dict[")
+            or normalized.startswith("typing.Dict[")
+            or normalized.startswith("Dict[")
+            or normalized.startswith("typing.Mapping[")
+            or normalized.startswith("Mapping[")
+        )
+    return False
+
+
+def osmosis_rubric(func: Callable) -> Callable:
+    """
+    Decorator for rubric functions that enforces the signature:
+    (solution_str: str, ground_truth: str, extra_info: dict) -> float
+
+    The `extra_info` mapping must include the current `provider`, `model`, and `rubric`
+    values, and may optionally provide `system_prompt`, `score_min`, and `score_max`.
+
+    Args:
+        func: The rubric function to be wrapped.
+
+    Returns:
+        The wrapped function.
+
+    Raises:
+        TypeError: If the function doesn't have the required signature or doesn't return a float.
+
+    Example:
+        @osmosis_rubric
+        def evaluate_response(
+            solution_str: str,
+            ground_truth: str,
+            extra_info: dict,
+        ) -> float:
+            return some_evaluation(solution_str, ground_truth, extra_info)
+    """
+    sig = inspect.signature(func)
+    params = list(sig.parameters.values())
+    try:
+        resolved_annotations = get_type_hints(
+            func,
+            globalns=getattr(func, "__globals__", {}),
+            include_extras=True,
+        )
+    except Exception:  # pragma: no cover - best effort for forward refs
+        resolved_annotations = {}
+
+    if len(params) < 3:
+        raise TypeError(f"Function {func.__name__} must have at least 3 parameters, got {len(params)}")
+
+    solution_param = params[0]
+    if solution_param.name != "solution_str":
+        raise TypeError(f"First parameter must be named 'solution_str', got '{solution_param.name}'")
+    solution_annotation = resolved_annotations.get(solution_param.name, solution_param.annotation)
+    if not _is_str_annotation(solution_annotation):
+        raise TypeError(f"First parameter 'solution_str' must be annotated as str, got {solution_annotation}")
+    if solution_param.default is not inspect.Parameter.empty:
+        raise TypeError("First parameter 'solution_str' cannot have a default value")
+
+    ground_truth_param = params[1]
+    if ground_truth_param.name != "ground_truth":
+        raise TypeError(f"Second parameter must be named 'ground_truth', got '{ground_truth_param.name}'")
+    ground_truth_annotation = resolved_annotations.get(ground_truth_param.name, ground_truth_param.annotation)
+    if not _is_optional_str(ground_truth_annotation):
+        raise TypeError(
+            f"Second parameter 'ground_truth' must be annotated as str or Optional[str], got {ground_truth_annotation}"
+        )
+    if ground_truth_param.default is not inspect.Parameter.empty:
+        raise TypeError("Second parameter 'ground_truth' cannot have a default value")
+
+    extra_info_param = params[2]
+    if extra_info_param.name != "extra_info":
+        raise TypeError(f"Third parameter must be named 'extra_info', got '{extra_info_param.name}'")
+    extra_info_annotation = resolved_annotations.get(extra_info_param.name, extra_info_param.annotation)
+    if not _is_dict_annotation(extra_info_annotation):
+        raise TypeError(
+            f"Third parameter 'extra_info' must be annotated as a dict or mapping, got {extra_info_annotation}"
+        )
+    if extra_info_param.default is not inspect.Parameter.empty:
+        raise TypeError("Third parameter 'extra_info' cannot have a default value")
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        kwargs.pop("data_source", None)
+        bound = sig.bind_partial(*args, **kwargs)
+        bound.apply_defaults()
+
+        if "solution_str" not in bound.arguments:
+            raise TypeError("'solution_str' argument is required")
+        solution_value = bound.arguments["solution_str"]
+        if not isinstance(solution_value, str):
+            raise TypeError(f"'solution_str' must be a string, got {type(solution_value).__name__}")
+
+        if "ground_truth" not in bound.arguments:
+            raise TypeError("'ground_truth' argument is required")
+        ground_truth_value = bound.arguments["ground_truth"]
+        if ground_truth_value is not None and not isinstance(ground_truth_value, str):
+            raise TypeError(
+                f"'ground_truth' must be a string or None, got {type(ground_truth_value).__name__}"
+            )
+
+        if "extra_info" not in bound.arguments:
+            raise TypeError("'extra_info' argument is required")
+        extra_info_value = bound.arguments["extra_info"]
+        if not isinstance(extra_info_value, Mapping):
+            raise TypeError(f"'extra_info' must be a mapping, got {type(extra_info_value).__name__}")
+
+        provider_value = extra_info_value.get("provider")
+        if not isinstance(provider_value, str) or not provider_value.strip():
+            raise TypeError("'extra_info[\"provider\"]' must be a non-empty string")
+
+        model_value = extra_info_value.get("model")
+        if not isinstance(model_value, str) or not model_value.strip():
+            raise TypeError("'extra_info[\"model\"]' must be a non-empty string")
+
+        if "rubric" not in extra_info_value:
+            raise TypeError("'extra_info' must include a 'rubric' string")
+        rubric_value = extra_info_value["rubric"]
+        if not isinstance(rubric_value, str):
+            raise TypeError(f"'extra_info[\"rubric\"]' must be a string, got {type(rubric_value).__name__}")
+
+        api_key_value = extra_info_value.get("api_key")
+        api_key_env_value = extra_info_value.get("api_key_env")
+        has_api_key = isinstance(api_key_value, str) and bool(api_key_value.strip())
+        has_api_key_env = isinstance(api_key_env_value, str) and bool(api_key_env_value.strip())
+        if not (has_api_key or has_api_key_env):
+            raise TypeError(
+                "'extra_info' must include either a non-empty 'api_key' or 'api_key_env' string"
+            )
+
+        system_prompt_value = extra_info_value.get("system_prompt")
+        if system_prompt_value is not None and not isinstance(system_prompt_value, str):
+            raise TypeError(
+                f"'extra_info[\"system_prompt\"]' must be a string or None, got {type(system_prompt_value).__name__}"
+            )
+
+        score_min_value = extra_info_value.get("score_min")
+        if score_min_value is not None and not _is_numeric(score_min_value):
+            raise TypeError(
+                f"'extra_info[\"score_min\"]' must be numeric, got {type(score_min_value).__name__}"
+            )
+
+        score_max_value = extra_info_value.get("score_max")
+        if score_max_value is not None and not _is_numeric(score_max_value):
+            raise TypeError(
+                f"'extra_info[\"score_max\"]' must be numeric, got {type(score_max_value).__name__}"
+            )
+
+        if score_min_value is not None and score_max_value is not None:
+            if float(score_max_value) <= float(score_min_value):
+                raise ValueError("'extra_info[\"score_max\"]' must be greater than 'extra_info[\"score_min\"]'")
+
         result = func(*args, **kwargs)
         if not isinstance(result, float):
             raise TypeError(f"Function {func.__name__} must return a float, got {type(result).__name__}")

osmosis_ai-0.2.4.dist-info/METADATA

@@ -0,0 +1,314 @@
+Metadata-Version: 2.4
+Name: osmosis-ai
+Version: 0.2.4
+Summary: A Python library for reward function validation with strict type enforcement.
+Author-email: Osmosis AI <jake@osmosis.ai>
+License: MIT License
+        
+        Copyright (c) 2025 Gulp AI
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE. 
+Project-URL: Homepage, https://github.com/Osmosis-AI/osmosis-sdk-python
+Project-URL: Issues, https://github.com/Osmosis-AI/osmosis-sdk-python/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyYAML<7.0,>=6.0
+Requires-Dist: python-dotenv<2.0.0,>=0.1.0
+Requires-Dist: requests<3.0.0,>=2.0.0
+Requires-Dist: xxhash<4.0.0,>=3.0.0
+Requires-Dist: anthropic<0.50.0,>=0.36.0
+Requires-Dist: openai>=2.0.0
+Requires-Dist: google-genai>=1.0.0
+Requires-Dist: xai-sdk>=1.2.0
+Requires-Dist: tqdm<5.0.0,>=4.0.0
+Dynamic: license-file
+
+# osmosis-ai
+
+A Python library that provides reward and rubric validation helpers for LLM applications with strict type enforcement.
+
+## Installation
+
+```bash
+pip install osmosis-ai
+```
+
+Requires Python 3.9 or newer.
+
+This installs the Osmosis CLI and pulls in the required provider SDKs (`openai`, `anthropic`, `google-genai`, `xai-sdk`) along with supporting utilities such as `PyYAML`, `python-dotenv`, `requests`, and `xxhash`.
+
+For development:
+```bash
+git clone https://github.com/Osmosis-AI/osmosis-sdk-python
+cd osmosis-sdk-python
+pip install -e .
+```
+
+## Quick Start
+
+```python
+from osmosis_ai import osmosis_reward
+
+@osmosis_reward
+def simple_reward(solution_str: str, ground_truth: str, extra_info: dict = None) -> float:
+    """Basic exact match reward function."""
+    return 1.0 if solution_str.strip() == ground_truth.strip() else 0.0
+
+# Use the reward function
+score = simple_reward("hello world", "hello world")  # Returns 1.0
+```
+
+```python
+from osmosis_ai import evaluate_rubric
+
+solution = "The capital of France is Paris."
+
+# Export OPENAI_API_KEY in your shell before running this snippet.
+rubric_score = evaluate_rubric(
+    rubric="Assistant must mention the verified capital city.",
+    solution_str=solution,
+    model_info={
+        "provider": "openai",
+        "model": "gpt-5",
+        "api_key_env": "OPENAI_API_KEY",
+    },
+    ground_truth="Paris",
+)
+
+print(rubric_score)  # -> 1.0 (full payload available via return_details=True)
+```
+
+## Remote Rubric Evaluation
+
+`evaluate_rubric` talks to each provider through its official Python SDK while enforcing the same JSON schema everywhere:
+
+- **OpenAI / xAI** – Uses `OpenAI(...).responses.create` (or `chat.completions.create`) with `response_format={"type": "json_schema"}` and falls back to `json_object` when needed.
+- **Anthropic** – Forces a tool call with a JSON schema via `Anthropic(...).messages.create`, extracting the returned tool arguments.
+- **Google Gemini** – Invokes `google.genai.Client(...).models.generate_content` with `response_mime_type="application/json"` and `response_schema`.
+
+Every provider therefore returns a strict JSON object with `{"score": number, "explanation": string}`. The helper clamps the score into your configured range, validates the structure, and exposes the raw payload when `return_details=True`.
+
+Credentials are resolved from environment variables by default:
+
+- `OPENAI_API_KEY` for OpenAI
+- `ANTHROPIC_API_KEY` for Anthropic
+- `GOOGLE_API_KEY` for Google Gemini
+- `XAI_API_KEY` for xAI
+
+Override the environment variable name with `model_info={"api_key_env": "CUSTOM_ENV_NAME"}` when needed, or supply an inline secret with `model_info={"api_key": "sk-..."}` for ephemeral credentials. Missing API keys raise a `MissingAPIKeyError` that explains how to export the secret before trying again.
+
+`api_key` and `api_key_env` are mutually exclusive ways to provide the same credential. When `api_key` is present and non-empty it is used directly, skipping any environment lookup. Otherwise the resolver falls back to `api_key_env` (or the provider default) and pulls the value from your local environment with `os.getenv`.
+
+`model_info` accepts additional rubric-specific knobs:
+
+- `score_min` / `score_max` – change the default `[0.0, 1.0]` scoring bounds.
+- `system_prompt` / `original_input` – provide optional context strings that will be quoted in the judging prompt.
+- `timeout` – customise the provider timeout in seconds.
+
+Pass `metadata={...}` to `evaluate_rubric` when you need structured context quoted in the judge prompt, and set `return_details=True` to receive the full `RewardRubricRunResult` payload (including the provider’s raw response).
+
+Remote failures surface as `ProviderRequestError` instances, with `ModelNotFoundError` reserved for missing model identifiers so you can retry with a new snapshot.
+
+> Older SDK versions that lack schema parameters automatically fall back to instruction-only JSON; the helper still validates the response payload before returning.
+> Provider model snapshot names change frequently. Check each vendor's dashboard for the latest identifier if you encounter a “model not found” error.
+
+### Provider Architecture
+
+All remote integrations live in `osmosis_ai/providers/` and implement the `RubricProvider` interface. At import time the default registry registers OpenAI, xAI, Anthropic, and Google Gemini so `evaluate_rubric` can route requests without additional configuration. The request/response plumbing is encapsulated in each provider module, keeping `evaluate_rubric` focused on prompt construction, payload validation, and credential resolution.
+
+Add your own provider by subclassing `RubricProvider`, implementing `run()` with the vendor SDK, and calling `register_provider()` during start-up. A step-by-step guide is available in [`osmosis_ai/providers/README.md`](osmosis_ai/providers/README.md).
+
+## Required Function Signature
+
+All functions decorated with `@osmosis_reward` must have exactly this signature:
+
+```python
+@osmosis_reward
+def your_function(solution_str: str, ground_truth: str, extra_info: dict = None) -> float:
+    # Your reward logic here
+    return float_score
+```
+
+### Parameters
+
+- **`solution_str: str`** - The solution string to evaluate (required)
+- **`ground_truth: str`** - The correct/expected answer (required)
+- **`extra_info: dict = None`** - Optional dictionary for additional configuration
+
+### Return Value
+
+- **`-> float`** - Must return a float value representing the reward score
+
+The decorator will raise a `TypeError` if the function doesn't match this exact signature or doesn't return a float.
+
+## Rubric Function Signature
+
+Rubric functions decorated with `@osmosis_rubric` must match this signature:
+
+```python
+@osmosis_rubric
+def your_rubric(solution_str: str, ground_truth: str | None, extra_info: dict) -> float:
+    # Your rubric logic here
+    return float_score
+```
+
+> The runtime forwards `None` for `ground_truth` when no reference answer exists. Annotate the parameter as `Optional[str]` (or handle `None` explicitly) if your rubric logic expects to run in that scenario.
+
+### Required `extra_info` fields
+
+- **`provider`** – Non-empty string identifying the judge provider.
+- **`model`** – Non-empty string naming the provider model to call.
+- **`rubric`** – Natural-language rubric instructions for the judge model.
+- **`api_key` / `api_key_env`** – Supply either the raw key or the environment variable name that exposes it.
+
+### Optional `extra_info` fields
+
+- **`system_prompt`** – Optional string prepended to the provider’s base system prompt when invoking the judge; include it inside `extra_info` rather than as a separate argument.
+- **`score_min` / `score_max`** – Optional numeric overrides for the expected score range.
+- **`model_info_overrides`** – Optional dict merged into the provider configuration passed to the judge.
+
+Additional keys are passthrough and can be used for custom configuration. If you need to extend the provider payload (for example adding `api_key_env`), add a dict under `model_info_overrides` and it will be merged with the required `provider`/`model` pair before invoking `evaluate_rubric`. The decorator enforces the parameter names/annotations, validates the embedded configuration at call time, and ensures the wrapped function returns a `float`.
+
+> Annotation quirk: `extra_info` must be annotated as `dict` **without** a default value, unlike `@osmosis_reward`.
+
+> Tip: When delegating to `evaluate_rubric`, pass the raw `solution_str` directly and include any extra context inside the `metadata` payload.
+
+## Examples
+
+See the [`examples/`](examples/) directory for complete examples:
+
+```python
+@osmosis_reward
+def case_insensitive_match(solution_str: str, ground_truth: str, extra_info: dict = None) -> float:
+    """Case-insensitive string matching with partial credit."""
+    match = solution_str.lower().strip() == ground_truth.lower().strip()
+
+    if extra_info and 'partial_credit' in extra_info:
+        if not match and extra_info['partial_credit']:
+            len_diff = abs(len(solution_str) - len(ground_truth))
+            if len_diff <= 2:
+                return 0.5
+
+    return 1.0 if match else 0.0
+
+@osmosis_reward
+def numeric_tolerance(solution_str: str, ground_truth: str, extra_info: dict = None) -> float:
+    """Numeric comparison with configurable tolerance."""
+    try:
+        solution_num = float(solution_str.strip())
+        truth_num = float(ground_truth.strip())
+
+        tolerance = extra_info.get('tolerance', 0.01) if extra_info else 0.01
+        return 1.0 if abs(solution_num - truth_num) <= tolerance else 0.0
+    except ValueError:
+        return 0.0
+```
+
+- `examples/rubric_functions.py` demonstrates `evaluate_rubric` with OpenAI, Anthropic, Gemini, and xAI using the schema-enforced SDK integrations.
+- `examples/reward_functions.py` keeps local reward helpers that showcase the decorator contract without external calls.
+- `examples/rubric_configs.yaml` bundles two rubric definitions with provider configuration and scoring bounds.
+- `examples/sample_data.jsonl` contains two rubric-aligned solution strings so you can trial dataset validation.
+
+```yaml
+# examples/rubric_configs.yaml (excerpt)
+version: 1
+rubrics:
+  - id: support_followup
+    model_info:
+      provider: openai
+      model: gpt-5-mini
+      api_key_env: OPENAI_API_KEY
+```
+
+```jsonl
+{"conversation_id": "ticket-001", "rubric_id": "support_followup", "original_input": "...", "solution_str": "..."}
+{"conversation_id": "ticket-047", "rubric_id": "policy_grounding", "original_input": "...", "solution_str": "..."}
+```
+
+## CLI Tools
+
+Installing the SDK also provides a lightweight CLI available as `osmosis` (aliases: `osmosis_ai`, `osmosis-ai`) for inspecting rubric YAML files and JSONL test payloads.
+
+Preview a rubric file and print every configuration discovered, including nested entries:
+
+```bash
+osmosis preview --path path/to/rubric.yaml
+```
+
+Preview a dataset of rubric-scored solutions stored as JSONL:
+
+```bash
+osmosis preview --path path/to/data.jsonl
+```
+
+Evaluate a dataset against a hosted rubric configuration and print the returned scores:
+
+```bash
+osmosis eval --rubric support_followup --data examples/sample_data.jsonl
+```
+
+- Supply the dataset with `-d`/`--data path/to/data.jsonl`; the path is resolved relative to the current working directory.
+- Use `--config path/to/rubric_configs.yaml` when the rubric definitions are not located alongside the dataset.
+- Pass `-n`/`--number` to sample the provider multiple times per record; the CLI prints every run along with aggregate statistics (average, variance, standard deviation, and min/max).
+- Provide `--output path/to/dir` to create the directory (if needed) and emit `rubric_eval_result_<unix_timestamp>.json`, or supply a full file path (any extension) to control the filename; each file captures every run, provider payloads, timestamps, and aggregate statistics for downstream analysis.
+- Skip `--output` to collect results under `~/.cache/osmosis/eval_result/<rubric_id>/rubric_eval_result_<identifier>.json`; the CLI writes this JSON whether the evaluation finishes cleanly or hits provider/runtime errors so you can inspect failures later (only a manual Ctrl+C interrupt leaves no file behind).
+- Dataset rows whose `rubric_id` does not match the requested rubric are skipped automatically.
+- Each dataset record must provide a non-empty `solution_str`; optional fields such as `original_input`, `ground_truth`, and `extra_info` travel with the record and are forwarded to the evaluator when present.
+- When delegating to a custom `@osmosis_rubric` function, the CLI enriches `extra_info` with the active `provider`, `model`, `rubric`, score bounds, any configured `system_prompt`, the resolved `original_input`, and the record’s metadata/extra fields so the decorator’s required entries are always present.
+- Rubric configuration files intentionally reject `extra_info`; provide per-example context through the dataset instead.
+
+Both commands validate the file, echo a short summary (`Loaded <n> ...`), and pretty-print the parsed records so you can confirm that new rubrics or test fixtures look correct before committing them. Invalid files raise a descriptive error and exit with a non-zero status code.
+
+## Running Examples
+
+```bash
+PYTHONPATH=. python examples/reward_functions.py
+PYTHONPATH=. python examples/rubric_functions.py  # Uncomment the provider you need before running
+```
+
+## Testing
+
+Run `python -m pytest` (or any subset under `tests/`) to exercise the updated helpers:
+
+- `tests/test_rubric_eval.py` covers prompt construction for `solution_str` evaluations.
+- `tests/test_cli_services.py` validates dataset parsing, extra-info enrichment, and engine interactions.
+- `tests/test_cli.py` ensures the CLI pathways surface the new fields end to end.
+
+Add additional tests under `tests/` as you extend the library.
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file for details.
+
+## Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Run tests and examples
+5. Submit a pull request
+
+## Links
+
+- [Homepage](https://github.com/Osmosis-AI/osmosis-sdk-python)
+- [Issues](https://github.com/Osmosis-AI/osmosis-sdk-python/issues)

osmosis_ai-0.2.4.dist-info/RECORD

@@ -0,0 +1,27 @@
+osmosis_ai/__init__.py,sha256=2_qXxu18Yc7UicqxFZds8PjR4q0mTY1Xt17iR38OFbw,725
+osmosis_ai/cli.py,sha256=EPCttBnj1TEqQuO2gmS9iHadYcudiizVM38jACztRFE,1320
+osmosis_ai/cli_commands.py,sha256=CmTcb5N3delW7z3fwucss89xw5MHgIrJJ2Z5xdAuIeU,6165
+osmosis_ai/consts.py,sha256=dAC-7yKjt7CIBpExhm1TKOhZ-1U5XyQD_tRiCewcse0,73
+osmosis_ai/rubric_eval.py,sha256=wsrHU4OUWx3jy0kjyMZh2TYaJjbbz3hG4QvbajV_4Dk,12196
+osmosis_ai/rubric_types.py,sha256=kJvNAjLd3Y-1Q-_Re9HLTprLAUO3qtwR-IWOBeMkFI8,1279
+osmosis_ai/utils.py,sha256=G8xU2tKFZfbUpznp42tHBW7UNP0zCZzoWQ9EP79yDLE,12960
+osmosis_ai/cli_services/__init__.py,sha256=nfbzDMXTJQfNUt7npIuO5JwUWTp6WWnuWMRWwntJXQo,1461
+osmosis_ai/cli_services/config.py,sha256=pHMU7EY3J_t51ojJeBUVBFnw_jIYllpyUDTpt5hCq4o,13873
+osmosis_ai/cli_services/dataset.py,sha256=oA1n7Nfqkn0xVJxls6Aflpk85rEx625T9TnuZcoM0Ig,6651
+osmosis_ai/cli_services/engine.py,sha256=DaXHAPXpiqHnWoqWXKIGGZOIbn6X7t63qtQlwbP9e40,15866
+osmosis_ai/cli_services/errors.py,sha256=nI6jlICyA4MMNKmwDHQBwyJVah5PVwstmra1HpGkVLE,136
+osmosis_ai/cli_services/reporting.py,sha256=H2g0BmEE2stVey4RmurQM713VowH8984a9r7oDstSkA,12499
+osmosis_ai/cli_services/session.py,sha256=Ru3HA80eqRYZGD1e38N8yd96FiAY8cIYpJvEOHKakM0,6597
+osmosis_ai/cli_services/shared.py,sha256=PilPfW5oDvNL5VG8oObSq2ZL35QPFmhBDf0V4gfd2Ro,5942
+osmosis_ai/providers/__init__.py,sha256=yLSExLbJToZ8AUOVxt4LDplxtIuwv-etSJJyZOcOE2Q,927
+osmosis_ai/providers/anthropic_provider.py,sha256=zrWCVP8co4v8xhcJDFLASwvwEADKN-1p34cY_GH4q5M,3758
+osmosis_ai/providers/base.py,sha256=fN5cnWXYAHN53RR_x6ykbUkM4bictNPDj4U8yd4b2a0,1492
+osmosis_ai/providers/gemini_provider.py,sha256=QANSCmkKungpkpDP2RClmKYnwNVrGv3MKxJwkh68IhY,12045
+osmosis_ai/providers/openai_family.py,sha256=DeQWPMcafEvG4xcI97m3AADTKP2pYw9KwcQTcQg-h_4,26078
+osmosis_ai/providers/shared.py,sha256=dmVe8JDgafPmo6HkP-Kl0aWfffhAT6u3ElV_wLlYD34,2957
+osmosis_ai-0.2.4.dist-info/licenses/LICENSE,sha256=FV2ZmyhdCYinoLLvU_ci-7pZ3DeNYY9XqZjVjOd3h94,1064
+osmosis_ai-0.2.4.dist-info/METADATA,sha256=kyx1K0U5C2Jz9cW7c_zqqEVkAMeeUxjtrE1GfrezicI,15440
+osmosis_ai-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+osmosis_ai-0.2.4.dist-info/entry_points.txt,sha256=aF1CR36a9I9_vcF7nlK9JnK1Iqu614vPy2_jh4QU26A,114
+osmosis_ai-0.2.4.dist-info/top_level.txt,sha256=UPNRTKIBSrxsJVNxwXnLCqSoBS4bAiL_3jMtjvf5zEY,11
+osmosis_ai-0.2.4.dist-info/RECORD,,

osmosis_ai-0.2.4.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+osmosis = osmosis_ai.cli:main
+osmosis-ai = osmosis_ai.cli:main
+osmosis_ai = osmosis_ai.cli:main