osmosis-ai 0.1.8__py3-none-any.whl → 0.2.4__py3-none-any.whl

osmosis_ai/cli_services/reporting.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+from .engine import EvaluationRecordResult, EvaluationReport, EvaluationRun
+from .errors import CLIError
+from .shared import calculate_stat_deltas
+
+
+class TextReportFormatter:
+    """Builds human-readable text lines for an evaluation report."""
+
+    def build(
+        self,
+        report: EvaluationReport,
+        baseline: Optional["BaselineStatistics"] = None,
+    ) -> list[str]:
+        lines: list[str] = []
+        provider = str(report.rubric_config.model_info.get("provider", "")).strip() or "<unknown>"
+        model_name = str(report.rubric_config.model_info.get("model", "")).strip() or "<unspecified>"
+
+        lines.append(
+            f"Rubric '{report.rubric_config.rubric_id}' "
+            f"({report.rubric_config.source_label}) -> provider '{provider}' model '{model_name}'"
+        )
+        lines.append(f"Loaded {len(report.record_results)} matching record(s) from {report.data_path}")
+        lines.append(f"Running {report.number} evaluation(s) per record")
+        lines.append("")
+
+        for record_result in report.record_results:
+            lines.extend(self._format_record(record_result))
+        if baseline is not None:
+            lines.extend(self._format_baseline(report, baseline))
+        return lines
+
+    def _format_record(self, record_result: EvaluationRecordResult) -> list[str]:
+        lines: list[str] = [f"[{record_result.conversation_label}]"]
+        total_runs = len(record_result.runs)
+        for index, run in enumerate(record_result.runs):
+            lines.extend(self._format_run(run))
+            if index < total_runs - 1:
+                lines.append("")
+
+        summary_lines = self._format_summary(record_result.statistics, len(record_result.runs))
+        if summary_lines:
+            lines.extend(summary_lines)
+        lines.append("")
+        return lines
+
+    def _format_run(self, run: EvaluationRun) -> list[str]:
+        lines: list[str] = [f"  Run {run.run_index:02d} [{run.status.upper()}]"]
+        if run.status == "success":
+            score_text = "n/a" if run.score is None else f"{run.score:.4f}"
+            lines.append(self._format_detail_line("score", score_text))
+            if run.preview:
+                lines.append(self._format_detail_line("preview", run.preview))
+            explanation = run.explanation or "(no explanation provided)"
+            lines.append(self._format_detail_line("explanation", explanation))
+        else:
+            error_text = run.error or "(no error message provided)"
+            lines.append(self._format_detail_line("error", error_text))
+            if run.preview:
+                lines.append(self._format_detail_line("preview", run.preview))
+            if run.explanation:
+                lines.append(self._format_detail_line("explanation", run.explanation))
+        lines.append(self._format_detail_line("duration", f"{run.duration_seconds:.2f}s"))
+        return lines
+
+    def _format_summary(self, statistics: dict[str, float], total_runs: int) -> list[str]:
+        if not statistics:
+            return []
+        success_count = int(round(statistics.get("success_count", total_runs)))
+        failure_count = int(round(statistics.get("failure_count", total_runs - success_count)))
+        if total_runs <= 1 and failure_count == 0:
+            return []
+
+        lines = ["  Summary:"]
+        lines.append(f"    total:     {int(round(statistics.get('total_runs', total_runs)))}")
+        lines.append(f"    successes: {success_count}")
+        lines.append(f"    failures:  {failure_count}")
+        if success_count > 0:
+            lines.append(f"    average:   {statistics.get('average', 0.0):.4f}")
+            lines.append(f"    variance:  {statistics.get('variance', 0.0):.6f}")
+            lines.append(f"    stdev:     {statistics.get('stdev', 0.0):.4f}")
+            lines.append(f"    min/max:   {statistics.get('min', 0.0):.4f} / {statistics.get('max', 0.0):.4f}")
+        else:
+            lines.append("    average:   n/a")
+            lines.append("    variance:  n/a")
+            lines.append("    stdev:     n/a")
+            lines.append("    min/max:   n/a")
+        return lines
+
+    def _format_baseline(
+        self,
+        report: EvaluationReport,
+        baseline: "BaselineStatistics",
+    ) -> list[str]:
+        lines = [f"Baseline comparison (source: {baseline.source_path}):"]
+        deltas = baseline.delta(report.overall_statistics)
+        keys = ["average", "variance", "stdev", "min", "max", "success_count", "failure_count", "total_runs"]
+
+        for key in keys:
+            if key not in baseline.statistics or key not in report.overall_statistics:
+                continue
+            baseline_value = float(baseline.statistics[key])
+            current_value = float(report.overall_statistics[key])
+            delta_value = float(deltas.get(key, current_value - baseline_value))
+
+            if key in {"success_count", "failure_count", "total_runs"}:
+                baseline_str = f"{int(round(baseline_value))}"
+                current_str = f"{int(round(current_value))}"
+                delta_str = f"{delta_value:+.0f}"
+            else:
+                precision = 6 if key == "variance" else 4
+                baseline_str = format(baseline_value, f".{precision}f")
+                current_str = format(current_value, f".{precision}f")
+                delta_str = format(delta_value, f"+.{precision}f")
+
+            lines.append(
+                f"  {key:12s} baseline={baseline_str} current={current_str} delta={delta_str}"
+            )
+        return lines
+
+    @staticmethod
+    def _format_detail_line(label: str, value: str, *, indent: int = 4) -> str:
+        indent_str = " " * indent
+        value_str = str(value)
+        continuation_indent = indent_str + "  "
+        value_str = value_str.replace("\n", f"\n{continuation_indent}")
+        return f"{indent_str}{label}: {value_str}"
+
+
+class ConsoleReportRenderer:
+    """Pretty prints evaluation reports to stdout (or any printer function)."""
+
+    def __init__(
+        self,
+        printer: Callable[[str], None] = print,
+        formatter: Optional[TextReportFormatter] = None,
+    ):
+        self._printer = printer
+        self._formatter = formatter or TextReportFormatter()
+
+    def render(
+        self,
+        report: EvaluationReport,
+        baseline: Optional["BaselineStatistics"] = None,
+    ) -> None:
+        for line in self._formatter.build(report, baseline):
+            self._printer(line)
+
+
+class BaselineStatistics:
+    def __init__(self, source_path: Path, statistics: dict[str, float]):
+        self.source_path = source_path
+        self.statistics = statistics
+
+    def delta(self, current: dict[str, float]) -> dict[str, float]:
+        return calculate_stat_deltas(self.statistics, current)
+
+
+class BaselineComparator:
+    """Loads baseline JSON payloads and extracts statistics."""
+
+    def load(self, path: Path) -> BaselineStatistics:
+        if not path.exists():
+            raise CLIError(f"Baseline path '{path}' does not exist.")
+        if path.is_dir():
+            raise CLIError(f"Baseline path '{path}' is a directory, expected JSON file.")
+
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError as exc:
+            raise CLIError(f"Failed to parse baseline JSON: {exc}") from exc
+
+        if not isinstance(payload, dict):
+            raise CLIError("Baseline JSON must contain an object.")
+
+        source = None
+        if isinstance(payload.get("overall_statistics"), dict):
+            source = payload["overall_statistics"]
+        elif all(key in payload for key in ("average", "variance", "stdev")):
+            source = payload
+        if source is None:
+            raise CLIError(
+                "Baseline JSON must include an 'overall_statistics' object or top-level statistics."
+            )
+
+        statistics: dict[str, float] = {}
+        for key, value in source.items():
+            try:
+                statistics[key] = float(value)
+            except (TypeError, ValueError):
+                continue
+        if not statistics:
+            raise CLIError("Baseline statistics could not be parsed into numeric values.")
+
+        return BaselineStatistics(source_path=path, statistics=statistics)
+
+
+class JsonReportFormatter:
+    """Builds JSON-serialisable payloads for evaluation reports."""
+
+    def build(
+        self,
+        report: EvaluationReport,
+        *,
+        output_identifier: Optional[str],
+        baseline: Optional[BaselineStatistics],
+    ) -> dict[str, Any]:
+        provider = str(report.rubric_config.model_info.get("provider", "")).strip() or "<unknown>"
+        model_name = str(report.rubric_config.model_info.get("model", "")).strip() or "<unspecified>"
+
+        generated: dict[str, Any] = {
+            "generated_at": datetime.now(timezone.utc).isoformat(),
+            "rubric_id": report.rubric_config.rubric_id,
+            "rubric_source": report.rubric_config.source_label,
+            "provider": provider,
+            "model": model_name,
+            "number": report.number,
+            "config_path": str(report.config_path),
+            "data_path": str(report.data_path),
+            "overall_statistics": _normalise_statistics(report.overall_statistics),
+            "records": [],
+        }
+        if output_identifier is not None:
+            generated["output_identifier"] = output_identifier
+
+        for record_result in report.record_results:
+            conversation_label = record_result.conversation_label
+            record_identifier = record_result.record.record_identifier(conversation_label)
+
+            record_payload: dict[str, Any] = {
+                "id": record_identifier,
+                "record_index": record_result.record_index,
+                "conversation_id": conversation_label,
+                "input_record": record_result.record.payload,
+                "statistics": _normalise_statistics(record_result.statistics),
+                "runs": [],
+            }
+
+            for run in record_result.runs:
+                record_payload["runs"].append(
+                    {
+                        "run_index": run.run_index,
+                        "status": run.status,
+                        "started_at": run.started_at.isoformat(),
+                        "completed_at": run.completed_at.isoformat(),
+                        "duration_seconds": run.duration_seconds,
+                        "score": run.score,
+                        "explanation": run.explanation,
+                        "preview": run.preview,
+                        "error": run.error,
+                        "raw": run.raw,
+                    }
+                )
+
+            generated["records"].append(record_payload)
+
+        if baseline is not None:
+            generated["baseline_comparison"] = {
+                "source_path": str(baseline.source_path),
+                "baseline_statistics": _normalise_statistics(baseline.statistics),
+                "delta_statistics": _normalise_statistics(baseline.delta(report.overall_statistics)),
+            }
+
+        return generated
+
+
+class JsonReportWriter:
+    """Serialises an evaluation report to disk."""
+
+    def __init__(self, formatter: Optional[JsonReportFormatter] = None):
+        self._formatter = formatter or JsonReportFormatter()
+
+    def write(
+        self,
+        report: EvaluationReport,
+        *,
+        output_path: Path,
+        output_identifier: Optional[str],
+        baseline: Optional[BaselineStatistics],
+    ) -> Path:
+        parent_dir = output_path.parent
+        if parent_dir and not parent_dir.exists():
+            parent_dir.mkdir(parents=True, exist_ok=True)
+
+        payload = self._formatter.build(
+            report,
+            output_identifier=output_identifier,
+            baseline=baseline,
+        )
+        with output_path.open("w", encoding="utf-8") as fh:
+            json.dump(payload, fh, indent=2, ensure_ascii=False)
+        return output_path
+
+def _normalise_statistics(stats: dict[str, float]) -> dict[str, Any]:
+    normalised: dict[str, Any] = {}
+    for key, value in stats.items():
+        if key in {"success_count", "failure_count", "total_runs"}:
+            normalised[key] = int(round(value))
+        else:
+            normalised[key] = float(value)
+    return normalised

osmosis_ai/cli_services/session.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, Sequence
+
+from ..rubric_eval import ensure_api_key_available
+from ..rubric_types import MissingAPIKeyError
+from .config import RubricConfig, RubricSuite, discover_rubric_config_path, load_rubric_suite
+from .dataset import DatasetLoader, DatasetRecord
+from .engine import RubricEvaluationEngine, EvaluationReport
+from .errors import CLIError
+from .reporting import BaselineComparator, BaselineStatistics, JsonReportWriter
+
+
+_CACHE_ROOT = Path("~/.cache/osmosis/eval_result").expanduser()
+
+
+def _sanitise_rubric_folder(rubric_id: str) -> str:
+    """Produce a filesystem-safe folder name for the rubric id."""
+    clean = "".join(ch if ch.isalnum() or ch in {"-", "_"} else "_" for ch in rubric_id.strip())
+    clean = clean.strip("_") or "rubric"
+    return clean.lower()
+
+
+@dataclass(frozen=True)
+class EvaluationSessionRequest:
+    rubric_id: str
+    data_path: Path
+    number: int = 1
+    config_path: Optional[Path] = None
+    output_path: Optional[Path] = None
+    output_identifier: Optional[str] = None
+    baseline_path: Optional[Path] = None
+
+
+@dataclass
+class EvaluationSessionResult:
+    request: EvaluationSessionRequest
+    config_path: Path
+    data_path: Path
+    rubric_config: RubricConfig
+    records: Sequence[DatasetRecord]
+    report: EvaluationReport
+    baseline: Optional[BaselineStatistics]
+    written_path: Optional[Path]
+    output_identifier: Optional[str]
+
+
+class EvaluationSession:
+    """Coordinates rubric evaluation end-to-end for reusable orchestration."""
+
+    def __init__(
+        self,
+        *,
+        config_locator: Callable[[Optional[str], Path], Path] = discover_rubric_config_path,
+        suite_loader: Callable[[Path], RubricSuite] = load_rubric_suite,
+        dataset_loader: Optional[DatasetLoader] = None,
+        engine: Optional[RubricEvaluationEngine] = None,
+        baseline_comparator: Optional[BaselineComparator] = None,
+        report_writer: Optional[JsonReportWriter] = None,
+        identifier_factory: Optional[Callable[[], str]] = None,
+    ):
+        self._config_locator = config_locator
+        self._suite_loader = suite_loader
+        self._dataset_loader = dataset_loader or DatasetLoader()
+        self._engine = engine or RubricEvaluationEngine()
+        self._baseline_comparator = baseline_comparator or BaselineComparator()
+        self._report_writer = report_writer or JsonReportWriter()
+        self._identifier_factory = identifier_factory or self._default_identifier
+
+    def execute(self, request: EvaluationSessionRequest) -> EvaluationSessionResult:
+        rubric_id = request.rubric_id.strip()
+        if not rubric_id:
+            raise CLIError("Rubric identifier cannot be empty.")
+
+        number_value = request.number if request.number is not None else 1
+        number = int(number_value)
+        if number < 1:
+            raise CLIError("Number of runs must be a positive integer.")
+
+        data_path = request.data_path.expanduser()
+        if not data_path.exists():
+            raise CLIError(f"Data path '{data_path}' does not exist.")
+        if data_path.is_dir():
+            raise CLIError(f"Expected a JSONL file but received directory '{data_path}'.")
+
+        config_override = str(request.config_path.expanduser()) if request.config_path else None
+        config_path = self._config_locator(config_override, data_path)
+        suite = self._suite_loader(config_path)
+        rubric_config = suite.get(rubric_id)
+
+        try:
+            ensure_api_key_available(rubric_config.model_info)
+        except (MissingAPIKeyError, TypeError) as exc:
+            raise CLIError(str(exc)) from exc
+
+        all_records = self._dataset_loader.load(data_path)
+        matching_records = [
+            record for record in all_records if record.rubric_id.lower() == rubric_id.lower()
+        ]
+        if not matching_records:
+            raise CLIError(f"No records in '{data_path}' reference rubric '{rubric_id}'.")
+
+        baseline_stats = self._load_baseline(request.baseline_path)
+
+        resolved_output_path, resolved_identifier = self._resolve_output_path(
+            request.output_path,
+            request.output_identifier,
+            rubric_id=rubric_id,
+        )
+
+        report = self._engine.execute(
+            rubric_config=rubric_config,
+            config_path=config_path,
+            data_path=data_path,
+            records=matching_records,
+            number=number,
+        )
+
+        written_path = None
+        if resolved_output_path is not None:
+            written_path = self._report_writer.write(
+                report,
+                output_path=resolved_output_path,
+                output_identifier=resolved_identifier,
+                baseline=baseline_stats,
+            )
+
+        return EvaluationSessionResult(
+            request=request,
+            config_path=config_path,
+            data_path=data_path,
+            rubric_config=rubric_config,
+            records=matching_records,
+            report=report,
+            baseline=baseline_stats,
+            written_path=written_path,
+            output_identifier=resolved_identifier,
+        )
+
+    def _load_baseline(self, baseline_path: Optional[Path]) -> Optional[BaselineStatistics]:
+        if baseline_path is None:
+            return None
+        resolved = baseline_path.expanduser()
+        return self._baseline_comparator.load(resolved)
+
+    def _resolve_output_path(
+        self,
+        output_candidate: Optional[Path],
+        output_identifier: Optional[str],
+        *,
+        rubric_id: str,
+    ) -> tuple[Optional[Path], Optional[str]]:
+        if output_candidate is None:
+            identifier = output_identifier or self._identifier_factory()
+            target_dir = _CACHE_ROOT / _sanitise_rubric_folder(rubric_id)
+            target_dir.mkdir(parents=True, exist_ok=True)
+            return target_dir / f"rubric_eval_result_{identifier}.json", identifier
+
+        candidate = output_candidate.expanduser()
+        if candidate.suffix:
+            if candidate.exists() and candidate.is_dir():
+                raise CLIError(f"Output path '{candidate}' is a directory.")
+            return candidate, output_identifier
+
+        candidate.mkdir(parents=True, exist_ok=True)
+        identifier = output_identifier or self._identifier_factory()
+        return candidate / f"rubric_eval_result_{identifier}.json", identifier
+
+    @staticmethod
+    def _default_identifier() -> str:
+        return str(int(time.time()))

osmosis_ai/cli_services/shared.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from statistics import mean, pvariance, pstdev
+from typing import Any, Collection, Optional, Set
+
+from .errors import CLIError
+
+
+def coerce_optional_float(value: Any, field_name: str, source_label: str) -> Optional[float]:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    raise CLIError(
+        f"Expected '{field_name}' in {source_label} to be numeric, got {type(value).__name__}."
+    )
+
+
+def collapse_preview_text(value: Any, *, max_length: int = 140) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    collapsed = " ".join(value.strip().split())
+    if not collapsed:
+        return None
+    if len(collapsed) > max_length:
+        collapsed = collapsed[: max_length - 3].rstrip() + "..."
+    return collapsed
+
+
+def calculate_statistics(scores: list[float]) -> dict[str, float]:
+    if not scores:
+        return {
+            "average": 0.0,
+            "variance": 0.0,
+            "stdev": 0.0,
+            "min": 0.0,
+            "max": 0.0,
+        }
+    average = mean(scores)
+    variance = pvariance(scores)
+    std_dev = pstdev(scores)
+    return {
+        "average": average,
+        "variance": variance,
+        "stdev": std_dev,
+        "min": min(scores),
+        "max": max(scores),
+    }
+
+
+def calculate_stat_deltas(baseline: dict[str, float], current: dict[str, float]) -> dict[str, float]:
+    delta: dict[str, float] = {}
+    for key, current_value in current.items():
+        if key not in baseline:
+            continue
+        try:
+            baseline_value = float(baseline[key])
+            current_numeric = float(current_value)
+        except (TypeError, ValueError):
+            continue
+        delta[key] = current_numeric - baseline_value
+    return delta
+
+
+def gather_text_fragments(
+    node: Any,
+    fragments: list[str],
+    *,
+    allow_free_strings: bool = False,
+    seen: Optional[Set[int]] = None,
+    string_key_allowlist: Optional[Collection[str]] = None,
+) -> None:
+    """Collect textual snippets from nested message-like structures.
+
+    The traversal favours common chat-completions shapes (e.g. ``{"type": "text"}``
+    blocks) and avoids indiscriminately pulling in metadata values such as IDs.
+    ``allow_free_strings`` controls whether bare strings encountered at the current
+    level should be considered textual content (useful for raw message content but
+    typically disabled for metadata fields).
+    """
+
+    if seen is None:
+        seen = set()
+
+    if isinstance(node, str):
+        if allow_free_strings:
+            stripped = node.strip()
+            if stripped:
+                fragments.append(stripped)
+        return
+
+    if isinstance(node, list):
+        for item in node:
+            gather_text_fragments(
+                item,
+                fragments,
+                allow_free_strings=allow_free_strings,
+                seen=seen,
+                string_key_allowlist=string_key_allowlist,
+            )
+        return
+
+    if not isinstance(node, dict):
+        return
+
+    node_id = id(node)
+    if node_id in seen:
+        return
+    seen.add(node_id)
+
+    allowlist = {"text", "value", "message"}
+    if string_key_allowlist is not None:
+        allowlist = {key.lower() for key in string_key_allowlist}
+    else:
+        allowlist = {key.lower() for key in allowlist}
+
+    prioritized_keys = ("text", "value")
+    handled_keys: Set[str] = {
+        "text",
+        "value",
+        "content",
+        "message",
+        "parts",
+        "input_text",
+        "output_text",
+        "type",
+        "role",
+        "name",
+        "id",
+        "index",
+        "finish_reason",
+        "reason",
+        "tool_call_id",
+        "metadata",
+    }
+
+    for key in prioritized_keys:
+        if key not in node:
+            continue
+        before_count = len(fragments)
+        gather_text_fragments(
+            node[key],
+            fragments,
+            allow_free_strings=True,
+            seen=seen,
+            string_key_allowlist=string_key_allowlist,
+        )
+        if len(fragments) > before_count:
+            break
+
+    if node.get("type") == "tool_result" and "content" in node:
+        gather_text_fragments(
+            node["content"],
+            fragments,
+            allow_free_strings=True,
+            seen=seen,
+            string_key_allowlist=string_key_allowlist,
+        )
+    elif "content" in node:
+        gather_text_fragments(
+            node["content"],
+            fragments,
+            allow_free_strings=True,
+            seen=seen,
+            string_key_allowlist=string_key_allowlist,
+        )
+
+    for key in ("message", "parts", "input_text", "output_text"):
+        if key in node:
+            gather_text_fragments(
+                node[key],
+                fragments,
+                allow_free_strings=True,
+                seen=seen,
+                string_key_allowlist=string_key_allowlist,
+            )
+
+    for key, value in node.items():
+        if key in handled_keys:
+            continue
+        if isinstance(value, (list, dict)):
+            gather_text_fragments(
+                value,
+                fragments,
+                allow_free_strings=False,
+                seen=seen,
+                string_key_allowlist=string_key_allowlist,
+            )
+        elif isinstance(value, str) and key.lower() in allowlist:
+            stripped = value.strip()
+            if stripped:
+                fragments.append(stripped)
+
+
+def collect_text_fragments(
+    node: Any,
+    *,
+    allow_free_strings: bool = False,
+    string_key_allowlist: Optional[Collection[str]] = None,
+) -> list[str]:
+    fragments: list[str] = []
+    gather_text_fragments(
+        node,
+        fragments,
+        allow_free_strings=allow_free_strings,
+        seen=set(),
+        string_key_allowlist=string_key_allowlist,
+    )
+    return fragments