judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/v1/data/scorer_data.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+from judgeval.v1.internal.api.api_types import ScorerData as APIScorerData
+
+
+@dataclass(slots=True)
+class ScorerData:
+    name: str
+    threshold: float
+    success: bool
+    score: Optional[float] = None
+    reason: Optional[str] = None
+    strict_mode: Optional[bool] = None
+    evaluation_model: Optional[str] = None
+    error: Optional[str] = None
+    additional_metadata: Dict[str, Any] = field(default_factory=dict)
+    id: Optional[str] = None
+
+    def to_dict(self) -> APIScorerData:
+        result: APIScorerData = {
+            "name": self.name,
+            "threshold": self.threshold,
+            "success": self.success,
+        }
+        if self.score is not None:
+            result["score"] = self.score
+        if self.reason is not None:
+            result["reason"] = self.reason
+        if self.strict_mode is not None:
+            result["strict_mode"] = self.strict_mode
+        if self.evaluation_model is not None:
+            result["evaluation_model"] = self.evaluation_model
+        if self.error is not None:
+            result["error"] = self.error
+        if self.additional_metadata:
+            result["additional_metadata"] = self.additional_metadata
+        if self.id is not None:
+            result["id"] = self.id
+        return result

judgeval/v1/data/scoring_result.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+from judgeval.v1.internal.api.api_types import (
+    OtelTraceSpan,
+    ScorerData as APIScorerData,
+    ScoringResult as APIScoringResult,
+)
+from judgeval.v1.data.example import Example
+from judgeval.v1.data.scorer_data import ScorerData
+
+
+@dataclass(slots=True)
+class ScoringResult:
+    success: bool
+    scorers_data: List[ScorerData]
+    name: Optional[str] = None
+    data_object: Optional[Union[OtelTraceSpan, Example]] = None
+    trace_id: Optional[str] = None
+    run_duration: Optional[float] = None
+    evaluation_cost: Optional[float] = None
+
+    def to_dict(self) -> APIScoringResult:
+        scorers_list: List[APIScorerData] = [s.to_dict() for s in self.scorers_data]
+        result: APIScoringResult = {
+            "success": self.success,
+            "scorers_data": scorers_list,
+        }
+        if self.name is not None:
+            result["name"] = self.name
+        if self.data_object is not None:
+            if isinstance(self.data_object, Example):
+                result["data_object"] = self.data_object.to_dict()
+            else:
+                result["data_object"] = self.data_object
+        if self.trace_id is not None:
+            result["trace_id"] = self.trace_id
+        if self.run_duration is not None:
+            result["run_duration"] = self.run_duration
+        if self.evaluation_cost is not None:
+            result["evaluation_cost"] = self.evaluation_cost
+        return result

judgeval/v1/datasets/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.datasets.dataset import Dataset, DatasetInfo
+from judgeval.v1.datasets.dataset_factory import DatasetFactory
+
+__all__ = ["Dataset", "DatasetInfo", "DatasetFactory"]

judgeval/v1/datasets/dataset.py

@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import datetime
+import orjson
+import os
+import yaml
+from dataclasses import dataclass
+from typing import List, Literal, Optional, Iterable, Iterator
+from itertools import islice
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    BarColumn,
+    TaskProgressColumn,
+)
+
+
+from judgeval.v1.data.example import Example
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.logger import judgeval_logger
+
+
+def _batch_examples(
+    examples: Iterable[Example], batch_size: int = 100
+) -> Iterator[List[Example]]:
+    """Generator that yields batches of examples for efficient memory usage.
+
+    Works with any iterable including generators, consuming only batch_size items at a time.
+    """
+    iterator = iter(examples)
+    while True:
+        batch = list(islice(iterator, batch_size))
+        if not batch:
+            break
+        yield batch
+
+
+@dataclass
+class DatasetInfo:
+    dataset_id: str
+    name: str
+    created_at: str
+    kind: str
+    entries: int
+    creator: str
+
+
+@dataclass
+class Dataset:
+    name: str
+    project_name: str
+    dataset_kind: str = "example"
+    examples: Optional[List[Example]] = None
+    client: Optional[JudgmentSyncClient] = None
+
+    def add_from_json(self, file_path: str, batch_size: int = 100) -> None:
+        with open(file_path, "rb") as file:
+            data = orjson.loads(file.read())
+        examples = []
+        for e in data:
+            if isinstance(e, dict):
+                name = e.get("name")
+                example = Example(name=name)
+                for key, value in e.items():
+                    if key != "name":
+                        example.set_property(key, value)
+                examples.append(example)
+            else:
+                examples.append(e)
+        self.add_examples(examples, batch_size=batch_size)
+
+    def add_from_yaml(self, file_path: str, batch_size: int = 100) -> None:
+        with open(file_path, "r") as file:
+            data = yaml.safe_load(file)
+        examples = []
+        for e in data:
+            if isinstance(e, dict):
+                name = e.get("name")
+                example = Example(name=name)
+                for key, value in e.items():
+                    if key != "name":
+                        example.set_property(key, value)
+                examples.append(example)
+            else:
+                examples.append(e)
+        self.add_examples(examples, batch_size=batch_size)
+
+    def add_examples(self, examples: Iterable[Example], batch_size: int = 100) -> None:
+        if not self.client:
+            return
+
+        batches = _batch_examples(examples, batch_size)
+        total_uploaded = 0
+
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[bold blue]{task.description}"),
+            BarColumn(pulse_style="green"),
+            TaskProgressColumn(),
+            TextColumn("[dim]{task.fields[info]}"),
+        ) as progress:
+            task = progress.add_task(
+                f"Uploading to {self.name}",
+                total=None,
+                info="",
+            )
+
+            batch_num = 0
+            for batch in batches:
+                if len(batch) > 0 and not isinstance(batch[0], Example):
+                    raise TypeError("Examples must be a list of Example objects")
+
+                batch_num += 1
+                batch_size_actual = len(batch)
+                total_uploaded += batch_size_actual
+
+                progress.update(
+                    task,
+                    advance=1,
+                    info=f"Batch {batch_num} ({batch_size_actual} examples, {total_uploaded} total)",
+                )
+
+                self.client.datasets_insert_examples_for_judgeval(
+                    {
+                        "dataset_name": self.name,
+                        "project_name": self.project_name,
+                        "examples": [e.to_dict() for e in batch],
+                    }
+                )
+
+        judgeval_logger.info(
+            f"Successfully added {total_uploaded} examples to dataset {self.name}"
+        )
+
+    def save_as(
+        self,
+        file_type: Literal["json", "yaml"],
+        dir_path: str,
+        save_name: Optional[str] = None,
+    ) -> None:
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+
+        file_name = save_name or datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        complete_path = os.path.join(dir_path, f"{file_name}.{file_type}")
+
+        examples_data = [e.to_dict() for e in self.examples] if self.examples else []
+
+        if file_type == "json":
+            with open(complete_path, "wb") as file:
+                file.write(
+                    orjson.dumps(
+                        {"examples": examples_data}, option=orjson.OPT_INDENT_2
+                    )
+                )
+        elif file_type == "yaml":
+            with open(complete_path, "w") as file:
+                yaml.dump({"examples": examples_data}, file, default_flow_style=False)
+
+    def __iter__(self):
+        return iter(self.examples or [])
+
+    def __len__(self):
+        return len(self.examples) if self.examples else 0
+
+    def __str__(self):
+        return f"Dataset(name={self.name}, examples={len(self.examples) if self.examples else 0})"
+
+    def display(self, max_examples: int = 5) -> None:
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+
+        total = len(self.examples) if self.examples else 0
+        console.print(f"\n[bold cyan]Dataset: {self.name}[/bold cyan]")
+        console.print(f"[dim]Project:[/dim] {self.project_name}")
+        console.print(f"[dim]Total examples:[/dim] {total}")
+
+        if not self.examples:
+            console.print("[dim]No examples found[/dim]")
+            return
+
+        display_count = min(max_examples, total)
+
+        if total > 0:
+            first_example = self.examples[0]
+            property_keys = list(first_example.properties.keys())
+
+            table = Table(show_header=True, header_style="bold")
+            table.add_column("#", style="dim", width=4)
+            table.add_column("Name", style="cyan")
+            for key in property_keys[:3]:
+                table.add_column(key, max_width=30)
+
+            for i, example in enumerate(self.examples[:display_count]):
+                row = [str(i + 1), example.name or "—"]
+                for key in property_keys[:3]:
+                    value = str(example.get_property(key) or "")
+                    if len(value) > 30:
+                        value = value[:27] + "..."
+                    row.append(value)
+                table.add_row(*row)
+
+            console.print()
+            console.print(table)
+
+            if total > display_count:
+                console.print(
+                    f"[dim]... and {total - display_count} more examples[/dim]"
+                )
+
+        console.print()

judgeval/v1/datasets/dataset_factory.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from typing import List, Iterable
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.v1.datasets.dataset import Dataset, DatasetInfo
+from judgeval.v1.data.example import Example
+from judgeval.logger import judgeval_logger
+
+
+class DatasetFactory:
+    __slots__ = "_client"
+
+    def __init__(self, client: JudgmentSyncClient):
+        self._client = client
+
+    def get(self, name: str, project_name: str) -> Dataset:
+        dataset = self._client.datasets_pull_for_judgeval(
+            {
+                "dataset_name": name,
+                "project_name": project_name,
+            }
+        )
+
+        dataset_kind = dataset.get("dataset_kind", "example")
+        examples_data = dataset.get("examples", []) or []
+
+        examples = []
+        for e in examples_data:
+            if isinstance(e, dict):
+                judgeval_logger.debug(f"Raw example keys: {e.keys()}")
+
+                data_obj = e.get("data", {})
+                if isinstance(data_obj, dict):
+                    example_id = data_obj.get("example_id", "")
+                    created_at = data_obj.get("created_at", "")
+                    name_field = data_obj.get("name")
+
+                    example = Example(
+                        example_id=example_id, created_at=created_at, name=name_field
+                    )
+
+                    for key, value in data_obj.items():
+                        if key not in ["example_id", "created_at", "name"]:
+                            example.set_property(key, value)
+
+                    examples.append(example)
+                    judgeval_logger.debug(
+                        f"Created example with name={name_field}, properties={list(example.properties.keys())}"
+                    )
+
+        judgeval_logger.info(f"Retrieved dataset {name} with {len(examples)} examples")
+        return Dataset(
+            name=name,
+            project_name=project_name,
+            dataset_kind=dataset_kind,
+            examples=examples,
+            client=self._client,
+        )
+
+    def create(
+        self,
+        name: str,
+        project_name: str,
+        examples: Iterable[Example] = [],
+        overwrite: bool = False,
+        batch_size: int = 100,
+    ) -> Dataset:
+        self._client.datasets_create_for_judgeval(
+            {
+                "name": name,
+                "project_name": project_name,
+                "examples": [],
+                "dataset_kind": "example",
+                "overwrite": overwrite,
+            }
+        )
+        judgeval_logger.info(f"Created dataset {name}")
+
+        if not isinstance(examples, list):
+            examples = list(examples)
+
+        dataset = Dataset(
+            name=name, project_name=project_name, examples=examples, client=self._client
+        )
+        dataset.add_examples(examples, batch_size=batch_size)
+        return dataset
+
+    def list(self, project_name: str) -> List[DatasetInfo]:
+        datasets = self._client.datasets_pull_all_for_judgeval(
+            {"project_name": project_name}
+        )
+        judgeval_logger.info(f"Fetched datasets for project {project_name}")
+        return [DatasetInfo(**d) for d in datasets]

judgeval/v1/evaluation/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.evaluation.evaluation import Evaluation
+from judgeval.v1.evaluation.evaluation_factory import EvaluationFactory
+
+__all__ = ["Evaluation", "EvaluationFactory"]

judgeval/v1/evaluation/evaluation.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import List, Optional
+
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.v1.internal.api.api_types import ExampleEvaluationRun
+from judgeval.v1.data.example import Example
+from judgeval.v1.data.scoring_result import ScoringResult
+from judgeval.v1.data.scorer_data import ScorerData
+from judgeval.v1.scorers.base_scorer import BaseScorer
+from judgeval.logger import judgeval_logger
+
+
+class Evaluation:
+    __slots__ = ("_client",)
+
+    def __init__(self, client: JudgmentSyncClient):
+        self._client = client
+
+    def run(
+        self,
+        examples: List[Example],
+        scorers: List[BaseScorer],
+        project_name: str,
+        eval_run_name: str,
+        model: Optional[str] = None,
+        assert_test: bool = False,
+        timeout_seconds: int = 300,
+    ) -> List[ScoringResult]:
+        console = Console()
+        eval_id = str(uuid.uuid4())
+        created_at = datetime.now(timezone.utc).isoformat()
+
+        console.print("\n[bold cyan]Starting Evaluation[/bold cyan]")
+        console.print(f"[dim]Run:[/dim] {eval_run_name}")
+        console.print(f"[dim]Project:[/dim] {project_name}")
+        console.print(
+            f"[dim]Examples:[/dim] {len(examples)} | [dim]Scorers:[/dim] {len(scorers)}"
+        )
+        if model:
+            console.print(f"[dim]Model:[/dim] {model}")
+
+        judgeval_logger.info(f"Starting evaluation: {eval_run_name}")
+        judgeval_logger.info(f"Examples: {len(examples)}, Scorers: {len(scorers)}")
+
+        payload: ExampleEvaluationRun = {
+            "id": eval_id,
+            "project_name": project_name,
+            "eval_name": eval_run_name,
+            "created_at": created_at,
+            "examples": [e.to_dict() for e in examples],
+            "judgment_scorers": [s.get_scorer_config() for s in scorers],
+            "custom_scorers": [],
+        }
+
+        console.print()
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            TimeElapsedColumn(),
+            console=console,
+        ) as progress:
+            task = progress.add_task("Submitting evaluation...", total=None)
+            self._client.add_to_run_eval_queue_examples(payload)
+            judgeval_logger.info(f"Evaluation submitted: {eval_id}")
+
+            progress.update(task, description="Running evaluation...")
+            start_time = time.time()
+            poll_count = 0
+
+            while True:
+                elapsed = time.time() - start_time
+                if elapsed > timeout_seconds:
+                    raise TimeoutError(f"Evaluation timed out after {timeout_seconds}s")
+
+                response = self._client.fetch_experiment_run(
+                    {"experiment_run_id": eval_id, "project_name": project_name}
+                )
+                results_data = response.get("results", []) or []
+                poll_count += 1
+
+                completed = len(results_data)
+                total = len(examples)
+                progress.update(
+                    task,
+                    description=f"Running evaluation... ({completed}/{total} completed)",
+                )
+                judgeval_logger.info(
+                    f"Poll {poll_count}: {completed}/{total} results ready"
+                )
+
+                if completed == total:
+                    break
+                time.sleep(2)
+
+        console.print(
+            f"[green]✓[/green] Evaluation completed in [bold]{elapsed:.1f}s[/bold]"
+        )
+        judgeval_logger.info(f"Evaluation completed in {elapsed:.1f}s")
+
+        console.print()
+        results = []
+        passed = 0
+        failed = 0
+
+        for i, res in enumerate(results_data):
+            judgeval_logger.info(f"Processing result {i + 1}: {res.keys()}")
+
+            scorers_raw = res.get("scorers", [])
+            scorers_data = []
+            for scorer_dict in scorers_raw:
+                judgeval_logger.debug(f"Scorer data fields: {scorer_dict.keys()}")
+
+                scorer_fields = {
+                    "name": scorer_dict.get("name"),
+                    "threshold": scorer_dict.get("threshold"),
+                    "success": scorer_dict.get("success"),
+                    "score": scorer_dict.get("score"),
+                    "reason": scorer_dict.get("reason"),
+                    "strict_mode": scorer_dict.get("strict_mode"),
+                    "evaluation_model": scorer_dict.get("evaluation_model"),
+                    "error": scorer_dict.get("error"),
+                    "additional_metadata": scorer_dict.get("additional_metadata", {}),
+                    "id": scorer_dict.get("scorer_data_id") or scorer_dict.get("id"),
+                }
+                scorers_data.append(ScorerData(**scorer_fields))
+
+            success = all(s.success for s in scorers_data)
+
+            if success:
+                passed += 1
+                console.print(
+                    f"[green]✓[/green] Example {i + 1}: [green]PASSED[/green]"
+                )
+            else:
+                failed += 1
+                console.print(f"[red]✗[/red] Example {i + 1}: [red]FAILED[/red]")
+
+            for scorer_data in scorers_data:
+                score_str = (
+                    f"{scorer_data.score:.3f}"
+                    if scorer_data.score is not None
+                    else "N/A"
+                )
+                status_color = "green" if scorer_data.success else "red"
+                console.print(
+                    f"  [dim]{scorer_data.name}:[/dim] [{status_color}]{score_str}[/{status_color}] (threshold: {scorer_data.threshold})"
+                )
+
+            results.append(
+                ScoringResult(
+                    success=success,
+                    scorers_data=scorers_data,
+                )
+            )
+
+        console.print()
+        url = response.get("ui_results_url", "")
+
+        if passed == len(results):
+            console.print(
+                f"[bold green]✓ All tests passed![/bold green] ({passed}/{len(results)})"
+            )
+        else:
+            console.print(
+                f"[bold yellow]⚠ Results:[/bold yellow] [green]{passed} passed[/green] | [red]{failed} failed[/red]"
+            )
+
+        console.print(f"[dim]View full details:[/dim] [link={url}]{url}[/link]\n")
+
+        if assert_test and not all(r.success for r in results):
+            raise AssertionError(
+                f"Evaluation failed: {failed}/{len(results)} tests failed"
+            )
+
+        return results

judgeval/v1/evaluation/evaluation_factory.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.v1.evaluation.evaluation import Evaluation
+
+
+class EvaluationFactory:
+    __slots__ = "_client"
+
+    def __init__(
+        self,
+        client: JudgmentSyncClient,
+    ):
+        self._client = client
+
+    def create(self) -> Evaluation:
+        return Evaluation(client=self._client)

judgeval/v1/instrumentation/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from .llm import *
+
+
+__all__ = ["wrap_provider"]

judgeval/v1/instrumentation/llm/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+from .config import _detect_provider, wrap_provider
+
+
+__all__ = ["_detect_provider", "wrap_provider"]