judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/api/api_types.py

@@ -0,0 +1,413 @@
+# generated by datamodel-codegen:
+#   filename:  .openapi.json
+#   timestamp: 2025-11-18T18:52:11+00:00
+
+from __future__ import annotations
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+from typing_extensions import NotRequired
+
+
+TraceAndSpanId = List
+
+
+class LogEvalResultsResponse(TypedDict):
+    ui_results_url: str
+
+
+class EvalResultsFetch(TypedDict):
+    experiment_run_id: str
+    project_name: str
+
+
+class FetchExperimentRunResponse(TypedDict):
+    results: NotRequired[Optional[List]]
+    ui_results_url: NotRequired[Optional[str]]
+
+
+class DatasetFetch(TypedDict):
+    dataset_name: str
+    project_name: str
+
+
+class DatasetsFetch(TypedDict):
+    project_name: str
+
+
+class ProjectAdd(TypedDict):
+    project_name: str
+
+
+class ProjectAddResponse(TypedDict):
+    project_id: str
+
+
+class ProjectDeleteFromJudgevalResponse(TypedDict):
+    project_name: str
+
+
+class ProjectDeleteResponse(TypedDict):
+    message: str
+
+
+class ScorerExistsRequest(TypedDict):
+    name: str
+
+
+class ScorerExistsResponse(TypedDict):
+    exists: bool
+
+
+class SavePromptScorerRequest(TypedDict):
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+
+
+class FetchPromptScorersRequest(TypedDict):
+    names: NotRequired[Optional[List[str]]]
+    is_trace: NotRequired[Optional[bool]]
+
+
+class CustomScorerUploadPayload(TypedDict):
+    scorer_name: str
+    scorer_code: str
+    requirements_text: str
+    overwrite: NotRequired[bool]
+
+
+class CustomScorerTemplateResponse(TypedDict):
+    scorer_name: str
+    status: str
+    message: str
+
+
+class PromptInsertRequest(TypedDict):
+    project_id: str
+    name: str
+    prompt: str
+    tags: List[str]
+
+
+class PromptInsertResponse(TypedDict):
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+
+
+class PromptTagRequest(TypedDict):
+    project_id: str
+    name: str
+    commit_id: str
+    tags: List[str]
+
+
+class PromptTagResponse(TypedDict):
+    commit_id: str
+
+
+class PromptUntagRequest(TypedDict):
+    project_id: str
+    name: str
+    tags: List[str]
+
+
+class PromptUntagResponse(TypedDict):
+    commit_ids: List[str]
+
+
+class ResolveProjectNameRequest(TypedDict):
+    project_name: str
+
+
+class ResolveProjectNameResponse(TypedDict):
+    project_id: str
+
+
+class TraceIdRequest(TypedDict):
+    trace_id: str
+
+
+class SpanScoreRequest(TypedDict):
+    span_id: str
+    trace_id: str
+
+
+class BaseScorer(TypedDict):
+    score_type: str
+    threshold: NotRequired[float]
+    name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    score: NotRequired[Optional[float]]
+    score_breakdown: NotRequired[Optional[Dict[str, Any]]]
+    reason: NotRequired[Optional[str]]
+    using_native_model: NotRequired[Optional[bool]]
+    success: NotRequired[Optional[bool]]
+    model: NotRequired[Optional[str]]
+    model_client: NotRequired[Any]
+    strict_mode: NotRequired[bool]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    user: NotRequired[Optional[str]]
+    server_hosted: NotRequired[bool]
+
+
+class ScorerConfig(TypedDict):
+    score_type: str
+    name: NotRequired[Optional[str]]
+    threshold: NotRequired[float]
+    strict_mode: NotRequired[bool]
+    required_params: NotRequired[List[str]]
+    kwargs: NotRequired[Optional[Dict[str, Any]]]
+
+
+class Example(TypedDict):
+    example_id: NotRequired[str]
+    created_at: NotRequired[str]
+    name: NotRequired[Optional[str]]
+
+
+class ValidationError(TypedDict):
+    loc: List[Union[str, int]]
+    msg: str
+    type: str
+
+
+class UsageInfo(TypedDict):
+    total_judgees: int
+    regular_use: int
+    pay_as_you_go_use: int
+    remaining_regular: int
+    remaining_after: int
+
+
+DatasetKind = Literal["trace", "example"]
+
+
+class PromptScorer(TypedDict):
+    id: str
+    user_id: str
+    organization_id: str
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+    created_at: NotRequired[Optional[str]]
+    updated_at: NotRequired[Optional[str]]
+    is_trace: NotRequired[Optional[bool]]
+    is_bucket_rubric: NotRequired[Optional[bool]]
+
+
+class PromptCommitInfo(TypedDict):
+    name: str
+    prompt: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+    first_name: str
+    last_name: str
+    user_email: str
+
+
+class ScorerData(TypedDict):
+    id: NotRequired[str]
+    name: str
+    threshold: float
+    success: bool
+    score: NotRequired[Optional[float]]
+    reason: NotRequired[Optional[str]]
+    strict_mode: NotRequired[Optional[bool]]
+    evaluation_model: NotRequired[Optional[str]]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+
+
+class OtelTraceSpan(TypedDict):
+    organization_id: str
+    project_id: NotRequired[Optional[str]]
+    user_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[List[Dict[str, Any]]]]
+
+
+class OtelSpanListItemScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+
+
+class OtelSpanDetailScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+    example_id: NotRequired[Optional[str]]
+
+
+class ExampleEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    created_at: NotRequired[str]
+    examples: List[Example]
+    trace_span_id: NotRequired[Optional[str]]
+    trace_id: NotRequired[Optional[str]]
+
+
+class HTTPValidationError(TypedDict):
+    detail: NotRequired[List[ValidationError]]
+
+
+class TraceEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    created_at: NotRequired[str]
+    trace_and_span_ids: List[TraceAndSpanId]
+    is_offline: NotRequired[bool]
+    is_bucket_run: NotRequired[bool]
+
+
+class DatasetInsertExamples(TypedDict):
+    dataset_name: str
+    examples: List[Example]
+    project_name: str
+
+
+class DatasetInfo(TypedDict):
+    dataset_id: str
+    name: str
+    created_at: str
+    kind: DatasetKind
+    entries: int
+    creator: str
+
+
+class DatasetCreate(TypedDict):
+    name: str
+    dataset_kind: DatasetKind
+    project_name: str
+    examples: List[Example]
+    overwrite: bool
+
+
+class SavePromptScorerResponse(TypedDict):
+    scorer_response: PromptScorer
+
+
+class FetchPromptScorersResponse(TypedDict):
+    scorers: List[PromptScorer]
+
+
+class PromptFetchResponse(TypedDict):
+    commit: NotRequired[Optional[PromptCommitInfo]]
+
+
+class PromptVersionsResponse(TypedDict):
+    versions: List[PromptCommitInfo]
+
+
+class ScoringResult(TypedDict):
+    success: bool
+    scorers_data: List[ScorerData]
+    name: NotRequired[Optional[str]]
+    data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
+    trace_id: NotRequired[Optional[str]]
+    run_duration: NotRequired[Optional[float]]
+    evaluation_cost: NotRequired[Optional[float]]
+
+
+class OtelTraceListItem(TypedDict):
+    organization_id: str
+    project_id: str
+    trace_id: str
+    created_at: str
+    duration: NotRequired[Optional[int]]
+    tags: NotRequired[Optional[List[str]]]
+    experiment_run_id: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    llm_cost: NotRequired[Optional[float]]
+    error: NotRequired[str]
+    scores: NotRequired[List[OtelSpanListItemScores]]
+    rules_invoked: NotRequired[List[str]]
+    customer_id: NotRequired[Optional[str]]
+    input: NotRequired[Optional[str]]
+    output: NotRequired[Optional[str]]
+    input_preview: NotRequired[Optional[str]]
+    output_preview: NotRequired[Optional[str]]
+    annotation_count: NotRequired[int]
+    span_id: str
+    rule_id: NotRequired[Optional[str]]
+
+
+class OtelSpanDetail(TypedDict):
+    organization_id: str
+    project_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]]
+    llm_cost: NotRequired[Optional[float]]
+    prompt_tokens: NotRequired[Optional[int]]
+    completion_tokens: NotRequired[Optional[int]]
+    scores: NotRequired[Optional[List[OtelSpanDetailScores]]]
+
+
+class EvaluateResponse(TypedDict):
+    status: str
+    results: List[ScoringResult]
+    resource_usage: NotRequired[Optional[UsageInfo]]
+
+
+class EvalResults(TypedDict):
+    results: List[ScoringResult]
+    run: Union[ExampleEvaluationRun, TraceEvaluationRun]
+
+
+class DatasetTraceWithSpans(TypedDict):
+    dataset_id: str
+    trace_detail: OtelTraceListItem
+    spans: List[OtelSpanDetail]
+
+
+class DatasetReturn(TypedDict):
+    name: str
+    project_name: str
+    dataset_kind: DatasetKind
+    examples: NotRequired[List[Example]]
+    traces: NotRequired[Optional[List[DatasetTraceWithSpans]]]

judgeval/cli.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import sys
+import typer
+from pathlib import Path
+from dotenv import load_dotenv
+from judgeval.logger import judgeval_logger
+from judgeval import JudgmentClient
+from judgeval.version import get_version
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.utils.project import _resolve_project_id
+from judgeval.utils.url import url_for
+
+load_dotenv()
+
+app = typer.Typer(
+    no_args_is_help=True,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_short=False,
+    rich_help_panel=None,
+    rich_markup_mode=None,
+)
+
+
+@app.command(
+    context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
+)
+def load_otel_env(
+    ctx: typer.Context,
+    project_name: str = typer.Argument(help="Project name to send telemetry to"),
+    api_key: str = typer.Option(None, envvar="JUDGMENT_API_KEY"),
+    organization_id: str = typer.Option(None, envvar="JUDGMENT_ORG_ID"),
+):
+    """Run command with OpenTelemetry environment variables configured for Judgment."""
+    if not api_key or not organization_id:
+        raise typer.BadParameter("JUDGMENT_API_KEY and JUDGMENT_ORG_ID required")
+
+    project_id = _resolve_project_id(project_name, api_key, organization_id)
+    if not project_id:
+        raise typer.BadParameter(f"Project '{project_name}' not found")
+
+    if not ctx.args:
+        raise typer.BadParameter(
+            "No command provided. Usage: judgeval load_otel_env PROJECT_NAME -- COMMAND"
+        )
+
+    env = os.environ.copy()
+    env["OTEL_TRACES_EXPORTER"] = "otlp"
+    env["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] = "http/protobuf"
+    env["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = url_for("/otel/v1/traces")
+    env["OTEL_EXPORTER_OTLP_HEADERS"] = (
+        f"Authorization=Bearer {api_key},X-Organization-Id={organization_id},X-Project-Id={project_id}"
+    )
+
+    result = subprocess.run(ctx.args, env=env)
+    sys.exit(result.returncode)
+
+
+@app.command()
+def upload_scorer(
+    scorer_file_path: str = typer.Argument(help="Path to scorer Python file"),
+    requirements_file_path: str = typer.Argument(help="Path to requirements.txt file"),
+    unique_name: str = typer.Option(
+        None, help="Custom scorer name (auto-detected if not provided)"
+    ),
+    overwrite: bool = typer.Option(
+        False, "--overwrite", "-o", help="Overwrite if exists"
+    ),
+    api_key: str = typer.Option(None, envvar="JUDGMENT_API_KEY"),
+    organization_id: str = typer.Option(None, envvar="JUDGMENT_ORG_ID"),
+):
+    """Upload custom scorer to Judgment."""
+    scorer_path = Path(scorer_file_path)
+    requirements_path = Path(requirements_file_path)
+
+    if not scorer_path.exists():
+        raise typer.BadParameter(f"Scorer file not found: {scorer_file_path}")
+    if not requirements_path.exists():
+        raise typer.BadParameter(
+            f"Requirements file not found: {requirements_file_path}"
+        )
+
+    client = JudgmentClient(api_key=api_key, organization_id=organization_id)
+
+    try:
+        result = client.upload_custom_scorer(
+            scorer_file_path=scorer_file_path,
+            requirements_file_path=requirements_file_path,
+            unique_name=unique_name,
+            overwrite=overwrite,
+        )
+        if not result:
+            raise typer.Abort()
+        judgeval_logger.info("Custom scorer uploaded successfully!")
+    except JudgmentAPIError as e:
+        if e.status_code == 409:
+            judgeval_logger.error("Scorer exists. Use --overwrite to replace")
+            raise typer.Exit(1)
+        raise
+
+
+@app.command()
+def version():
+    """Show Judgeval CLI version."""
+    typer.echo(f"Judgeval CLI v{get_version()}")
+
+
+if __name__ == "__main__":
+    app()

judgeval/constants.py

@@ -1,10 +1,8 @@
-"""
-Constant variables used throughout source code
-"""
+from __future__ import annotations
 
 from enum import Enum
+from typing import Set
 import litellm
-import os
 
 
 class APIScorerType(str, Enum):
@@ -16,37 +14,25 @@ class APIScorerType(str, Enum):
     """
 
     PROMPT_SCORER = "Prompt Scorer"
+    TRACE_PROMPT_SCORER = "Trace Prompt Scorer"
     FAITHFULNESS = "Faithfulness"
     ANSWER_RELEVANCY = "Answer Relevancy"
     ANSWER_CORRECTNESS = "Answer Correctness"
     INSTRUCTION_ADHERENCE = "Instruction Adherence"
     EXECUTION_ORDER = "Execution Order"
-    DERAILMENT = "Derailment"
-    TOOL_ORDER = "Tool Order"
-    CLASSIFIER = "Classifier"
-    TOOL_DEPENDENCY = "Tool Dependency"
     CUSTOM = "Custom"
 
     @classmethod
-    def _missing_(cls, value):
-        # Handle case-insensitive lookup
+    def __missing__(cls, value: str) -> APIScorerType:
         for member in cls:
             if member.value == value.lower():
                 return member
 
+        raise ValueError(f"Invalid scorer type: {value}")
 
-UNBOUNDED_SCORERS: set[APIScorerType] = (
-    set()
-)  # scorers whose scores are not bounded between 0-1
 
-# RabbitMQ
-RABBITMQ_HOST = os.getenv(
-    "RABBITMQ_HOST", "rabbitmq-networklb-faa155df16ec9085.elb.us-west-1.amazonaws.com"
-)
-RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672)
-RABBITMQ_QUEUE = os.getenv("RABBITMQ_QUEUE", "task_queue")
-# Models
-LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
+LITELLM_SUPPORTED_MODELS: Set[str] = set(litellm.model_list)
+
 
 TOGETHER_SUPPORTED_MODELS = [
     "meta-llama/Meta-Llama-3-70B-Instruct-Turbo",
@@ -109,12 +95,3 @@ JUDGMENT_SUPPORTED_MODELS = {"osiris-large", "osiris-mini", "osiris"}
 ACCEPTABLE_MODELS = (
     set(litellm.model_list) | set(TOGETHER_SUPPORTED_MODELS) | JUDGMENT_SUPPORTED_MODELS
 )
-
-## System settings
-MAX_WORKER_THREADS = 10
-
-# Maximum number of concurrent operations for evaluation runs
-MAX_CONCURRENT_EVALUATIONS = 50  # Adjust based on system capabilities
-
-# Span lifecycle management
-SPAN_LIFECYCLE_END_UPDATE_ID = 20  # Default ending number for completed spans

judgeval/data/__init__.py

@@ -1,7 +1,7 @@
 from judgeval.data.example import Example, ExampleParams
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
-from judgeval.data.trace import Trace, TraceSpan, TraceUsage
+from judgeval.data.trace import TraceUsage
 
 
 __all__ = [
@@ -11,7 +11,5 @@ __all__ = [
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
-    "Trace",
-    "TraceSpan",
     "TraceUsage",
 ]