judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/api/api_types.py

@@ -0,0 +1,407 @@
+# generated by datamodel-codegen:
+#   filename:  .openapi.json
+#   timestamp: 2025-10-25T22:30:20+00:00
+
+from __future__ import annotations
+from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
+from typing_extensions import NotRequired
+
+
+TraceAndSpanId = List
+
+
+class EvalResultsFetch(TypedDict):
+    experiment_run_id: str
+    project_name: str
+
+
+class DatasetFetch(TypedDict):
+    dataset_name: str
+    project_name: str
+
+
+class DatasetsFetch(TypedDict):
+    project_name: str
+
+
+class ProjectAdd(TypedDict):
+    project_name: str
+
+
+class ProjectAddResponse(TypedDict):
+    project_id: str
+
+
+class ProjectDeleteFromJudgevalResponse(TypedDict):
+    project_name: str
+
+
+class ProjectDeleteResponse(TypedDict):
+    message: str
+
+
+class ScorerExistsRequest(TypedDict):
+    name: str
+
+
+class ScorerExistsResponse(TypedDict):
+    exists: bool
+
+
+class SavePromptScorerRequest(TypedDict):
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    is_trace: NotRequired[bool]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+
+
+class FetchPromptScorersRequest(TypedDict):
+    names: NotRequired[Optional[List[str]]]
+    is_trace: NotRequired[Optional[bool]]
+
+
+class CustomScorerUploadPayload(TypedDict):
+    scorer_name: str
+    scorer_code: str
+    requirements_text: str
+    overwrite: NotRequired[bool]
+
+
+class CustomScorerTemplateResponse(TypedDict):
+    scorer_name: str
+    status: str
+    message: str
+
+
+class PromptInsertRequest(TypedDict):
+    project_id: str
+    name: str
+    prompt: str
+    tags: List[str]
+
+
+class PromptInsertResponse(TypedDict):
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+
+
+class PromptTagRequest(TypedDict):
+    project_id: str
+    name: str
+    commit_id: str
+    tags: List[str]
+
+
+class PromptTagResponse(TypedDict):
+    commit_id: str
+
+
+class PromptUntagRequest(TypedDict):
+    project_id: str
+    name: str
+    tags: List[str]
+
+
+class PromptUntagResponse(TypedDict):
+    commit_ids: List[str]
+
+
+class ResolveProjectNameRequest(TypedDict):
+    project_name: str
+
+
+class ResolveProjectNameResponse(TypedDict):
+    project_id: str
+
+
+class TraceIdRequest(TypedDict):
+    trace_id: str
+
+
+class SpanScoreRequest(TypedDict):
+    span_id: str
+    trace_id: str
+
+
+class BaseScorer(TypedDict):
+    score_type: str
+    threshold: NotRequired[float]
+    name: NotRequired[Optional[str]]
+    class_name: NotRequired[Optional[str]]
+    score: NotRequired[Optional[float]]
+    score_breakdown: NotRequired[Optional[Dict[str, Any]]]
+    reason: NotRequired[Optional[str]]
+    using_native_model: NotRequired[Optional[bool]]
+    success: NotRequired[Optional[bool]]
+    model: NotRequired[Optional[str]]
+    model_client: NotRequired[Any]
+    strict_mode: NotRequired[bool]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+    user: NotRequired[Optional[str]]
+    server_hosted: NotRequired[bool]
+
+
+class ScorerConfig(TypedDict):
+    score_type: str
+    name: NotRequired[Optional[str]]
+    threshold: NotRequired[float]
+    model: NotRequired[Optional[str]]
+    strict_mode: NotRequired[bool]
+    required_params: NotRequired[List[str]]
+    kwargs: NotRequired[Optional[Dict[str, Any]]]
+
+
+class Example(TypedDict):
+    example_id: NotRequired[str]
+    created_at: NotRequired[str]
+    name: NotRequired[Optional[str]]
+
+
+class ValidationError(TypedDict):
+    loc: List[Union[str, int]]
+    msg: str
+    type: str
+
+
+class UsageInfo(TypedDict):
+    total_judgees: int
+    regular_use: int
+    pay_as_you_go_use: int
+    remaining_regular: int
+    remaining_after: int
+
+
+DatasetKind = Literal["trace", "example"]
+
+
+class PromptScorer(TypedDict):
+    id: str
+    user_id: str
+    organization_id: str
+    name: str
+    prompt: str
+    threshold: float
+    model: NotRequired[str]
+    options: NotRequired[Optional[Dict[str, float]]]
+    description: NotRequired[Optional[str]]
+    created_at: NotRequired[Optional[str]]
+    updated_at: NotRequired[Optional[str]]
+    is_trace: NotRequired[Optional[bool]]
+    is_bucket_rubric: NotRequired[Optional[bool]]
+
+
+class PromptCommitInfo(TypedDict):
+    name: str
+    prompt: str
+    tags: List[str]
+    commit_id: str
+    parent_commit_id: NotRequired[Optional[str]]
+    created_at: str
+    first_name: str
+    last_name: str
+    user_email: str
+
+
+class ScorerData(TypedDict):
+    id: NotRequired[str]
+    name: str
+    threshold: float
+    success: bool
+    score: NotRequired[Optional[float]]
+    reason: NotRequired[Optional[str]]
+    strict_mode: NotRequired[Optional[bool]]
+    evaluation_model: NotRequired[Optional[str]]
+    error: NotRequired[Optional[str]]
+    additional_metadata: NotRequired[Optional[Dict[str, Any]]]
+
+
+class OtelTraceSpan(TypedDict):
+    organization_id: str
+    project_id: NotRequired[Optional[str]]
+    user_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[List[Dict[str, Any]]]]
+
+
+class OtelSpanListItemScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+
+
+class OtelSpanDetailScores(TypedDict):
+    success: bool
+    score: float
+    reason: NotRequired[Optional[str]]
+    name: str
+    example_id: NotRequired[Optional[str]]
+
+
+class ExampleEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    model: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
+    examples: List[Example]
+    trace_span_id: NotRequired[Optional[str]]
+    trace_id: NotRequired[Optional[str]]
+
+
+class HTTPValidationError(TypedDict):
+    detail: NotRequired[List[ValidationError]]
+
+
+class TraceEvaluationRun(TypedDict):
+    id: NotRequired[str]
+    project_name: str
+    eval_name: str
+    custom_scorers: NotRequired[List[BaseScorer]]
+    judgment_scorers: NotRequired[List[ScorerConfig]]
+    model: NotRequired[Optional[str]]
+    created_at: NotRequired[str]
+    trace_and_span_ids: List[TraceAndSpanId]
+    is_offline: NotRequired[bool]
+    is_bucket_run: NotRequired[bool]
+
+
+class DatasetInsertExamples(TypedDict):
+    dataset_name: str
+    examples: List[Example]
+    project_name: str
+
+
+class DatasetInfo(TypedDict):
+    dataset_id: str
+    name: str
+    created_at: str
+    kind: DatasetKind
+    entries: int
+    creator: str
+
+
+class DatasetCreate(TypedDict):
+    name: str
+    dataset_kind: DatasetKind
+    project_name: str
+    examples: List[Example]
+    overwrite: bool
+
+
+class SavePromptScorerResponse(TypedDict):
+    scorer_response: PromptScorer
+
+
+class FetchPromptScorersResponse(TypedDict):
+    scorers: List[PromptScorer]
+
+
+class PromptFetchResponse(TypedDict):
+    commit: NotRequired[Optional[PromptCommitInfo]]
+
+
+class PromptVersionsResponse(TypedDict):
+    versions: List[PromptCommitInfo]
+
+
+class ScoringResult(TypedDict):
+    success: bool
+    scorers_data: List[ScorerData]
+    name: NotRequired[Optional[str]]
+    data_object: NotRequired[Optional[Union[OtelTraceSpan, Example]]]
+    trace_id: NotRequired[Optional[str]]
+    run_duration: NotRequired[Optional[float]]
+    evaluation_cost: NotRequired[Optional[float]]
+
+
+class OtelTraceListItem(TypedDict):
+    organization_id: str
+    project_id: str
+    trace_id: str
+    created_at: str
+    duration: NotRequired[Optional[int]]
+    tags: NotRequired[Optional[List[str]]]
+    experiment_run_id: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    llm_cost: NotRequired[Optional[float]]
+    error: NotRequired[str]
+    scores: NotRequired[List[OtelSpanListItemScores]]
+    rules_invoked: NotRequired[List[str]]
+    customer_id: NotRequired[Optional[str]]
+    input: NotRequired[Optional[str]]
+    output: NotRequired[Optional[str]]
+    input_preview: NotRequired[Optional[str]]
+    output_preview: NotRequired[Optional[str]]
+    annotation_count: NotRequired[int]
+    span_id: str
+    rule_id: NotRequired[Optional[str]]
+
+
+class OtelSpanDetail(TypedDict):
+    organization_id: str
+    project_id: str
+    timestamp: str
+    trace_id: str
+    span_id: str
+    parent_span_id: NotRequired[Optional[str]]
+    trace_state: NotRequired[Optional[str]]
+    span_name: NotRequired[Optional[str]]
+    span_kind: NotRequired[Optional[str]]
+    service_name: NotRequired[Optional[str]]
+    resource_attributes: NotRequired[Optional[Dict[str, Any]]]
+    span_attributes: NotRequired[Optional[Dict[str, Any]]]
+    duration: NotRequired[Optional[int]]
+    status_code: NotRequired[Optional[int]]
+    status_message: NotRequired[Optional[str]]
+    events: NotRequired[Optional[List[Dict[str, Any]]]]
+    links: NotRequired[Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]]
+    llm_cost: NotRequired[Optional[float]]
+    prompt_tokens: NotRequired[Optional[int]]
+    completion_tokens: NotRequired[Optional[int]]
+    scores: NotRequired[Optional[List[OtelSpanDetailScores]]]
+
+
+class EvaluateResponse(TypedDict):
+    status: str
+    results: List[ScoringResult]
+    resource_usage: NotRequired[Optional[UsageInfo]]
+
+
+class EvalResults(TypedDict):
+    results: List[ScoringResult]
+    run: Union[ExampleEvaluationRun, TraceEvaluationRun]
+
+
+class DatasetTraceWithSpans(TypedDict):
+    dataset_id: str
+    trace_detail: OtelTraceListItem
+    spans: List[OtelSpanDetail]
+
+
+class DatasetReturn(TypedDict):
+    name: str
+    project_name: str
+    dataset_kind: DatasetKind
+    examples: NotRequired[List[Example]]
+    traces: NotRequired[Optional[List[DatasetTraceWithSpans]]]

judgeval/cli.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import typer
+from pathlib import Path
+from dotenv import load_dotenv
+from judgeval.logger import judgeval_logger
+from judgeval import JudgmentClient
+from judgeval.version import get_version
+from judgeval.exceptions import JudgmentAPIError
+
+load_dotenv()
+
+app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode=None,
+    rich_help_panel=None,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_short=False,
+)
+
+
+@app.command("upload_scorer")
+def upload_scorer(
+    scorer_file_path: str,
+    requirements_file_path: str,
+    unique_name: str = typer.Option(
+        None, help="Custom name for the scorer (auto-detected if not provided)"
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        "-o",
+        help="Overwrite existing scorer if it already exists",
+    ),
+):
+    # Validate file paths
+    if not Path(scorer_file_path).exists():
+        judgeval_logger.error(f"Scorer file not found: {scorer_file_path}")
+        raise typer.Exit(1)
+
+    if not Path(requirements_file_path).exists():
+        judgeval_logger.error(f"Requirements file not found: {requirements_file_path}")
+        raise typer.Exit(1)
+
+    try:
+        client = JudgmentClient()
+
+        result = client.upload_custom_scorer(
+            scorer_file_path=scorer_file_path,
+            requirements_file_path=requirements_file_path,
+            unique_name=unique_name,
+            overwrite=overwrite,
+        )
+
+        if not result:
+            judgeval_logger.error("Failed to upload custom scorer")
+            raise typer.Exit(1)
+
+        judgeval_logger.info("Custom scorer uploaded successfully!")
+        raise typer.Exit(0)
+    except Exception as e:
+        if isinstance(e, JudgmentAPIError) and e.status_code == 409:
+            judgeval_logger.error(
+                "Duplicate scorer detected. Use --overwrite flag to replace the existing scorer"
+            )
+            raise typer.Exit(1)
+        # Re-raise other exceptions
+        raise
+
+
+@app.command()
+def version():
+    """Show version info"""
+    judgeval_logger.info(f"Judgeval CLI v{get_version()}")
+
+
+if __name__ == "__main__":
+    app()

judgeval/constants.py

@@ -1,68 +1,97 @@
-"""
-Constant variables used throughout source code
-"""
+from __future__ import annotations
 
 from enum import Enum
+from typing import Set
 import litellm
-import os
 
-class APIScorer(str, Enum):  
+
+class APIScorerType(str, Enum):
     """
     Collection of proprietary scorers implemented by Judgment.
 
     These are ready-made evaluation scorers that can be used to evaluate
     Examples via the Judgment API.
     """
-    FAITHFULNESS = "faithfulness"
-    ANSWER_RELEVANCY = "answer_relevancy"
-    ANSWER_CORRECTNESS = "answer_correctness"
-    HALLUCINATION = "hallucination"
-    SUMMARIZATION = "summarization"
-    CONTEXTUAL_RECALL = "contextual_recall"
-    CONTEXTUAL_RELEVANCY = "contextual_relevancy"
-    CONTEXTUAL_PRECISION = "contextual_precision"
-    TOOL_CORRECTNESS = "tool_correctness"
-    JSON_CORRECTNESS = "json_correctness"
+
+    PROMPT_SCORER = "Prompt Scorer"
+    TRACE_PROMPT_SCORER = "Trace Prompt Scorer"
+    FAITHFULNESS = "Faithfulness"
+    ANSWER_RELEVANCY = "Answer Relevancy"
+    ANSWER_CORRECTNESS = "Answer Correctness"
+    INSTRUCTION_ADHERENCE = "Instruction Adherence"
+    EXECUTION_ORDER = "Execution Order"
+    CUSTOM = "Custom"
 
     @classmethod
-    def _missing_(cls, value):
-        # Handle case-insensitive lookup
+    def __missing__(cls, value: str) -> APIScorerType:
         for member in cls:
             if member.value == value.lower():
                 return member
 
-ROOT_API = os.getenv("JUDGMENT_API_URL", "https://api.judgmentlabs.ai")
-# API URLs
-JUDGMENT_EVAL_API_URL = f"{ROOT_API}/evaluate/"
-JUDGMENT_DATASETS_PUSH_API_URL = f"{ROOT_API}/datasets/push/"
-JUDGMENT_DATASETS_PULL_API_URL = f"{ROOT_API}/datasets/pull/"
-JUDGMENT_DATASETS_PULL_ALL_API_URL = f"{ROOT_API}/datasets/get_all_stats/"
-JUDGMENT_EVAL_LOG_API_URL = f"{ROOT_API}/log_eval_results/"
-JUDGMENT_EVAL_FETCH_API_URL = f"{ROOT_API}/fetch_eval_results/"
-JUDGMENT_EVAL_DELETE_API_URL = f"{ROOT_API}/delete_eval_results_by_project_and_run_name/"
-JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
-JUDGMENT_TRACES_FETCH_API_URL = f"{ROOT_API}/traces/fetch/"
-JUDGMENT_TRACES_SAVE_API_URL = f"{ROOT_API}/traces/save/"
-JUDGMENT_TRACES_DELETE_API_URL = f"{ROOT_API}/traces/delete/"
+        raise ValueError(f"Invalid scorer type: {value}")
+
 
-# RabbitMQ
-RABBITMQ_HOST = os.getenv("RABBITMQ_HOST", "rabbitmq-networklb-faa155df16ec9085.elb.us-west-1.amazonaws.com")
-RABBITMQ_PORT = os.getenv("RABBITMQ_PORT", 5672)
-RABBITMQ_QUEUE = os.getenv("RABBITMQ_QUEUE", "task_queue")
+LITELLM_SUPPORTED_MODELS: Set[str] = set(litellm.model_list)
 
-# Models
-TOGETHER_SUPPORTED_MODELS = {
-    "QWEN": "Qwen/Qwen2-72B-Instruct",
-    "LLAMA3_70B_INSTRUCT_TURBO": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
-    "LLAMA3_405B_INSTRUCT_TURBO": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
-    "LLAMA3_8B_INSTRUCT_TURBO": "meta-llama/Llama-3.2-3B-Instruct-Turbo",
-    "MISTRAL_8x22B_INSTRUCT": "mistralai/Mixtral-8x22B-Instruct-v0.1",
-    "MISTRAL_8x7B_INSTRUCT": "mistralai/Mixtral-8x7B-Instruct-v0.1",
-}
 
-JUDGMENT_SUPPORTED_MODELS = {"osiris-large", "osiris-mini"}
+TOGETHER_SUPPORTED_MODELS = [
+    "meta-llama/Meta-Llama-3-70B-Instruct-Turbo",
+    "Qwen/Qwen2-VL-72B-Instruct",
+    "meta-llama/Llama-Vision-Free",
+    "Gryphe/MythoMax-L2-13b",
+    "Qwen/Qwen2.5-72B-Instruct-Turbo",
+    "meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
+    "deepseek-ai/DeepSeek-R1",
+    "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo",
+    "meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
+    "google/gemma-2-27b-it",
+    "mistralai/Mistral-Small-24B-Instruct-2501",
+    "mistralai/Mixtral-8x22B-Instruct-v0.1",
+    "meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
+    "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo-classifier",
+    "deepseek-ai/DeepSeek-V3",
+    "Qwen/Qwen2-72B-Instruct",
+    "meta-llama/Meta-Llama-3-8B-Instruct-Lite",
+    "deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+    "upstage/SOLAR-10.7B-Instruct-v1.0",
+    "togethercomputer/MoA-1",
+    "Qwen/QwQ-32B-Preview",
+    "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
+    "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+    "mistralai/Mistral-7B-Instruct-v0.2",
+    "databricks/dbrx-instruct",
+    "meta-llama/Llama-3-8b-chat-hf",
+    "google/gemma-2b-it",
+    "meta-llama/Meta-Llama-3-70B-Instruct-Lite",
+    "google/gemma-2-9b-it",
+    "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo-p",
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+    "Gryphe/MythoMax-L2-13b-Lite",
+    "meta-llama/Llama-2-7b-chat-hf",
+    "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
+    "meta-llama/Llama-2-13b-chat-hf",
+    "scb10x/scb10x-llama3-typhoon-v1-5-8b-instruct",
+    "scb10x/scb10x-llama3-typhoon-v1-5x-4f316",
+    "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF",
+    "Qwen/Qwen2.5-Coder-32B-Instruct",
+    "microsoft/WizardLM-2-8x22B",
+    "mistralai/Mistral-7B-Instruct-v0.3",
+    "scb10x/scb10x-llama3-1-typhoon2-60256",
+    "Qwen/Qwen2.5-7B-Instruct-Turbo",
+    "scb10x/scb10x-llama3-1-typhoon-18370",
+    "meta-llama/Llama-3.2-3B-Instruct-Turbo",
+    "meta-llama/Llama-3-70b-chat-hf",
+    "mistralai/Mixtral-8x7B-Instruct-v0.1",
+    "togethercomputer/MoA-1-Turbo",
+    "deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free",
+    "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
+    "mistralai/Mistral-7B-Instruct-v0.1",
+]
 
-ACCEPTABLE_MODELS = set(litellm.model_list) | set(TOGETHER_SUPPORTED_MODELS.keys()) | JUDGMENT_SUPPORTED_MODELS
+JUDGMENT_SUPPORTED_MODELS = {"osiris-large", "osiris-mini", "osiris"}
 
-## System settings
-MAX_WORKER_THREADS = 10
+ACCEPTABLE_MODELS = (
+    set(litellm.model_list) | set(TOGETHER_SUPPORTED_MODELS) | JUDGMENT_SUPPORTED_MODELS
+)

judgeval/data/__init__.py

@@ -1,15 +1,15 @@
 from judgeval.data.example import Example, ExampleParams
-from judgeval.data.api_example import ProcessExample, create_process_example
 from judgeval.data.scorer_data import ScorerData, create_scorer_data
 from judgeval.data.result import ScoringResult, generate_scoring_result
+from judgeval.data.trace import TraceUsage
+
 
 __all__ = [
     "Example",
     "ExampleParams",
-    "ProcessExample",
-    "create_process_example",
     "ScorerData",
     "create_scorer_data",
     "ScoringResult",
     "generate_scoring_result",
+    "TraceUsage",
 ]