judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/common/api/constants.py

@@ -1,165 +0,0 @@
-import os
-from typing import Optional, TypedDict, List, Dict, Any
-
-ROOT_API = os.getenv("JUDGMENT_API_URL", "https://api.judgmentlabs.ai")
-
-# Traces API
-JUDGMENT_TRACES_FETCH_API_URL = f"{ROOT_API}/traces/fetch/"
-JUDGMENT_TRACES_SAVE_API_URL = f"{ROOT_API}/traces/save/"
-JUDGMENT_TRACES_UPSERT_API_URL = f"{ROOT_API}/traces/upsert/"
-JUDGMENT_TRACES_DELETE_API_URL = f"{ROOT_API}/traces/delete/"
-JUDGMENT_TRACES_SPANS_BATCH_API_URL = f"{ROOT_API}/traces/spans/batch/"
-JUDGMENT_TRACES_EVALUATION_RUNS_BATCH_API_URL = (
-    f"{ROOT_API}/traces/evaluation_runs/batch/"
-)
-
-
-class TraceFetchPayload(TypedDict):
-    trace_id: str
-
-
-class TraceDeletePayload(TypedDict):
-    trace_ids: List[str]
-
-
-class SpansBatchPayload(TypedDict):
-    spans: List[Dict[str, Any]]
-    organization_id: str
-
-
-class EvaluationEntryResponse(TypedDict):
-    evaluation_run: Dict[str, Any]
-    associated_span: Dict[str, Any]
-    queued_at: Optional[float]
-
-
-class EvaluationRunsBatchPayload(TypedDict):
-    organization_id: str
-    evaluation_entries: List[EvaluationEntryResponse]
-
-
-# Evaluation API
-JUDGMENT_EVAL_API_URL = f"{ROOT_API}/evaluate/"
-JUDGMENT_TRACE_EVAL_API_URL = f"{ROOT_API}/evaluate_trace/"
-JUDGMENT_EVAL_LOG_API_URL = f"{ROOT_API}/log_eval_results/"
-JUDGMENT_EVAL_FETCH_API_URL = f"{ROOT_API}/fetch_experiment_run/"
-JUDGMENT_EVAL_DELETE_API_URL = (
-    f"{ROOT_API}/delete_eval_results_by_project_and_run_names/"
-)
-JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
-JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL = f"{ROOT_API}/add_to_run_eval_queue/"
-JUDGMENT_GET_EVAL_STATUS_API_URL = f"{ROOT_API}/get_evaluation_status/"
-JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL = f"{ROOT_API}/check_experiment_type/"
-JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL = f"{ROOT_API}/eval-run-name-exists/"
-
-
-# Evaluation API Payloads
-class EvalRunRequestBody(TypedDict):
-    eval_name: str
-    project_name: str
-    judgment_api_key: str
-
-
-class DeleteEvalRunRequestBody(TypedDict):
-    eval_names: List[str]
-    project_name: str
-    judgment_api_key: str
-
-
-class EvalLogPayload(TypedDict):
-    results: List[Dict[str, Any]]
-    run: Dict[str, Any]
-
-
-class EvalStatusPayload(TypedDict):
-    eval_name: str
-    project_name: str
-    judgment_api_key: str
-
-
-class CheckExperimentTypePayload(TypedDict):
-    eval_name: str
-    project_name: str
-    judgment_api_key: str
-    is_trace: bool
-
-
-class EvalRunNameExistsPayload(TypedDict):
-    eval_name: str
-    project_name: str
-    judgment_api_key: str
-
-
-# Datasets API
-JUDGMENT_DATASETS_PUSH_API_URL = f"{ROOT_API}/datasets/push/"
-JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL = f"{ROOT_API}/datasets/insert_examples/"
-JUDGMENT_DATASETS_PULL_API_URL = f"{ROOT_API}/datasets/pull_for_judgeval/"
-JUDGMENT_DATASETS_DELETE_API_URL = f"{ROOT_API}/datasets/delete/"
-JUDGMENT_DATASETS_EXPORT_JSONL_API_URL = f"{ROOT_API}/datasets/export_jsonl/"
-JUDGMENT_DATASETS_PROJECT_STATS_API_URL = f"{ROOT_API}/datasets/fetch_stats_by_project/"
-JUDGMENT_DATASETS_INSERT_API_URL = f"{ROOT_API}/datasets/insert_examples/"
-
-
-class DatasetPushPayload(TypedDict):
-    dataset_alias: str
-    project_name: str
-    examples: List[Dict[str, Any]]
-    traces: List[Dict[str, Any]]
-    overwrite: bool
-
-
-class DatasetAppendPayload(TypedDict):
-    dataset_alias: str
-    project_name: str
-    examples: List[Dict[str, Any]]
-
-
-class DatasetPullPayload(TypedDict):
-    dataset_alias: str
-    project_name: str
-
-
-class DatasetDeletePayload(TypedDict):
-    dataset_alias: str
-    project_name: str
-
-
-class DatasetExportPayload(TypedDict):
-    dataset_alias: str
-    project_name: str
-
-
-class DatasetStatsPayload(TypedDict):
-    project_name: str
-
-
-# Projects API
-JUDGMENT_PROJECT_DELETE_API_URL = f"{ROOT_API}/projects/delete/"
-JUDGMENT_PROJECT_CREATE_API_URL = f"{ROOT_API}/projects/add/"
-
-
-class ProjectDeletePayload(TypedDict):
-    project_list: List[str]
-
-
-class ProjectCreatePayload(TypedDict):
-    project_name: str
-
-
-JUDGMENT_SCORER_SAVE_API_URL = f"{ROOT_API}/save_scorer/"
-JUDGMENT_SCORER_FETCH_API_URL = f"{ROOT_API}/fetch_scorer/"
-JUDGMENT_SCORER_EXISTS_API_URL = f"{ROOT_API}/scorer_exists/"
-
-
-class ScorerSavePayload(TypedDict):
-    name: str
-    prompt: str
-    options: dict
-
-
-class ScorerFetchPayload(TypedDict):
-    name: str
-
-
-class ScorerExistsPayload(TypedDict):
-    name: str

judgeval/common/exceptions.py

@@ -1,27 +0,0 @@
-"""
-Common Exceptions in Judgeval
-"""
-
-
-class MissingTestCaseParamsError(Exception):
-    pass
-
-
-class JudgmentAPIError(Exception):
-    """
-    Exception raised when an error occurs while executing a Judgment API request
-    """
-
-    def __init__(self, message: str):
-        super().__init__(message)
-        self.message = message
-
-
-class InvalidJudgeModelError(Exception):
-    """
-    Exception raised when an invalid judge model is provided
-    """
-
-    def __init__(self, message: str):
-        super().__init__(message)
-        self.message = message

judgeval/common/storage/__init__.py

@@ -1,6 +0,0 @@
-from judgeval.common.storage.s3_storage import S3Storage
-
-
-__all__ = [
-    "S3Storage",
-]

judgeval/common/storage/s3_storage.py

@@ -1,98 +0,0 @@
-import os
-import json
-import boto3
-from typing import Optional
-from datetime import datetime, UTC
-from botocore.exceptions import ClientError
-from judgeval.common.logger import judgeval_logger
-
-
-class S3Storage:
-    """Utility class for storing and retrieving trace data from S3."""
-
-    def __init__(
-        self,
-        bucket_name: str,
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        region_name: Optional[str] = None,
-    ):
-        """Initialize S3 storage with credentials and bucket name.
-
-        Args:
-            bucket_name: Name of the S3 bucket to store traces in
-            aws_access_key_id: AWS access key ID (optional, will use environment variables if not provided)
-            aws_secret_access_key: AWS secret access key (optional, will use environment variables if not provided)
-            region_name: AWS region name (optional, will use environment variables if not provided)
-        """
-        self.bucket_name = bucket_name
-        self.s3_client = boto3.client(
-            "s3",
-            aws_access_key_id=aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
-            aws_secret_access_key=aws_secret_access_key
-            or os.getenv("AWS_SECRET_ACCESS_KEY"),
-            region_name=region_name or os.getenv("AWS_REGION", "us-west-1"),
-        )
-
-    def _ensure_bucket_exists(self):
-        """Ensure the S3 bucket exists, creating it if necessary."""
-        try:
-            self.s3_client.head_bucket(Bucket=self.bucket_name)
-        except ClientError as e:
-            error_code = e.response["Error"]["Code"]
-            if error_code == "404":
-                # Bucket doesn't exist, create it
-                try:
-                    self.s3_client.create_bucket(
-                        Bucket=self.bucket_name,
-                        CreateBucketConfiguration={
-                            "LocationConstraint": self.s3_client.meta.region_name
-                        },
-                    ) if self.s3_client.meta.region_name != "us-east-1" else self.s3_client.create_bucket(
-                        Bucket=self.bucket_name
-                    )
-                except ClientError as create_error:
-                    if (
-                        create_error.response["Error"]["Code"]
-                        == "BucketAlreadyOwnedByYou"
-                    ):
-                        # Bucket was just created by another process
-                        judgeval_logger.warning(
-                            f"Bucket {self.bucket_name} was just created by another process"
-                        )
-                        pass
-                    else:
-                        raise create_error
-            else:
-                # Some other error occurred
-                raise e
-
-    def save_trace(self, trace_data: dict, trace_id: str, project_name: str) -> str:
-        """Save trace data to S3.
-
-        Args:
-            trace_data: The trace data to save
-            trace_id: Unique identifier for the trace
-            project_name: Name of the project the trace belongs to
-
-        Returns:
-            str: S3 key where the trace was saved
-        """
-        # Ensure bucket exists before saving
-        self._ensure_bucket_exists()
-
-        # Create a timestamped key for the trace
-        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
-        s3_key = f"traces/{project_name}/{trace_id}_{timestamp}.json"
-
-        # Convert trace data to JSON string
-        trace_json = json.dumps(trace_data)
-
-        self.s3_client.put_object(
-            Bucket=self.bucket_name,
-            Key=s3_key,
-            Body=trace_json,
-            ContentType="application/json",
-        )
-
-        return s3_key

judgeval/common/tracer/__init__.py

@@ -1,31 +0,0 @@
-from judgeval.common.tracer.core import (
-    TraceClient,
-    _DeepTracer,
-    Tracer,
-    wrap,
-    current_span_var,
-    current_trace_var,
-    SpanType,
-    cost_per_token,
-)
-from judgeval.common.tracer.otel_exporter import JudgmentAPISpanExporter
-from judgeval.common.tracer.otel_span_processor import JudgmentSpanProcessor
-from judgeval.common.tracer.span_processor import SpanProcessorBase
-from judgeval.common.tracer.trace_manager import TraceManagerClient
-from judgeval.data import TraceSpan
-
-__all__ = [
-    "_DeepTracer",
-    "TraceClient",
-    "Tracer",
-    "wrap",
-    "current_span_var",
-    "current_trace_var",
-    "TraceManagerClient",
-    "JudgmentAPISpanExporter",
-    "JudgmentSpanProcessor",
-    "SpanProcessorBase",
-    "SpanType",
-    "cost_per_token",
-    "TraceSpan",
-]

judgeval/common/tracer/constants.py

@@ -1,22 +0,0 @@
-import os
-import site
-import sysconfig
-
-
-# NOTE: This builds once, can be tweaked if we are missing / capturing other unncessary modules
-# @link https://docs.python.org/3.13/library/sysconfig.html
-_TRACE_FILEPATH_BLOCKLIST = tuple(
-    os.path.realpath(p) + os.sep
-    for p in {
-        sysconfig.get_paths()["stdlib"],
-        sysconfig.get_paths().get("platstdlib", ""),
-        *site.getsitepackages(),
-        site.getusersitepackages(),
-        *(
-            [os.path.join(os.path.dirname(__file__), "../../judgeval/")]
-            if os.environ.get("JUDGMENT_DEV")
-            else []
-        ),
-    }
-    if p
-)