judgeval 0.3.2__tar.gz → 0.5.0__tar.gz

{judgeval-0.3.2 → judgeval-0.5.0}/.github/pull_request_template.md

@@ -10,14 +10,7 @@
 -->
 - [ ] 1. ...
 
-## 🎥 Demo of Changes
-
-<!-- Add a short 1-3 minute video describing/demoing the changes -->
-
 ## ✅ Checklist
 
-- [ ] Tagged Linear ticket in PR title. Ie. PR Title (JUD-XXXX)
-- [ ] Video demo of changes
-- [ ] Reviewers assigned
 - [ ] Docs updated ([if necessary](https://github.com/JudgmentLabs/docs))
-- [ ] Cookbooks updated ([if necessary](https://github.com/JudgmentLabs/judgment-cookbook))
+- [ ] Changelogs are updated ([if necessary](https://github.com/JudgmentLabs/docs/tree/main/content/docs/changelog/%28weekly%29))

{judgeval-0.3.2 → judgeval-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.3.2
+Version: 0.5.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -10,27 +10,24 @@ License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
-Requires-Dist: anthropic
 Requires-Dist: boto3
-Requires-Dist: datamodel-code-generator>=0.31.1
-Requires-Dist: google-genai
-Requires-Dist: groq>=0.30.0
 Requires-Dist: langchain-anthropic
 Requires-Dist: langchain-core
 Requires-Dist: langchain-huggingface
 Requires-Dist: langchain-openai
 Requires-Dist: litellm>=1.61.15
-Requires-Dist: matplotlib>=3.10.3
-Requires-Dist: nest-asyncio
-Requires-Dist: openai
+Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: opentelemetry-api>=1.34.1
 Requires-Dist: opentelemetry-sdk>=1.34.1
 Requires-Dist: orjson>=3.9.0
-Requires-Dist: pandas
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: python-dotenv
 Requires-Dist: requests
-Requires-Dist: together
+Requires-Dist: rich
+Provides-Extra: langchain
+Requires-Dist: langchain-anthropic; extra == 'langchain'
+Requires-Dist: langchain-core; extra == 'langchain'
+Requires-Dist: langchain-huggingface; extra == 'langchain'
+Requires-Dist: langchain-openai; extra == 'langchain'
 Description-Content-Type: text/markdown
 
 <div align="center">

{judgeval-0.3.2 → judgeval-0.5.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [project]
 name = "judgeval"
-version = "0.3.2"
+version = "0.5.0"
 authors = [
-    { name="Andrew Li", email="andrew@judgmentlabs.ai" },
-    { name="Alex Shan", email="alex@judgmentlabs.ai" },
-    { name="Joseph Camyre", email="joseph@judgmentlabs.ai" },
+    { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
+    { name = "Alex Shan", email = "alex@judgmentlabs.ai" },
+    { name = "Joseph Camyre", email = "joseph@judgmentlabs.ai" },
 ]
 description = "Judgeval Package"
 readme = "README.md"
@@ -16,27 +16,19 @@ classifiers = [
 license = "Apache-2.0"
 license-files = ["LICENSE.md"]
 dependencies = [
+    "rich",
     "litellm>=1.61.15",
-    "python-dotenv==1.0.1",
+    "python-dotenv",
     "requests",
-    "pandas",
-    "openai",
-    "together",
-    "anthropic",
-    "nest-asyncio",
-    "langchain-huggingface",
-    "langchain-openai",
-    "langchain-anthropic",
-    "langchain-core",
-    "google-genai",
     "boto3",
-    "matplotlib>=3.10.3",
-    "python-slugify>=8.0.4",
-    "datamodel-code-generator>=0.31.1",
-    "groq>=0.30.0",
     "opentelemetry-api>=1.34.1",
     "opentelemetry-sdk>=1.34.1",
     "orjson>=3.9.0",
+    "nest-asyncio>=1.6.0",
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
 ]
 
 [project.urls]
@@ -49,21 +41,24 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/judgeval"]
-include = [
-    "/src/judgeval",
-    "/src/judgeval/**/*.py",
+include = ["/src/judgeval", "/src/judgeval/**/*.py"]
+
+[project.optional-dependencies]
+langchain = [
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
 ]
 
 [dependency-groups]
 dev = [
     "chromadb>=1.0.12",
-    "langchain-community>=0.3.24",
     "pytest>=8.4.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.1.1",
     "pytest-mock>=3.14.1",
     "tavily-python>=0.7.5",
-    "langgraph>=0.4.3",
     "pre-commit>=4.2.0",
     "types-requests>=2.32.4.20250611",
     "mypy>=1.17.0",
@@ -90,18 +85,19 @@ dev = [
     "types-tqdm>=4.67.0.20250516",
     "types-tree-sitter-languages>=1.10.0.20250530",
     "types-xmltodict>=0.14.0.20241009",
+    "datamodel-code-generator>=0.31.2",
+    "openai",
+    "together",
+    "anthropic",
+    "google-genai",
+    "groq",
+    "langgraph>=0.4.3",
 ]
 
 [tool.hatch.build]
 directory = "dist"
-artifacts = [
-    "src/judgeval/**/*.py",
-]
-exclude = [
-    "src/e2etests/*",
-    "src/tests/*",
-    "src/demo/*"
-]
+artifacts = ["src/judgeval/**/*.py"]
+exclude = ["src/e2etests/*", "src/tests/*", "src/demo/*"]
 
 [tool.ruff]
 exclude = ["docs"]

{judgeval-0.3.2 → judgeval-0.5.0}/src/judgeval/__init__.py

@@ -2,6 +2,7 @@
 from judgeval.clients import client, together_client
 from judgeval.judgment_client import JudgmentClient
 from judgeval.version_check import check_latest_version
+from judgeval.local_eval_queue import LocalEvaluationQueue
 
 check_latest_version()
 
@@ -10,4 +11,5 @@ __all__ = [
     "client",
     "together_client",
     "JudgmentClient",
+    "LocalEvaluationQueue",
 ]

{judgeval-0.3.2 → judgeval-0.5.0}/src/judgeval/clients.py

@@ -2,7 +2,6 @@ import os
 from dotenv import load_dotenv
 from openai import OpenAI
 from typing import Optional
-from together import Together, AsyncTogether
 
 PATH_TO_DOTENV = os.path.join(os.path.dirname(__file__), ".env")
 load_dotenv(dotenv_path=PATH_TO_DOTENV)
@@ -28,6 +27,8 @@ async_together_client: Optional["AsyncTogether"] = None
 together_api_key = os.getenv("TOGETHERAI_API_KEY") or os.getenv("TOGETHER_API_KEY")
 if together_api_key:
     try:
+        from together import Together, AsyncTogether
+
         together_client = Together(api_key=together_api_key)
         async_together_client = AsyncTogether(api_key=together_api_key)
     except Exception:

{judgeval-0.3.2 → judgeval-0.5.0}/src/judgeval/common/api/api.py

@@ -1,4 +1,4 @@
-from typing import Literal, List, Dict, Any, Union
+from typing import Literal, List, Dict, Any, Union, Optional
 from requests import exceptions
 from judgeval.common.api.constants import (
     JUDGMENT_TRACES_FETCH_API_URL,
@@ -53,8 +53,7 @@ from judgeval.common.api.constants import (
     CheckExampleKeysPayload,
 )
 from judgeval.utils.requests import requests
-
-import orjson
+from judgeval.common.api.json_encoder import json_encoder
 
 
 class JudgmentAPIException(exceptions.HTTPError):
@@ -111,7 +110,7 @@ class JudgmentApiClient:
             r = requests.request(
                 method,
                 url,
-                data=self._serialize(payload),
+                json=json_encoder(payload),
                 headers=self._headers(),
                 **self._request_kwargs(),
             )
@@ -238,7 +237,7 @@ class JudgmentApiClient:
         }
         return self._do_request("POST", JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL, payload)
 
-    def save_scorer(self, name: str, prompt: str, options: dict):
+    def save_scorer(self, name: str, prompt: str, options: Optional[dict] = None):
         payload: ScorerSavePayload = {
             "name": name,
             "prompt": prompt,
@@ -368,16 +367,3 @@ class JudgmentApiClient:
             "verify": True,
             "timeout": 30,
         }
-
-    def _serialize(self, data: Any) -> str:
-        def fallback_encoder(obj):
-            try:
-                return repr(obj)
-            except Exception:
-                try:
-                    return str(obj)
-                except Exception as e:
-                    return f"<Unserializable object of type {type(obj).__name__}: {e}>"
-
-        # orjson returns bytes, so we need to decode to str
-        return orjson.dumps(data, default=fallback_encoder).decode("utf-8")

{judgeval-0.3.2 → judgeval-0.5.0}/src/judgeval/common/api/constants.py

@@ -162,7 +162,7 @@ JUDGMENT_SCORER_EXISTS_API_URL = f"{ROOT_API}/scorer_exists/"
 class ScorerSavePayload(TypedDict):
     name: str
     prompt: str
-    options: dict
+    options: Optional[dict]
 
 
 class ScorerFetchPayload(TypedDict):

judgeval-0.5.0/src/judgeval/common/api/json_encoder.py

@@ -0,0 +1,242 @@
+"""
+
+This is a modified version of https://docs.powertools.aws.dev/lambda/python/2.35.1/api/event_handler/openapi/encoders.html
+
+"""
+
+import dataclasses
+import datetime
+from collections import defaultdict, deque
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path, PurePath
+from re import Pattern
+from types import GeneratorType
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union
+from uuid import UUID
+
+from pydantic import BaseModel
+from pydantic.types import SecretBytes, SecretStr
+
+
+"""
+This module contains the encoders used by jsonable_encoder to convert Python objects to JSON serializable data types.
+"""
+
+
+def _model_dump(
+    model: BaseModel, mode: Literal["json", "python"] = "json", **kwargs: Any
+) -> Any:
+    return model.model_dump(mode=mode, **kwargs)
+
+
+def json_encoder(
+    obj: Any,
+    custom_serializer: Optional[Callable[[Any], str]] = None,
+) -> Any:
+    """
+    JSON encodes an arbitrary Python object into JSON serializable data types.
+
+    This is a modified version of fastapi.encoders.jsonable_encoder that supports
+    encoding of pydantic.BaseModel objects.
+
+    Parameters
+    ----------
+    obj : Any
+        The object to encode
+    custom_serializer : Callable, optional
+        A custom serializer to use for encoding the object, when everything else fails.
+
+    Returns
+    -------
+    Any
+        The JSON serializable data types
+    """
+    # Pydantic models
+    if isinstance(obj, BaseModel):
+        return _dump_base_model(
+            obj=obj,
+        )
+
+    # Dataclasses
+    if dataclasses.is_dataclass(obj):
+        obj_dict = dataclasses.asdict(obj)
+        return json_encoder(
+            obj_dict,
+        )
+
+    # Enums
+    if isinstance(obj, Enum):
+        return obj.value
+
+    # Paths
+    if isinstance(obj, PurePath):
+        return str(obj)
+
+    # Scalars
+    if isinstance(obj, (str, int, float, type(None))):
+        return obj
+
+    # Dictionaries
+    if isinstance(obj, dict):
+        return _dump_dict(
+            obj=obj,
+        )
+
+    # Sequences
+    if isinstance(obj, (list, set, frozenset, GeneratorType, tuple, deque)):
+        return _dump_sequence(
+            obj=obj,
+        )
+
+    # Other types
+    if type(obj) in ENCODERS_BY_TYPE:
+        return ENCODERS_BY_TYPE[type(obj)](obj)
+
+    for encoder, classes_tuple in encoders_by_class_tuples.items():
+        if isinstance(obj, classes_tuple):
+            return encoder(obj)
+
+    # Use custom serializer if present
+    if custom_serializer:
+        return custom_serializer(obj)
+
+    # Default
+    return _dump_other(
+        obj=obj,
+    )
+
+
+def _dump_base_model(
+    *,
+    obj: Any,
+):
+    """
+    Dump a BaseModel object to a dict, using the same parameters as jsonable_encoder
+    """
+    obj_dict = _model_dump(
+        obj,
+        mode="json",
+    )
+    if "__root__" in obj_dict:
+        obj_dict = obj_dict["__root__"]
+
+    return json_encoder(
+        obj_dict,
+    )
+
+
+def _dump_dict(
+    *,
+    obj: Any,
+) -> Dict[str, Any]:
+    """
+    Dump a dict to a dict, using the same parameters as jsonable_encoder
+    """
+    encoded_dict = {}
+    allowed_keys = set(obj.keys())
+    for key, value in obj.items():
+        if key in allowed_keys:
+            encoded_key = json_encoder(
+                key,
+            )
+            encoded_value = json_encoder(
+                value,
+            )
+            encoded_dict[encoded_key] = encoded_value
+    return encoded_dict
+
+
+def _dump_sequence(
+    *,
+    obj: Any,
+) -> List[Any]:
+    """
+    Dump a sequence to a list, using the same parameters as jsonable_encoder
+    """
+    encoded_list = []
+    for item in obj:
+        encoded_list.append(
+            json_encoder(
+                item,
+            ),
+        )
+    return encoded_list
+
+
+def _dump_other(
+    *,
+    obj: Any,
+) -> Any:
+    """
+    Dump an object to a hashable object, using the same parameters as jsonable_encoder
+    """
+    try:
+        data = dict(obj)
+    except Exception:
+        return repr(obj)
+
+    return json_encoder(
+        data,
+    )
+
+
+def iso_format(o: Union[datetime.date, datetime.time]) -> str:
+    """
+    ISO format for date and time
+    """
+    return o.isoformat()
+
+
+def decimal_encoder(dec_value: Decimal) -> Union[int, float]:
+    """
+    Encodes a Decimal as int of there's no exponent, otherwise float
+
+    This is useful when we use ConstrainedDecimal to represent Numeric(x,0)
+    where an integer (but not int typed) is used. Encoding this as a float
+    results in failed round-tripping between encode and parse.
+
+    >>> decimal_encoder(Decimal("1.0"))
+    1.0
+
+    >>> decimal_encoder(Decimal("1"))
+    1
+    """
+    if dec_value.as_tuple().exponent >= 0:  # type: ignore[operator]
+        return int(dec_value)
+    else:
+        return float(dec_value)
+
+
+ENCODERS_BY_TYPE: Dict[Type[Any], Callable[[Any], Any]] = {
+    bytes: lambda o: o.decode(),
+    datetime.date: iso_format,
+    datetime.datetime: iso_format,
+    datetime.time: iso_format,
+    datetime.timedelta: lambda td: td.total_seconds(),
+    Decimal: decimal_encoder,
+    Enum: lambda o: o.value,
+    frozenset: list,
+    deque: list,
+    GeneratorType: list,
+    Path: str,
+    Pattern: lambda o: o.pattern,
+    SecretBytes: str,
+    SecretStr: str,
+    set: list,
+    UUID: str,
+}
+
+
+# Generates a mapping of encoders to a tuple of classes that they can encode
+def generate_encoders_by_class_tuples(
+    type_encoder_map: Dict[Any, Callable[[Any], Any]],
+) -> Dict[Callable[[Any], Any], Tuple[Any, ...]]:
+    encoders: Dict[Callable[[Any], Any], Tuple[Any, ...]] = defaultdict(tuple)
+    for type_, encoder in type_encoder_map.items():
+        encoders[encoder] += (type_,)
+    return encoders
+
+
+# Mapping of encoders to a tuple of classes that they can encode
+encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)