judgeval 0.4.0__tar.gz → 0.6.0__tar.gz

{judgeval-0.4.0 → judgeval-0.6.0}/.github/pull_request_template.md

@@ -10,14 +10,7 @@
 -->
 - [ ] 1. ...
 
-## 🎥 Demo of Changes
-
-<!-- Add a short 1-3 minute video describing/demoing the changes -->
-
 ## ✅ Checklist
 
-- [ ] Tagged Linear ticket in PR title. Ie. PR Title (JUD-XXXX)
-- [ ] Video demo of changes
-- [ ] Reviewers assigned
 - [ ] Docs updated ([if necessary](https://github.com/JudgmentLabs/docs))
-- [ ] Cookbooks updated ([if necessary](https://github.com/JudgmentLabs/judgment-cookbook))
+- [ ] Changelogs are updated ([if necessary](https://github.com/JudgmentLabs/docs/tree/main/content/docs/changelog/%28weekly%29))

{judgeval-0.4.0 → judgeval-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.4.0
+Version: 0.6.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -10,27 +10,26 @@ License-File: LICENSE.md
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
-Requires-Dist: anthropic
 Requires-Dist: boto3
-Requires-Dist: datamodel-code-generator>=0.31.1
-Requires-Dist: google-genai
-Requires-Dist: groq>=0.30.0
+Requires-Dist: click<8.2.0
 Requires-Dist: langchain-anthropic
 Requires-Dist: langchain-core
 Requires-Dist: langchain-huggingface
 Requires-Dist: langchain-openai
 Requires-Dist: litellm>=1.61.15
-Requires-Dist: matplotlib>=3.10.3
-Requires-Dist: nest-asyncio
-Requires-Dist: openai
+Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: opentelemetry-api>=1.34.1
 Requires-Dist: opentelemetry-sdk>=1.34.1
 Requires-Dist: orjson>=3.9.0
-Requires-Dist: pandas
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: python-dotenv
 Requires-Dist: requests
-Requires-Dist: together
+Requires-Dist: rich
+Requires-Dist: typer>=0.9.0
+Provides-Extra: langchain
+Requires-Dist: langchain-anthropic; extra == 'langchain'
+Requires-Dist: langchain-core; extra == 'langchain'
+Requires-Dist: langchain-huggingface; extra == 'langchain'
+Requires-Dist: langchain-openai; extra == 'langchain'
 Description-Content-Type: text/markdown
 
 <div align="center">

{judgeval-0.4.0 → judgeval-0.6.0}/pyproject.toml

@@ -1,10 +1,10 @@
 [project]
 name = "judgeval"
-version = "0.4.0"
+version = "0.6.0"
 authors = [
-    { name="Andrew Li", email="andrew@judgmentlabs.ai" },
-    { name="Alex Shan", email="alex@judgmentlabs.ai" },
-    { name="Joseph Camyre", email="joseph@judgmentlabs.ai" },
+    { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
+    { name = "Alex Shan", email = "alex@judgmentlabs.ai" },
+    { name = "Joseph Camyre", email = "joseph@judgmentlabs.ai" },
 ]
 description = "Judgeval Package"
 readme = "README.md"
@@ -16,54 +16,54 @@ classifiers = [
 license = "Apache-2.0"
 license-files = ["LICENSE.md"]
 dependencies = [
+    "rich",
     "litellm>=1.61.15",
-    "python-dotenv==1.0.1",
+    "python-dotenv",
     "requests",
-    "pandas",
-    "openai",
-    "together",
-    "anthropic",
-    "nest-asyncio",
-    "langchain-huggingface",
-    "langchain-openai",
-    "langchain-anthropic",
-    "langchain-core",
-    "google-genai",
     "boto3",
-    "matplotlib>=3.10.3",
-    "python-slugify>=8.0.4",
-    "datamodel-code-generator>=0.31.1",
-    "groq>=0.30.0",
     "opentelemetry-api>=1.34.1",
     "opentelemetry-sdk>=1.34.1",
     "orjson>=3.9.0",
+    "nest-asyncio>=1.6.0",
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
+    "click<8.2.0",
+    "typer>=0.9.0",
 ]
 
 [project.urls]
 Homepage = "https://github.com/JudgmentLabs/judgeval"
 Issues = "https://github.com/JudgmentLabs/judgeval/issues"
 
+[project.scripts]
+judgeval = "judgeval.cli:app"
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/judgeval"]
-include = [
-    "/src/judgeval",
-    "/src/judgeval/**/*.py",
+include = ["/src/judgeval", "/src/judgeval/**/*.py"]
+
+[project.optional-dependencies]
+langchain = [
+    "langchain-huggingface",
+    "langchain-openai",
+    "langchain-anthropic",
+    "langchain-core",
 ]
 
 [dependency-groups]
 dev = [
     "chromadb>=1.0.12",
-    "langchain-community>=0.3.24",
     "pytest>=8.4.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.1.1",
     "pytest-mock>=3.14.1",
     "tavily-python>=0.7.5",
-    "langgraph>=0.4.3",
     "pre-commit>=4.2.0",
     "types-requests>=2.32.4.20250611",
     "mypy>=1.17.0",
@@ -90,18 +90,19 @@ dev = [
     "types-tqdm>=4.67.0.20250516",
     "types-tree-sitter-languages>=1.10.0.20250530",
     "types-xmltodict>=0.14.0.20241009",
+    "datamodel-code-generator>=0.31.2",
+    "openai",
+    "together",
+    "anthropic",
+    "google-genai",
+    "groq",
+    "langgraph>=0.4.3",
 ]
 
 [tool.hatch.build]
 directory = "dist"
-artifacts = [
-    "src/judgeval/**/*.py",
-]
-exclude = [
-    "src/e2etests/*",
-    "src/tests/*",
-    "src/demo/*"
-]
+artifacts = ["src/judgeval/**/*.py"]
+exclude = ["src/e2etests/*", "src/tests/*", "src/demo/*"]
 
 [tool.ruff]
 exclude = ["docs"]

{judgeval-0.4.0 → judgeval-0.6.0}/src/judgeval/__init__.py

@@ -2,6 +2,7 @@
 from judgeval.clients import client, together_client
 from judgeval.judgment_client import JudgmentClient
 from judgeval.version_check import check_latest_version
+from judgeval.local_eval_queue import LocalEvaluationQueue
 
 check_latest_version()
 
@@ -10,4 +11,5 @@ __all__ = [
     "client",
     "together_client",
     "JudgmentClient",
+    "LocalEvaluationQueue",
 ]

judgeval-0.6.0/src/judgeval/cli.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+import typer
+from pathlib import Path
+from dotenv import load_dotenv
+from judgeval.common.logger import judgeval_logger
+from judgeval.judgment_client import JudgmentClient
+
+load_dotenv()
+
+app = typer.Typer(
+    no_args_is_help=True,
+    rich_markup_mode=None,
+    rich_help_panel=None,
+    pretty_exceptions_enable=False,
+    pretty_exceptions_show_locals=False,
+    pretty_exceptions_short=False,
+)
+
+
+@app.command("upload_scorer")
+def upload_scorer(
+    scorer_file_path: str,
+    requirements_file_path: str,
+    unique_name: str = typer.Option(
+        None, help="Custom name for the scorer (auto-detected if not provided)"
+    ),
+):
+    # Validate file paths
+    if not Path(scorer_file_path).exists():
+        judgeval_logger.error(f"Scorer file not found: {scorer_file_path}")
+        raise typer.Exit(1)
+
+    if not Path(requirements_file_path).exists():
+        judgeval_logger.error(f"Requirements file not found: {requirements_file_path}")
+        raise typer.Exit(1)
+
+    try:
+        client = JudgmentClient()
+
+        result = client.save_custom_scorer(
+            scorer_file_path=scorer_file_path,
+            requirements_file_path=requirements_file_path,
+            unique_name=unique_name,
+        )
+
+        if not result:
+            judgeval_logger.error("Failed to upload custom scorer")
+            raise typer.Exit(1)
+
+        raise typer.Exit(0)
+    except Exception:
+        raise
+
+
+@app.command()
+def version():
+    """Show version info"""
+    judgeval_logger.info("JudgEval CLI v0.0.0")
+
+
+if __name__ == "__main__":
+    app()
+
+# judgeval upload_scorer /Users/alanzhang/repo/JudgmentLabs/judgeval/src/demo/profile_match_scorer.py /Users/alanzhang/repo/JudgmentLabs/judgeval/src/demo/requirements.txt

{judgeval-0.4.0 → judgeval-0.6.0}/src/judgeval/clients.py

@@ -2,7 +2,6 @@ import os
 from dotenv import load_dotenv
 from openai import OpenAI
 from typing import Optional
-from together import Together, AsyncTogether
 
 PATH_TO_DOTENV = os.path.join(os.path.dirname(__file__), ".env")
 load_dotenv(dotenv_path=PATH_TO_DOTENV)
@@ -28,6 +27,8 @@ async_together_client: Optional["AsyncTogether"] = None
 together_api_key = os.getenv("TOGETHERAI_API_KEY") or os.getenv("TOGETHER_API_KEY")
 if together_api_key:
     try:
+        from together import Together, AsyncTogether
+
         together_client = Together(api_key=together_api_key)
         async_together_client = AsyncTogether(api_key=together_api_key)
     except Exception:

{judgeval-0.4.0 → judgeval-0.6.0}/src/judgeval/common/api/api.py

@@ -20,13 +20,11 @@ from judgeval.common.api.constants import (
     JUDGMENT_EVAL_DELETE_API_URL,
     JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL,
     JUDGMENT_GET_EVAL_STATUS_API_URL,
-    JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL,
-    JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL,
     JUDGMENT_SCORER_SAVE_API_URL,
     JUDGMENT_SCORER_FETCH_API_URL,
     JUDGMENT_SCORER_EXISTS_API_URL,
+    JUDGMENT_CUSTOM_SCORER_UPLOAD_API_URL,
     JUDGMENT_DATASETS_APPEND_TRACES_API_URL,
-    JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL,
 )
 from judgeval.common.api.constants import (
     TraceFetchPayload,
@@ -45,16 +43,14 @@ from judgeval.common.api.constants import (
     DeleteEvalRunRequestBody,
     EvalLogPayload,
     EvalStatusPayload,
-    CheckExperimentTypePayload,
-    EvalRunNameExistsPayload,
     ScorerSavePayload,
     ScorerFetchPayload,
     ScorerExistsPayload,
-    CheckExampleKeysPayload,
+    CustomScorerUploadPayload,
+    CustomScorerTemplateResponse,
 )
 from judgeval.utils.requests import requests
-
-import orjson
+from judgeval.common.api.json_encoder import json_encoder
 
 
 class JudgmentAPIException(exceptions.HTTPError):
@@ -98,22 +94,28 @@ class JudgmentApiClient:
         method: Literal["POST", "PATCH", "GET", "DELETE"],
         url: str,
         payload: Any,
+        timeout: Optional[Union[float, tuple]] = None,
     ) -> Any:
+        # Prepare request kwargs with optional timeout
+        request_kwargs = self._request_kwargs()
+        if timeout is not None:
+            request_kwargs["timeout"] = timeout
+
         if method == "GET":
             r = requests.request(
                 method,
                 url,
                 params=payload,
                 headers=self._headers(),
-                **self._request_kwargs(),
+                **request_kwargs,
             )
         else:
             r = requests.request(
                 method,
                 url,
-                data=self._serialize(payload),
+                json=json_encoder(payload),
                 headers=self._headers(),
-                **self._request_kwargs(),
+                **request_kwargs,
             )
 
         try:
@@ -187,10 +189,10 @@ class JudgmentApiClient:
         payload: EvalLogPayload = {"results": results, "run": run}
         return self._do_request("POST", JUDGMENT_EVAL_LOG_API_URL, payload)
 
-    def fetch_evaluation_results(self, project_name: str, eval_name: str):
+    def fetch_evaluation_results(self, experiment_run_id: str, project_name: str):
         payload: EvalRunRequestBody = {
             "project_name": project_name,
-            "eval_name": eval_name,
+            "experiment_run_id": experiment_run_id,
         }
         return self._do_request("POST", JUDGMENT_EVAL_FETCH_API_URL, payload)
 
@@ -205,43 +207,21 @@ class JudgmentApiClient:
     def add_to_evaluation_queue(self, payload: Dict[str, Any]):
         return self._do_request("POST", JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, payload)
 
-    def get_evaluation_status(self, eval_name: str, project_name: str):
+    def get_evaluation_status(self, experiment_run_id: str, project_name: str):
         payload: EvalStatusPayload = {
-            "eval_name": eval_name,
+            "experiment_run_id": experiment_run_id,
             "project_name": project_name,
             "judgment_api_key": self.api_key,
         }
         return self._do_request("GET", JUDGMENT_GET_EVAL_STATUS_API_URL, payload)
 
-    def check_experiment_type(self, eval_name: str, project_name: str, is_trace: bool):
-        payload: CheckExperimentTypePayload = {
-            "eval_name": eval_name,
-            "project_name": project_name,
-            "judgment_api_key": self.api_key,
-            "is_trace": is_trace,
-        }
-        return self._do_request("POST", JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL, payload)
-
-    def check_eval_run_name_exists(self, eval_name: str, project_name: str):
-        payload: EvalRunNameExistsPayload = {
-            "eval_name": eval_name,
-            "project_name": project_name,
-            "judgment_api_key": self.api_key,
-        }
-        return self._do_request("POST", JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL, payload)
-
-    def check_example_keys(self, keys: List[str], eval_name: str, project_name: str):
-        payload: CheckExampleKeysPayload = {
-            "keys": keys,
-            "eval_name": eval_name,
-            "project_name": project_name,
-        }
-        return self._do_request("POST", JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL, payload)
-
-    def save_scorer(self, name: str, prompt: str, options: Optional[dict] = None):
+    def save_scorer(
+        self, name: str, prompt: str, threshold: float, options: Optional[dict] = None
+    ):
         payload: ScorerSavePayload = {
             "name": name,
             "prompt": prompt,
+            "threshold": threshold,
             "options": options,
         }
         try:
@@ -293,6 +273,31 @@ class JudgmentApiClient:
                 request=e.request,
             )
 
+    def upload_custom_scorer(
+        self,
+        scorer_name: str,
+        scorer_code: str,
+        requirements_text: str,
+    ) -> CustomScorerTemplateResponse:
+        """Upload custom scorer to backend"""
+        payload: CustomScorerUploadPayload = {
+            "scorer_name": scorer_name,
+            "scorer_code": scorer_code,
+            "requirements_text": requirements_text,
+        }
+
+        try:
+            # Use longer timeout for custom scorer upload (5 minutes)
+            response = self._do_request(
+                "POST",
+                JUDGMENT_CUSTOM_SCORER_UPLOAD_API_URL,
+                payload,
+                timeout=(10, 300),
+            )
+            return response
+        except JudgmentAPIException as e:
+            raise e
+
     def push_dataset(
         self,
         dataset_alias: str,
@@ -368,16 +373,3 @@ class JudgmentApiClient:
             "verify": True,
             "timeout": 30,
         }
-
-    def _serialize(self, data: Any) -> str:
-        def fallback_encoder(obj):
-            try:
-                return repr(obj)
-            except Exception:
-                try:
-                    return str(obj)
-                except Exception as e:
-                    return f"<Unserializable object of type {type(obj).__name__}: {e}>"
-
-        # orjson returns bytes, so we need to decode to str
-        return orjson.dumps(data, default=fallback_encoder).decode("utf-8")

{judgeval-0.4.0 → judgeval-0.6.0}/src/judgeval/common/api/constants.py

@@ -49,9 +49,9 @@ JUDGMENT_EVAL_DELETE_API_URL = (
 JUDGMENT_EVAL_DELETE_PROJECT_API_URL = f"{ROOT_API}/delete_eval_results_by_project/"
 JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL = f"{ROOT_API}/add_to_run_eval_queue/"
 JUDGMENT_GET_EVAL_STATUS_API_URL = f"{ROOT_API}/get_evaluation_status/"
-JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL = f"{ROOT_API}/check_experiment_type/"
-JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL = f"{ROOT_API}/eval-run-name-exists/"
-JUDGMENT_CHECK_EXAMPLE_KEYS_API_URL = f"{ROOT_API}/check_example_keys/"
+
+# Custom Scorers API
+JUDGMENT_CUSTOM_SCORER_UPLOAD_API_URL = f"{ROOT_API}/build_sandbox_template/"
 
 
 # Evaluation API Payloads
@@ -73,9 +73,9 @@ class EvalLogPayload(TypedDict):
 
 
 class EvalStatusPayload(TypedDict):
-    eval_name: str
-    project_name: str
+    experiment_run_id: str
     judgment_api_key: str
+    project_name: str
 
 
 class CheckExperimentTypePayload(TypedDict):
@@ -162,6 +162,7 @@ JUDGMENT_SCORER_EXISTS_API_URL = f"{ROOT_API}/scorer_exists/"
 class ScorerSavePayload(TypedDict):
     name: str
     prompt: str
+    threshold: float
     options: Optional[dict]
 
 
@@ -171,3 +172,15 @@ class ScorerFetchPayload(TypedDict):
 
 class ScorerExistsPayload(TypedDict):
     name: str
+
+
+class CustomScorerUploadPayload(TypedDict):
+    scorer_name: str
+    scorer_code: str
+    requirements_text: str
+
+
+class CustomScorerTemplateResponse(TypedDict):
+    scorer_name: str
+    status: str
+    message: str