judgeval 0.11.0__tar.gz → 0.12.0__tar.gz

{judgeval-0.11.0 → judgeval-0.12.0}/.github/workflows/release.yaml

@@ -91,7 +91,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       
-      - name: Bump pyproject.toml version
+      - name: Bump pyproject.toml and version.py version
         run: |
           python update_version.py ${{ steps.bump_tag.outputs.new_version }}
 

{judgeval-0.11.0 → judgeval-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.11.0
+Version: 0.12.0
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.11.0 → judgeval-0.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.11.0"
+version = "0.12.0"
 authors = [
     { name = "Andrew Li", email = "andrew@judgmentlabs.ai" },
     { name = "Alex Shan", email = "alex@judgmentlabs.ai" },

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/__init__.py

@@ -5,8 +5,8 @@ from judgeval.evaluation import run_eval
 from judgeval.data.evaluation_run import ExampleEvaluationRun
 
 
-from typing import List, Optional, Union
-from judgeval.scorers import APIScorerConfig
+from typing import List, Optional, Union, Sequence
+from judgeval.scorers import ExampleAPIScorerConfig
 from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.data.example import Example
 from judgeval.logger import judgeval_logger
@@ -39,7 +39,7 @@ class JudgmentClient(metaclass=SingletonMeta):
     def run_evaluation(
         self,
         examples: List[Example],
-        scorers: List[Union[APIScorerConfig, ExampleScorer]],
+        scorers: Sequence[Union[ExampleAPIScorerConfig, ExampleScorer]],
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/data/evaluation_run.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Union, Tuple
+from typing import List, Optional, Union, Tuple, Sequence
 from pydantic import field_validator, model_validator, Field, BaseModel
 from datetime import datetime, timezone
 import uuid
@@ -19,8 +19,10 @@ class EvaluationRun(BaseModel):
         default_factory=lambda: datetime.now(timezone.utc).isoformat()
     )
     custom_scorers: List[ExampleScorer] = Field(default_factory=list)
-    judgment_scorers: List[APIScorerConfig] = Field(default_factory=list)
-    scorers: List[Union[ExampleScorer, APIScorerConfig]] = Field(default_factory=list)
+    judgment_scorers: Sequence[APIScorerConfig] = Field(default_factory=list)
+    scorers: Sequence[Union[ExampleScorer, APIScorerConfig]] = Field(
+        default_factory=list
+    )
     model: str
 
     def __init__(

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/__init__.py

@@ -1,7 +1,10 @@
 from judgeval.scorers.api_scorer import (
     APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
 )
 from judgeval.scorers.base_scorer import BaseScorer
+from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.scorers.judgeval_scorers.api_scorers import (
     FaithfulnessScorer,
     AnswerRelevancyScorer,
@@ -13,7 +16,10 @@ from judgeval.scorers.judgeval_scorers.api_scorers import (
 
 __all__ = [
     "APIScorerConfig",
+    "ExampleAPIScorerConfig",
+    "TraceAPIScorerConfig",
     "BaseScorer",
+    "ExampleScorer",
     "TracePromptScorer",
     "PromptScorer",
     "FaithfulnessScorer",

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/api_scorer.py

@@ -63,3 +63,11 @@ class APIScorerConfig(BaseModel):
 
     def __str__(self):
         return f"JudgmentScorer(score_type={self.score_type.value}, threshold={self.threshold})"
+
+
+class ExampleAPIScorerConfig(APIScorerConfig):
+    pass
+
+
+class TraceAPIScorerConfig(APIScorerConfig):
+    pass

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py

@@ -6,13 +6,13 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class AnswerCorrectnessScorer(APIScorerConfig):
+class AnswerCorrectnessScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.ANSWER_CORRECTNESS
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py

@@ -1,10 +1,10 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class AnswerRelevancyScorer(APIScorerConfig):
+class AnswerRelevancyScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.ANSWER_RELEVANCY
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py

@@ -6,13 +6,13 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 from typing import List
 
 
-class FaithfulnessScorer(APIScorerConfig):
+class FaithfulnessScorer(ExampleAPIScorerConfig):
     score_type: APIScorerType = APIScorerType.FAITHFULNESS
     required_params: List[ExampleParams] = [
         ExampleParams.INPUT,

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py

@@ -6,12 +6,12 @@ TODO add link to docs page for this scorer
 """
 
 # Internal imports
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import ExampleAPIScorerConfig
 from judgeval.constants import APIScorerType
 from judgeval.data import ExampleParams
 
 
-class InstructionAdherenceScorer(APIScorerConfig):
+class InstructionAdherenceScorer(ExampleAPIScorerConfig):
     def __init__(self, threshold: float):
         super().__init__(
             threshold=threshold,

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py

@@ -1,5 +1,7 @@
 from judgeval.scorers.api_scorer import (
     APIScorerConfig,
+    ExampleAPIScorerConfig,
+    TraceAPIScorerConfig,
 )
 from judgeval.constants import APIScorerType
 from typing import Dict, Any, Optional
@@ -282,9 +284,9 @@ class BasePromptScorer(ABC, APIScorerConfig):
         return base
 
 
-class PromptScorer(BasePromptScorer, APIScorerConfig):
+class PromptScorer(BasePromptScorer, ExampleAPIScorerConfig):
     pass
 
 
-class TracePromptScorer(BasePromptScorer, APIScorerConfig):
+class TracePromptScorer(BasePromptScorer, TraceAPIScorerConfig):
     pass

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/tracer/__init__.py

@@ -43,7 +43,7 @@ from judgeval.env import (
     JUDGMENT_ORG_ID,
 )
 from judgeval.logger import judgeval_logger
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import TraceAPIScorerConfig, ExampleAPIScorerConfig
 from judgeval.scorers.example_scorer import ExampleScorer
 from judgeval.tracer.constants import JUDGEVAL_TRACER_INSTRUMENTING_MODULE_NAME
 from judgeval.tracer.managers import (
@@ -328,7 +328,7 @@ class Tracer:
         run_condition = scorer_config.run_condition
         sampling_rate = scorer_config.sampling_rate
 
-        if not isinstance(scorer, (APIScorerConfig)):
+        if not isinstance(scorer, (TraceAPIScorerConfig)):
             judgeval_logger.error(
                 "Scorer must be an instance of TraceAPIScorerConfig, got %s, skipping evaluation."
                 % type(scorer)
@@ -861,7 +861,7 @@ class Tracer:
         self,
         /,
         *,
-        scorer: Union[APIScorerConfig, ExampleScorer],
+        scorer: Union[ExampleAPIScorerConfig, ExampleScorer],
         example: Example,
         model: str = JUDGMENT_DEFAULT_GPT_MODEL,
         sampling_rate: float = 1.0,
@@ -870,9 +870,9 @@ class Tracer:
             judgeval_logger.info("Evaluation is not enabled, skipping evaluation")
             return
 
-        if not isinstance(scorer, (APIScorerConfig, ExampleScorer)):
+        if not isinstance(scorer, (ExampleAPIScorerConfig, ExampleScorer)):
             judgeval_logger.error(
-                "Scorer must be an instance of ExampleAPIScorerConfig or BaseScorer, got %s, skipping evaluation."
+                "Scorer must be an instance of ExampleAPIScorerConfig or ExampleScorer, got %s, skipping evaluation."
                 % type(scorer)
             )
             return
@@ -901,7 +901,7 @@ class Tracer:
         span_context = self.get_current_span().get_span_context()
         trace_id = format(span_context.trace_id, "032x")
         span_id = format(span_context.span_id, "016x")
-        hosted_scoring = isinstance(scorer, APIScorerConfig) or (
+        hosted_scoring = isinstance(scorer, ExampleAPIScorerConfig) or (
             isinstance(scorer, ExampleScorer) and scorer.server_hosted
         )
         eval_run_name = f"async_evaluate_{span_id}"  # note this name doesnt matter because we don't save the experiment only the example and scorer_data

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/tracer/utils.py

@@ -2,7 +2,7 @@ from typing import Any
 from opentelemetry.trace import Span
 from pydantic import BaseModel
 from typing import Callable, Optional
-from judgeval.scorers.api_scorer import APIScorerConfig
+from judgeval.scorers.api_scorer import TraceAPIScorerConfig
 from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 
 
@@ -14,7 +14,7 @@ def set_span_attribute(span: Span, name: str, value: Any):
 
 
 class TraceScorerConfig(BaseModel):
-    scorer: APIScorerConfig
+    scorer: TraceAPIScorerConfig
     model: str = JUDGMENT_DEFAULT_GPT_MODEL
     sampling_rate: float = 1.0
     run_condition: Optional[Callable[..., bool]] = None

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/trainer/trainer.py

@@ -10,7 +10,7 @@ from judgeval.tracer.exporters.store import SpanStore
 from judgeval.tracer.exporters import InMemorySpanExporter
 from judgeval.tracer.keys import AttributeKeys
 from judgeval import JudgmentClient
-from judgeval.scorers import BaseScorer, APIScorerConfig
+from judgeval.scorers import ExampleScorer, ExampleAPIScorerConfig
 from judgeval.data import Example
 from .console import _spinner_progress, _print_progress, _print_progress_update
 from judgeval.exceptions import JudgmentRuntimeError
@@ -156,7 +156,7 @@ class JudgmentTrainer:
     async def generate_rollouts_and_rewards(
         self,
         agent_function: Callable[[Any], Any],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        scorers: List[Union[ExampleAPIScorerConfig, ExampleScorer]],
         prompts: List[Any],
         num_prompts_per_step: Optional[int] = None,
         num_generations_per_prompt: Optional[int] = None,
@@ -266,7 +266,7 @@ class JudgmentTrainer:
     async def run_reinforcement_learning(
         self,
         agent_function: Callable[[Any], Any],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        scorers: List[Union[ExampleAPIScorerConfig, ExampleScorer]],
         prompts: List[Any],
     ) -> ModelConfig:
         """
@@ -372,7 +372,7 @@ class JudgmentTrainer:
     async def train(
         self,
         agent_function: Callable[[Any], Any],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
+        scorers: List[Union[ExampleAPIScorerConfig, ExampleScorer]],
         prompts: List[Any],
         rft_provider: Optional[str] = None,
     ) -> ModelConfig:

{judgeval-0.11.0 → judgeval-0.12.0}/src/judgeval/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.0"
+__version__ = "0.12.0"
 
 
 def get_version() -> str:

judgeval-0.12.0/update_version.py

@@ -0,0 +1,35 @@
+import sys
+
+if len(sys.argv) != 2:
+    print("Usage: python set_version.py <new_version>")
+    sys.exit(1)
+
+new_version = sys.argv[1]
+version_placeholder = "0.0.0"
+
+
+def update_file(file_path, placeholder, new_val):
+    try:
+        with open(file_path, "r") as f:
+            content = f.read()
+    except IOError as e:
+        print(f"Error: Failed to read '{file_path}': {e}")
+        sys.exit(1)
+
+    if placeholder not in content:
+        print(f"Warning: No version placeholder '{placeholder}' found in '{file_path}'")
+        sys.exit(1)
+
+    new_content = content.replace(placeholder, new_val, 1)
+
+    try:
+        with open(file_path, "w") as f:
+            f.write(new_content)
+    except IOError as e:
+        print(f"Error: Failed to write to '{file_path}': {e}")
+        sys.exit(1)
+
+
+files_to_update = ["pyproject.toml", "src/judgeval/version.py"]
+for file in files_to_update:
+    update_file(file, version_placeholder, new_version)

judgeval-0.11.0/update_version.py

@@ -1,32 +0,0 @@
-import sys
-
-if len(sys.argv) != 2:
-    print("Usage: python set_version.py <new_version>")
-    sys.exit(1)
-
-new_version = sys.argv[1]
-version_placeholder = "0.0.0"
-found = False
-
-try:
-    with open("pyproject.toml", "r") as f:
-        lines = f.readlines()
-except IOError as e:
-    print(f"Error: Failed to read 'pyproject.toml': {e}")
-    sys.exit(1)
-
-try:
-    with open("pyproject.toml", "w") as f:
-        for line in lines:  # Assumes 'lines' was successfully read earlier
-            if not found and version_placeholder in line:
-                f.write(line.replace(version_placeholder, new_version))
-                found = True
-            else:
-                f.write(line)
-except IOError as e:
-    print(f"Error: Failed to write to 'pyproject.toml': {e}")
-    sys.exit(1)
-
-if not found:
-    print("Warning: No '<version_placeholder>' found in pyproject.toml")
-    sys.exit(1)