judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/v1/scorers/api_scorer.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from judgeval.v1.internal.api.api_types import ScorerConfig
+from judgeval.v1.scorers.base_scorer import BaseScorer
+
+
+class APIScorer(BaseScorer):
+    __slots__ = (
+        "_score_type",
+        "_required_params",
+        "_threshold",
+        "_name",
+        "_strict_mode",
+        "_model",
+        "_additional_properties",
+    )
+
+    def __init__(
+        self,
+        score_type: str,
+        required_params: Optional[List[str]] = None,
+        threshold: float = 0.5,
+        name: Optional[str] = None,
+        strict_mode: bool = False,
+        model: Optional[str] = None,
+        **additional_properties: Any,
+    ):
+        self._score_type = score_type
+        self._required_params = required_params or []
+        self._threshold = threshold
+        self._name = name or score_type
+        self._strict_mode = strict_mode
+        self._model = model
+        self._additional_properties = additional_properties
+
+    def get_name(self) -> str:
+        return self._name
+
+    def get_score_type(self) -> str:
+        return self._score_type
+
+    def get_threshold(self) -> float:
+        return self._threshold
+
+    def get_strict_mode(self) -> bool:
+        return self._strict_mode
+
+    def get_model(self) -> Optional[str]:
+        return self._model
+
+    def get_required_params(self) -> List[str]:
+        return self._required_params.copy()
+
+    def set_threshold(self, threshold: float) -> None:
+        if threshold < 0 or threshold > 1:
+            raise ValueError(f"Threshold must be between 0 and 1, got: {threshold}")
+        self._threshold = threshold
+
+    def set_name(self, name: str) -> None:
+        self._name = name
+
+    def set_strict_mode(self, strict_mode: bool) -> None:
+        self._strict_mode = strict_mode
+
+    def set_model(self, model: str) -> None:
+        self._model = model
+
+    def get_scorer_config(self) -> ScorerConfig:
+        kwargs: Dict[str, Any] = dict(self._additional_properties)
+        if self._model:
+            kwargs["model"] = self._model
+
+        return ScorerConfig(
+            score_type=self._score_type,
+            threshold=self._threshold,
+            name=self._name,
+            strict_mode=self._strict_mode,
+            required_params=self._required_params,
+            kwargs=kwargs,
+        )

judgeval/v1/scorers/base_scorer.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from judgeval.v1.internal.api.api_types import ScorerConfig
+
+
+class BaseScorer(ABC):
+    __slots__ = ()
+
+    @abstractmethod
+    def get_name(self) -> str:
+        pass
+
+    @abstractmethod
+    def get_scorer_config(self) -> ScorerConfig:
+        pass

judgeval/v1/scorers/built_in/__init__.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from judgeval.v1.scorers.built_in.answer_correctness import AnswerCorrectnessScorer
+from judgeval.v1.scorers.built_in.answer_relevancy import AnswerRelevancyScorer
+from judgeval.v1.scorers.built_in.built_in_factory import BuiltInScorersFactory
+from judgeval.v1.scorers.built_in.faithfulness import FaithfulnessScorer
+from judgeval.v1.scorers.built_in.instruction_adherence import (
+    InstructionAdherenceScorer,
+)
+
+__all__ = [
+    "AnswerCorrectnessScorer",
+    "AnswerRelevancyScorer",
+    "FaithfulnessScorer",
+    "InstructionAdherenceScorer",
+    "BuiltInScorersFactory",
+]

judgeval/v1/scorers/built_in/answer_correctness.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.constants import APIScorerType
+from judgeval.v1.scorers.api_scorer import APIScorer
+
+
+class AnswerCorrectnessScorer(APIScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        name: Optional[str] = None,
+        strict_mode: bool = False,
+        model: Optional[str] = None,
+    ):
+        super().__init__(
+            score_type=APIScorerType.ANSWER_CORRECTNESS.value,
+            required_params=["input", "actual_output", "expected_output"],
+            threshold=threshold,
+            name=name,
+            strict_mode=strict_mode,
+            model=model,
+        )
+
+    @staticmethod
+    def create(threshold: float = 0.5) -> AnswerCorrectnessScorer:
+        return AnswerCorrectnessScorer(threshold=threshold)

judgeval/v1/scorers/built_in/answer_relevancy.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.constants import APIScorerType
+from judgeval.v1.scorers.api_scorer import APIScorer
+
+
+class AnswerRelevancyScorer(APIScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        name: Optional[str] = None,
+        strict_mode: bool = False,
+        model: Optional[str] = None,
+    ):
+        super().__init__(
+            score_type=APIScorerType.ANSWER_RELEVANCY.value,
+            required_params=["input", "actual_output"],
+            threshold=threshold,
+            name=name,
+            strict_mode=strict_mode,
+            model=model,
+        )
+
+    @staticmethod
+    def create(threshold: float = 0.5) -> AnswerRelevancyScorer:
+        return AnswerRelevancyScorer(threshold=threshold)

judgeval/v1/scorers/built_in/built_in_factory.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from judgeval.v1.scorers.built_in.answer_correctness import AnswerCorrectnessScorer
+from judgeval.v1.scorers.built_in.answer_relevancy import AnswerRelevancyScorer
+from judgeval.v1.scorers.built_in.faithfulness import FaithfulnessScorer
+from judgeval.v1.scorers.built_in.instruction_adherence import (
+    InstructionAdherenceScorer,
+)
+
+
+class BuiltInScorersFactory:
+    __slots__ = ()
+
+    def answer_correctness(self, threshold: float = 0.5) -> AnswerCorrectnessScorer:
+        return AnswerCorrectnessScorer.create(threshold)
+
+    def answer_relevancy(self, threshold: float = 0.5) -> AnswerRelevancyScorer:
+        return AnswerRelevancyScorer.create(threshold)
+
+    def faithfulness(self, threshold: float = 0.5) -> FaithfulnessScorer:
+        return FaithfulnessScorer.create(threshold)
+
+    def instruction_adherence(
+        self, threshold: float = 0.5
+    ) -> InstructionAdherenceScorer:
+        return InstructionAdherenceScorer.create(threshold)

judgeval/v1/scorers/built_in/faithfulness.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.constants import APIScorerType
+from judgeval.v1.scorers.api_scorer import APIScorer
+
+
+class FaithfulnessScorer(APIScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        name: Optional[str] = None,
+        strict_mode: bool = False,
+        model: Optional[str] = None,
+    ):
+        super().__init__(
+            score_type=APIScorerType.FAITHFULNESS.value,
+            required_params=["context", "actual_output"],
+            threshold=threshold,
+            name=name,
+            strict_mode=strict_mode,
+            model=model,
+        )
+
+    @staticmethod
+    def create(threshold: float = 0.5) -> FaithfulnessScorer:
+        return FaithfulnessScorer(threshold=threshold)

judgeval/v1/scorers/built_in/instruction_adherence.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.constants import APIScorerType
+from judgeval.v1.scorers.api_scorer import APIScorer
+
+
+class InstructionAdherenceScorer(APIScorer):
+    def __init__(
+        self,
+        threshold: float = 0.5,
+        name: Optional[str] = None,
+        strict_mode: bool = False,
+        model: Optional[str] = None,
+    ):
+        super().__init__(
+            score_type=APIScorerType.INSTRUCTION_ADHERENCE.value,
+            required_params=["input", "actual_output"],
+            threshold=threshold,
+            name=name,
+            strict_mode=strict_mode,
+            model=model,
+        )
+
+    @staticmethod
+    def create(threshold: float = 0.5) -> InstructionAdherenceScorer:
+        return InstructionAdherenceScorer(threshold=threshold)

judgeval/v1/scorers/custom_scorer/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.scorers.custom_scorer.custom_scorer import CustomScorer
+from judgeval.v1.scorers.custom_scorer.custom_scorer_factory import CustomScorerFactory
+
+__all__ = ["CustomScorer", "CustomScorerFactory"]

judgeval/v1/scorers/custom_scorer/custom_scorer.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from judgeval.constants import APIScorerType
+
+if TYPE_CHECKING:
+    from judgeval.v1.internal.api.api_types import (
+        BaseScorer as BaseScorerDict,
+        ScorerConfig,
+    )
+
+from judgeval.v1.scorers.base_scorer import BaseScorer
+
+
+class CustomScorer(BaseScorer):
+    __slots__ = (
+        "_name",
+        "_class_name",
+        "_server_hosted",
+    )
+
+    def __init__(
+        self,
+        name: str,
+        class_name: str = "",
+        server_hosted: bool = True,
+    ):
+        self._name = name
+        self._class_name = class_name or name
+        self._server_hosted = server_hosted
+
+    def get_name(self) -> str:
+        return self._name
+
+    def get_class_name(self) -> str:
+        return self._class_name
+
+    def is_server_hosted(self) -> bool:
+        return self._server_hosted
+
+    def get_scorer_config(self) -> ScorerConfig:
+        raise NotImplementedError("CustomScorer does not use get_scorer_config")
+
+    def to_dict(self) -> BaseScorerDict:
+        return {
+            "score_type": APIScorerType.CUSTOM.value,
+            "name": self._name,
+            "class_name": self._class_name,
+            "server_hosted": self._server_hosted,
+        }

judgeval/v1/scorers/custom_scorer/custom_scorer_factory.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from judgeval.v1.scorers.custom_scorer.custom_scorer import CustomScorer
+
+
+class CustomScorerFactory:
+    __slots__ = ()
+
+    def get(self, name: str, class_name: Optional[str] = None) -> CustomScorer:
+        return CustomScorer(
+            name=name,
+            class_name=class_name or name,
+            server_hosted=True,
+        )

judgeval/v1/scorers/prompt_scorer/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from judgeval.v1.scorers.prompt_scorer.prompt_scorer import PromptScorer
+from judgeval.v1.scorers.prompt_scorer.prompt_scorer_factory import PromptScorerFactory
+
+__all__ = ["PromptScorer", "PromptScorerFactory"]

judgeval/v1/scorers/prompt_scorer/prompt_scorer.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from judgeval.constants import APIScorerType
+from judgeval.v1.internal.api.api_types import ScorerConfig
+from judgeval.v1.scorers.api_scorer import APIScorer
+
+
+class PromptScorer(APIScorer):
+    __slots__ = (
+        "_prompt",
+        "_options",
+        "_description",
+        "_judgment_api_key",
+        "_organization_id",
+        "_is_trace",
+    )
+
+    def __init__(
+        self,
+        name: str,
+        prompt: str,
+        threshold: float = 0.5,
+        options: Optional[Dict[str, float]] = None,
+        model: Optional[str] = None,
+        description: Optional[str] = None,
+        judgment_api_key: str = "",
+        organization_id: str = "",
+        is_trace: bool = False,
+    ):
+        score_type = (
+            APIScorerType.TRACE_PROMPT_SCORER
+            if is_trace
+            else APIScorerType.PROMPT_SCORER
+        )
+        super().__init__(
+            score_type=score_type,
+            threshold=threshold,
+            name=name,
+            model=model,
+        )
+        self._prompt = prompt
+        self._options = options.copy() if options else None
+        self._description = description
+        self._judgment_api_key = judgment_api_key
+        self._organization_id = organization_id
+        self._is_trace = is_trace
+
+    def get_prompt(self) -> str:
+        return self._prompt
+
+    def get_options(self) -> Optional[Dict[str, float]]:
+        return self._options.copy() if self._options else None
+
+    def get_description(self) -> Optional[str]:
+        return self._description
+
+    def set_prompt(self, prompt: str) -> None:
+        self._prompt = prompt
+
+    def set_options(self, options: Dict[str, float]) -> None:
+        self._options = options.copy()
+
+    def set_description(self, description: str) -> None:
+        self._description = description
+
+    def append_to_prompt(self, addition: str) -> None:
+        self._prompt = self._prompt + addition
+
+    def get_scorer_config(self) -> ScorerConfig:
+        kwargs: Dict[str, Any] = {"prompt": self._prompt}
+
+        if self._options:
+            kwargs["options"] = self._options
+        if self._model:
+            kwargs["model"] = self._model
+        if self._description:
+            kwargs["description"] = self._description
+
+        return ScorerConfig(
+            score_type=self._score_type,
+            threshold=self._threshold,
+            name=self._name,
+            kwargs=kwargs,
+        )

judgeval/v1/scorers/prompt_scorer/prompt_scorer_factory.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Dict, Tuple
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+from judgeval.v1.internal.api.api_types import (
+    FetchPromptScorersRequest,
+    FetchPromptScorersResponse,
+    PromptScorer as APIPromptScorer,
+)
+from judgeval.exceptions import JudgmentAPIError
+from judgeval.v1.scorers.prompt_scorer.prompt_scorer import PromptScorer
+
+
+class PromptScorerFactory:
+    __slots__ = ("_client", "_is_trace")
+    _cache: Dict[Tuple[str, str, str, bool], APIPromptScorer] = {}
+
+    def __init__(
+        self,
+        client: JudgmentSyncClient,
+        is_trace: bool,
+    ):
+        self._client = client
+        self._is_trace = is_trace
+
+    def get(self, name: str) -> PromptScorer:
+        cache_key = (
+            name,
+            self._client.organization_id,
+            self._client.api_key,
+            self._is_trace,
+        )
+        cached = self._cache.get(cache_key)
+
+        if cached is None:
+            request: FetchPromptScorersRequest = {"names": [name]}
+            if self._is_trace is not None:
+                request["is_trace"] = self._is_trace
+
+            try:
+                response: FetchPromptScorersResponse = self._client.fetch_scorers(
+                    request
+                )
+                scorers = response.get("scorers", [])
+
+                if not scorers:
+                    raise JudgmentAPIError(
+                        404, f"Failed to fetch prompt scorer '{name}': not found", None
+                    )
+
+                scorer = scorers[0]
+                scorer_is_trace = scorer.get("is_trace", False)
+
+                if scorer_is_trace != self._is_trace:
+                    expected_type = (
+                        "TracePromptScorer" if self._is_trace else "PromptScorer"
+                    )
+                    actual_type = (
+                        "TracePromptScorer" if scorer_is_trace else "PromptScorer"
+                    )
+                    raise JudgmentAPIError(
+                        400,
+                        f"Scorer with name {name} is a {actual_type}, not a {expected_type}",
+                        None,
+                    )
+
+                self._cache[cache_key] = scorer
+                cached = scorer
+            except JudgmentAPIError:
+                raise
+            except Exception as e:
+                raise JudgmentAPIError(
+                    500, f"Failed to fetch prompt scorer '{name}': {e}", None
+                )
+
+        return PromptScorer(
+            name=name,
+            prompt=cached.get("prompt", ""),
+            threshold=cached.get("threshold", 0.5),
+            options=cached.get("options"),
+            model=cached.get("model"),
+            description=cached.get("description"),
+            is_trace=self._is_trace,
+        )

judgeval/v1/scorers/scorers_factory.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+
+
+class ScorersFactory:
+    __slots__ = "_client"
+
+    def __init__(
+        self,
+        client: JudgmentSyncClient,
+    ):
+        self._client = client
+
+    @property
+    def prompt_scorer(self):
+        from judgeval.v1.scorers.prompt_scorer.prompt_scorer_factory import (
+            PromptScorerFactory,
+        )
+
+        return PromptScorerFactory(
+            client=self._client,
+            is_trace=False,
+        )
+
+    @property
+    def trace_prompt_scorer(self):
+        from judgeval.v1.scorers.prompt_scorer.prompt_scorer_factory import (
+            PromptScorerFactory,
+        )
+
+        return PromptScorerFactory(
+            client=self._client,
+            is_trace=True,
+        )
+
+    @property
+    def custom_scorer(self):
+        from judgeval.v1.scorers.custom_scorer.custom_scorer_factory import (
+            CustomScorerFactory,
+        )
+
+        return CustomScorerFactory()
+
+    @property
+    def built_in(self):
+        from judgeval.v1.scorers.built_in.built_in_factory import BuiltInScorersFactory
+
+        return BuiltInScorersFactory()

judgeval/v1/tracer/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from judgeval.v1.tracer.base_tracer import BaseTracer
+from judgeval.v1.tracer.tracer import Tracer
+from judgeval.v1.tracer.tracer_factory import TracerFactory
+
+__all__ = ["BaseTracer", "Tracer", "TracerFactory"]