judgeval 0.0.54__py3-none-any.whl → 0.1.0__py3-none-any.whl

judgeval/scorers/judgeval_scorers/api_scorers/classifier_scorer.py

@@ -1,73 +0,0 @@
-from judgeval.scorers.api_scorer import APIScorerConfig
-from judgeval.constants import APIScorerType
-from typing import List, Mapping, Optional, Dict, Any
-
-
-class ClassifierScorer(APIScorerConfig):
-    """
-    In the Judgment backend, this scorer is implemented as a PromptScorer that takes
-    1. a system role that may involve the Example object
-    2. options for scores on the example
-
-    and uses a judge to execute the evaluation from the system role and classify into one of the options
-
-    ex:
-    system_role = "You are a judge that evaluates whether the response is positive or negative. The response is: {example.actual_output}"
-    options = {"positive": 1, "negative": 0}
-
-    Args:
-        slug (str): A unique identifier for the scorer
-        conversation (List[dict]): The conversation template with placeholders (e.g., {{actual_output}})
-        options (Mapping[str, float]): A mapping of classification options to their corresponding scores
-    """
-
-    slug: Optional[str] = None
-    conversation: Optional[List[dict]] = None
-    options: Optional[Mapping[str, float]] = None
-    score_type: APIScorerType = APIScorerType.PROMPT_SCORER
-
-    def update_name(self, name: str):
-        """
-        Updates the name of the scorer.
-        """
-        self.name = name
-
-    def update_threshold(self, threshold: float):
-        """
-        Updates the threshold of the scorer.
-        """
-        self.threshold = threshold
-
-    def update_conversation(self, conversation: List[dict]):
-        """
-        Updates the conversation with the new conversation.
-
-        Sample conversation:
-        [{'role': 'system', 'content': "Did the chatbot answer the user's question in a kind way?: {{actual_output}}."}]
-        """
-        self.conversation = conversation
-
-    def update_options(self, options: Mapping[str, float]):
-        """
-        Updates the options with the new options.
-
-        Sample options:
-        {"yes": 1, "no": 0}
-        """
-        self.options = options
-
-    def __str__(self):
-        return f"ClassifierScorer(name={self.name}, slug={self.slug}, conversation={self.conversation}, threshold={self.threshold}, options={self.options})"
-
-    def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
-        base = super().model_dump(*args, **kwargs)
-        base_fields = set(APIScorerConfig.model_fields.keys())
-        all_fields = set(self.__class__.model_fields.keys())
-
-        extra_fields = all_fields - base_fields - {"kwargs"}
-
-        base["kwargs"] = {
-            k: getattr(self, k) for k in extra_fields if getattr(self, k) is not None
-        }
-
-        return base

judgeval/scorers/judgeval_scorers/classifiers/__init__.py

@@ -1,3 +0,0 @@
-from .text2sql import Text2SQLScorer
-
-__all__ = ["Text2SQLScorer"]

judgeval/scorers/judgeval_scorers/classifiers/text2sql/__init__.py

@@ -1,3 +0,0 @@
-from .text2sql_scorer import Text2SQLScorer
-
-__all__ = ["Text2SQLScorer"]

judgeval/scorers/judgeval_scorers/classifiers/text2sql/text2sql_scorer.py

@@ -1,53 +0,0 @@
-"""
-ClassifierScorer implementation for basic Text-to-SQL evaluation.
-
-Takes a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
-Determines if the LLM-generated SQL query is valid and works for the natural language query.
-"""
-
-from judgeval.scorers import ClassifierScorer
-
-Text2SQLScorer = ClassifierScorer(
-    name="Text to SQL",
-    slug="text2sql-1010101010",
-    threshold=1.0,
-    conversation=[
-        {
-            "role": "system",
-            "content": """You will be given a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
-
-** TASK INSTRUCTIONS **
-Your task is to decide whether the LLM generated SQL query properly filters for what the natural language query is asking, based on the table schema + (optional) metadata. 
-Additionally, you should check if the SQL query is valid based on the table schema (checking for syntax errors, false column names, etc.)
-
-** TIPS **
-- Look for correct references to the table schema for column names, table names, etc.
-- Check that the SQL query can be executed; make sure JOINs, GROUP BYs, ORDER BYs, etc. are valid with respect to the table schema.
-- Check that aggregation functions (COUNT, SUM, AVG, etc.) are used appropriately with GROUP BY clauses
-- Verify that WHERE conditions use the correct operators and data types for comparisons
-- Ensure LIMIT and OFFSET clauses make sense for the query's purpose
-- Check that JOINs use the correct keys and maintain referential integrity
-- Verify that ORDER BY clauses use valid column names and sort directions
-- Check for proper handling of NULL values where relevant
-- Ensure subqueries are properly constructed and correlated when needed
-- EVEN IF THE QUERY IS VALID, IF IT DOESN'T WORK FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "N" AS THE ANSWER.
-
-** FORMATTING YOUR ANSWER **
-If the SQL query is valid and works for the natural language query, choose option "Y" and otherwise "N". Provide a justification for your decision; if you choose "N", explain what about the LLM-generated SQL query is incorrect, or explain why it doesn't address the natural language query. 
-IF YOUR JUSTIFICATION SHOWS THAT THE SQL QUERY IS VALID AND WORKS FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "Y" AS THE ANSWER. 
-IF THE SQL QUERY IS INVALID, YOU SHOULD CHOOSE "N" AS THE ANSWER. 
-
-** YOUR TURN **
-Natural language query:
-{{input}}
-
-LLM generated SQL query:
-{{actual_output}}
-
-Table schema:
-{{context}}
-        """,
-        }
-    ],
-    options={"Y": 1.0, "N": 0.0},
-)