judgeval 0.0.8__tar.gz → 0.0.9__tar.gz

{judgeval-0.0.8 → judgeval-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.8
+Version: 0.0.9
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.0.8 → judgeval-0.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.8"
+version = "0.0.9"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

judgeval-0.0.9/src/demo/cookbooks/custom_scorers/competitor_mentions.py

@@ -0,0 +1,58 @@
+"""
+This script implements a custom scorer to evaluate customer support responses.
+
+It checks if a support response mentions competitors (like Adidas, Reebok, etc.) in a positive way.
+The scorer penalizes responses that promote competitor products, helping maintain brand focus in
+customer interactions. This would be useful to score customer support responses for Nike, for example.
+"""
+
+
+from judgeval import JudgmentClient
+from judgeval.scorers import ClassifierScorer
+from judgeval.data import Example
+
+
+competitor_mentions_scorer = ClassifierScorer(
+    "Competitor Mentions",
+    slug="competitor_mentions-487126418",
+    threshold=1.0,
+    conversation=[{
+        "role": "system",
+        "content": """Does the following customer support response discuss any of the following competitors in a positive way? (Y/N). 
+        
+        Competitors: Adidas, Reebok, Hoka, ON, Converse
+        
+        Customer Question: {{input}}
+        Customer Support Response: {{actual_output}}
+        """
+    }],
+    options={
+        "Y": 0.0, 
+        "N": 1.0
+    }
+)
+
+
+if __name__ == "__main__":
+    client = JudgmentClient()
+
+    positive_example = Example(
+        input="What are the best shoes for running priced under $130?",
+        actual_output="You'd want to check out the newest Nike Vaporfly, it's only $120 and built for performance. "
+    )
+
+    negative_example = Example(
+        input="What are the best shoes for running priced under $130?",
+        actual_output="The Nike Vaporfly is a great shoe built for performance. Other great options include the Adidas Ultraboost and the Reebok Nano X which are affordable and speedy."
+    )
+
+    client.run_evaluation(
+        examples=[positive_example, negative_example],
+        scorers=[competitor_mentions_scorer],
+        model="gpt-4o-mini",
+        project_name="competitor_mentions",
+        eval_run_name="competitor_mentions_test",
+    )
+
+
+

judgeval-0.0.9/src/demo/cookbooks/custom_scorers/text2sql.py

@@ -0,0 +1,205 @@
+"""
+This script is a cookbook of how to create a custom scorer using a ClassifierScorer.
+
+Simply use a natural language prompt and guide the LLM to output a score based on the input by 
+choosing from a set of options.
+"""
+
+from judgeval import JudgmentClient
+from judgeval.scorers import ClassifierScorer
+from judgeval.data import Example
+
+text2sql_scorer = ClassifierScorer(
+    "Text to SQL",
+    slug="text2sql-487126418",
+    threshold=1.0,
+    conversation=[{
+        "role": "system",
+        "content": """You will be given a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
+
+** TASK INSTRUCTIONS **
+Your task is to decide whether the LLM generated SQL query properly filters for what the natural language query is asking, based on the table schema + (optional) metadata. 
+Additionally, you should check if the SQL query is valid based on the table schema (checking for syntax errors, false column names, etc.)
+
+** TIPS **
+- Look for correct references to the table schema for column names, table names, etc.
+- Check that the SQL query can be executed; make sure JOINs, GROUP BYs, ORDER BYs, etc. are valid with respect to the table schema.
+- Check that aggregation functions (COUNT, SUM, AVG, etc.) are used appropriately with GROUP BY clauses
+- Verify that WHERE conditions use the correct operators and data types for comparisons
+- Ensure LIMIT and OFFSET clauses make sense for the query's purpose
+- Check that JOINs use the correct keys and maintain referential integrity
+- Verify that ORDER BY clauses use valid column names and sort directions
+- Check for proper handling of NULL values where relevant
+- Ensure subqueries are properly constructed and correlated when needed
+- EVEN IF THE QUERY IS VALID, IF IT DOESN'T WORK FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "N" AS THE ANSWER.
+
+** FORMATTING YOUR ANSWER **
+If the SQL query is valid and works for the natural language query, choose option "Y" and otherwise "N". Provide a justification for your decision; if you choose "N", explain what about the LLM-generated SQL query is incorrect, or explain why it doesn't address the natural language query. 
+IF YOUR JUSTIFICATION SHOWS THAT THE SQL QUERY IS VALID AND WORKS FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "Y" AS THE ANSWER. 
+IF THE SQL QUERY IS INVALID, YOU SHOULD CHOOSE "N" AS THE ANSWER. 
+
+** YOUR TURN **
+Natural language query:
+{{input}}
+
+LLM generated SQL query:
+{{actual_output}}
+
+Table schema:
+{{context}}
+        """
+    }],
+    options={
+        "Y": 1.0, 
+        "N": 0.0
+    }
+)
+
+
+if __name__ == "__main__":
+    client = JudgmentClient()
+
+    table_schema = """CREATE TABLE Artists (
+    artist_id VARCHAR(50) PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    genre VARCHAR(100),
+    followers INT,
+    popularity INT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE Albums (
+    album_id VARCHAR(50) PRIMARY KEY,
+    title VARCHAR(255) NOT NULL,
+    artist_id VARCHAR(50) NOT NULL,
+    release_date DATE,
+    total_tracks INT,
+    album_type VARCHAR(50) CHECK (album_type IN ('album', 'single', 'compilation')),
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (artist_id) REFERENCES Artists(artist_id) ON DELETE CASCADE
+);
+
+CREATE TABLE Tracks (
+    track_id VARCHAR(50) PRIMARY KEY,
+    title VARCHAR(255) NOT NULL,
+    album_id VARCHAR(50) NOT NULL,
+    artist_id VARCHAR(50) NOT NULL,
+    duration_ms INT NOT NULL,
+    explicit BOOLEAN DEFAULT FALSE,
+    popularity INT DEFAULT 0,
+    preview_url VARCHAR(255),
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (album_id) REFERENCES Albums(album_id) ON DELETE CASCADE,
+    FOREIGN KEY (artist_id) REFERENCES Artists(artist_id) ON DELETE CASCADE
+);
+
+CREATE TABLE Users (
+    user_id VARCHAR(50) PRIMARY KEY,
+    username VARCHAR(100) NOT NULL UNIQUE,
+    email VARCHAR(255) NOT NULL UNIQUE,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE Playlists (
+    playlist_id VARCHAR(50) PRIMARY KEY,
+    user_id VARCHAR(50) NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (user_id) REFERENCES Users(user_id) ON DELETE CASCADE
+);
+
+CREATE TABLE PlaylistTracks (
+    playlist_id VARCHAR(50),
+    track_id VARCHAR(50),
+    added_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (playlist_id, track_id),
+    FOREIGN KEY (playlist_id) REFERENCES Playlists(playlist_id) ON DELETE CASCADE,
+    FOREIGN KEY (track_id) REFERENCES Tracks(track_id) ON DELETE CASCADE
+);
+
+CREATE TABLE UserListeningHistory (
+    history_id SERIAL PRIMARY KEY,
+    user_id VARCHAR(50) NOT NULL,
+    track_id VARCHAR(50) NOT NULL,
+    listened_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (user_id) REFERENCES Users(user_id) ON DELETE CASCADE,
+    FOREIGN KEY (track_id) REFERENCES Tracks(track_id) ON DELETE CASCADE
+);
+"""
+
+    all_tracks_one_artist_correct = Example(
+        input="Find all tracks by the artist 'Drake', sorted by popularity.",
+        actual_output="""SELECT t.track_id, t.title, t.popularity, a.name AS artist_name
+FROM Tracks t
+JOIN Artists a ON t.artist_id = a.artist_id
+WHERE a.name = 'Drake'
+ORDER BY t.popularity DESC;
+""",
+        retrieval_context=[table_schema]
+    )
+
+    most_listened_to_one_user_correct = Example(
+        input="Find the most listened to track by user 'user123'.",
+        actual_output="""SELECT t.track_id, t.title, COUNT(uh.history_id) AS play_count
+FROM UserListeningHistory uh
+JOIN Tracks t ON uh.track_id = t.track_id
+WHERE uh.user_id = 'user123'
+GROUP BY t.track_id, t.title
+ORDER BY play_count DESC
+LIMIT 1;
+""",
+        retrieval_context=[table_schema]
+    )
+
+    highest_num_playlists_correct = Example(
+        input="Find the 5 users with the highest number of playlists.",
+        actual_output="""SELECT u.user_id, u.username, COUNT(p.playlist_id) AS total_playlists
+FROM Users u
+JOIN Playlists p ON u.user_id = p.user_id
+GROUP BY u.user_id, u.username
+ORDER BY total_playlists DESC
+LIMIT 5;
+""",
+        retrieval_context=[table_schema]
+    )
+
+    most_popular_tracks_all_users_correct = Example(
+        input="Find the 10 most popular tracks across all users.",
+        actual_output="""SELECT t.track_id, t.title, COUNT(uh.history_id) AS total_listens
+FROM Tracks t
+JOIN UserListeningHistory uh ON t.track_id = uh.track_id
+GROUP BY t.track_id, t.title
+ORDER BY total_listens DESC
+LIMIT 10;
+""",
+        retrieval_context=[table_schema]
+    )
+
+    most_popular_tracks_all_users_incorrect = Example(
+        input="Find the 10 most popular tracks across all users.",
+        actual_output="""SELECT t.track_user, t.title, COUNT(uh.history_id) AS total_listens
+FROM Tracks t
+JOIN UserHistory uh ON t.track_user = uh.track_user
+GROUP BY t.track_user, t.title
+ORDER BY total_listens DESC
+LIMIT 10;
+""",
+        retrieval_context=[table_schema]
+    )
+
+
+    client.run_evaluation(
+        examples=[
+            all_tracks_one_artist_correct, 
+            most_listened_to_one_user_correct, 
+            highest_num_playlists_correct, 
+            most_popular_tracks_all_users_correct, 
+            most_popular_tracks_all_users_incorrect
+        ],
+        scorers=[text2sql_scorer],
+        model="gpt-4o-mini",
+        project_name="text2sql",
+        eval_run_name="text2sql_test",
+        override=True
+    )

{judgeval-0.0.8 → judgeval-0.0.9}/src/judgeval/scorers/__init__.py

@@ -13,6 +13,7 @@ from judgeval.scorers.judgeval_scorers import (
     AnswerRelevancyScorer,
     ScorerWrapper,
     AnswerCorrectnessScorer,
+    Text2SQLScorer,
 )
 
 __all__ = [
@@ -31,4 +32,5 @@ __all__ = [
     "AnswerRelevancyScorer",
     "ScorerWrapper",
     "AnswerCorrectnessScorer",
+    "Text2SQLScorer",
 ]

{judgeval-0.0.8 → judgeval-0.0.9}/src/judgeval/scorers/judgeval_scorers/__init__.py

@@ -28,6 +28,9 @@ from judgeval.scorers.judgeval_scorers.local_implementations import (
     AnswerCorrectnessScorer as LocalAnswerCorrectnessScorer
 )
 
+from judgeval.scorers.judgeval_scorers.classifiers import Text2SQLScorer
+
+
 class ScorerWrapper:
     """
     Wrapper class that can dynamically load either API or local implementation of a scorer.
@@ -141,4 +144,5 @@ __all__ = [
     "ContextualPrecisionScorer",
     "ContextualRecallScorer",
     "AnswerRelevancyScorer",
+    "Text2SQLScorer",
 ]

judgeval-0.0.9/src/judgeval/scorers/judgeval_scorers/classifiers/__init__.py

@@ -0,0 +1,3 @@
+from .text2sql import Text2SQLScorer
+
+__all__ = ["Text2SQLScorer"]

judgeval-0.0.9/src/judgeval/scorers/judgeval_scorers/classifiers/text2sql/__init__.py

@@ -0,0 +1,3 @@
+from .text2sql_scorer import Text2SQLScorer
+
+__all__ = ["Text2SQLScorer"]

judgeval-0.0.9/src/judgeval/scorers/judgeval_scorers/classifiers/text2sql/text2sql_scorer.py

@@ -0,0 +1,54 @@
+"""
+ClassifierScorer implementation for basic Text-to-SQL evaluation.
+
+Takes a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
+Determines if the LLM-generated SQL query is valid and works for the natural language query.
+"""
+from judgeval.scorers import ClassifierScorer
+
+Text2SQLScorer = ClassifierScorer(
+    "Text to SQL",
+    slug="text2sql-1010101010",
+    threshold=1.0,
+    conversation=[{
+        "role": "system",
+        "content": """You will be given a natural language query, a corresponding LLM-generated SQL query, and a table schema + (optional) metadata.
+
+** TASK INSTRUCTIONS **
+Your task is to decide whether the LLM generated SQL query properly filters for what the natural language query is asking, based on the table schema + (optional) metadata. 
+Additionally, you should check if the SQL query is valid based on the table schema (checking for syntax errors, false column names, etc.)
+
+** TIPS **
+- Look for correct references to the table schema for column names, table names, etc.
+- Check that the SQL query can be executed; make sure JOINs, GROUP BYs, ORDER BYs, etc. are valid with respect to the table schema.
+- Check that aggregation functions (COUNT, SUM, AVG, etc.) are used appropriately with GROUP BY clauses
+- Verify that WHERE conditions use the correct operators and data types for comparisons
+- Ensure LIMIT and OFFSET clauses make sense for the query's purpose
+- Check that JOINs use the correct keys and maintain referential integrity
+- Verify that ORDER BY clauses use valid column names and sort directions
+- Check for proper handling of NULL values where relevant
+- Ensure subqueries are properly constructed and correlated when needed
+- EVEN IF THE QUERY IS VALID, IF IT DOESN'T WORK FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "N" AS THE ANSWER.
+
+** FORMATTING YOUR ANSWER **
+If the SQL query is valid and works for the natural language query, choose option "Y" and otherwise "N". Provide a justification for your decision; if you choose "N", explain what about the LLM-generated SQL query is incorrect, or explain why it doesn't address the natural language query. 
+IF YOUR JUSTIFICATION SHOWS THAT THE SQL QUERY IS VALID AND WORKS FOR THE NATURAL LANGUAGE QUERY, YOU SHOULD CHOOSE "Y" AS THE ANSWER. 
+IF THE SQL QUERY IS INVALID, YOU SHOULD CHOOSE "N" AS THE ANSWER. 
+
+** YOUR TURN **
+Natural language query:
+{{input}}
+
+LLM generated SQL query:
+{{actual_output}}
+
+Table schema:
+{{context}}
+        """
+    }],
+    options={
+        "Y": 1.0, 
+        "N": 0.0
+    }
+)
+