judgeval 0.0.33__tar.gz → 0.0.35__tar.gz

judgeval-0.0.35/.github/pull_request_template.md

@@ -0,0 +1,31 @@
+## 📝 Summary
+
+<!-- Provide a brief description of the changes introduced by this PR -->
+
+## 🎯 Purpose
+
+<!-- Explain the motivation behind these changes. Why are they necessary? -->
+
+## 🎥 Demo of Changes
+
+<!-- Add a short 1-3 minute video describing/demoing the changes -->
+
+## 🧪 Testing
+
+<!-- Describe how the changes were tested (unit/manual) -->
+
+## ✅ Checklist
+
+- [ ] Self-review
+- [ ] Video demo of changes
+- [ ] Unit Tests and CI/CD tests are passing
+- [ ] Reviewers assigned
+
+
+## 📌 Linear Issue
+
+<!-- Reference to associated Linear ticket, e.g., ABC-123 -->
+
+## ✏️ Additional Notes
+
+<!-- Any additional information that doesn't fit into the other sections -->

{judgeval-0.0.33 → judgeval-0.0.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.33
+Version: 0.0.35
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
@@ -11,6 +11,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.11
 Requires-Dist: anthropic
+Requires-Dist: boto3==1.38.3
 Requires-Dist: fastapi
 Requires-Dist: google-genai
 Requires-Dist: langchain

{judgeval-0.0.33 → judgeval-0.0.35}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.33"
+version = "0.0.35"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },
@@ -34,7 +34,8 @@ dependencies = [
     "langchain-openai",
     "langchain-anthropic",
     "langchain-core",
-    "google-genai"
+    "google-genai",
+    "boto3==1.38.3"
 ]
 
 [project.optional-dependencies]

judgeval-0.0.35/src/demo/demo.py

@@ -0,0 +1,54 @@
+from judgeval import JudgmentClient
+from judgeval.data import Example, Sequence
+from judgeval.scorers import DerailmentScorer
+
+client = JudgmentClient()
+
+dataset = client.pull_dataset("test-dataset", "default_project")
+
+print(dataset)
+
+# airlines_example = Example(
+#     input="Which airlines fly to Tokyo?",
+#     actual_output="Japan Airlines, All Nippon Airways, and Chinese Airlines offer direct flights."
+# )
+# weather_example = Example(
+#     input="What is the weather like in Japan?",
+#     actual_output="It's cloudy with a high of 75°F and a low of 60°F in Japan."
+# )
+# airline_sequence = Sequence(
+#     name="Flight Details",
+#     items=[airlines_example, weather_example],
+# )
+
+# # Level 1: Top-level sequence
+# top_example1 = Example(
+#     input="I want to plan a trip to Tokyok.",
+#     actual_output="That sounds great! When are you planning to go?"
+# )
+# top_example2 = Example(
+#     input="Can you book a flight for me and anything else I need to know?",
+#     actual_output="Sure, I'll help you with flights. hotels. and transportation."
+# )
+# top_level_sequence = Sequence(
+#     name="Travel Planning",
+#     items=[top_example1, top_example2, airline_sequence],
+# )
+
+# other_sequence = Sequence(
+#     name="Other",
+#     items=[Example(
+#         input="What is the weather like in Tokyo?",
+#         actual_output="It's cloudy with a high of 75°F and a low of 60°F in Tokyo."
+#     )]
+# )
+
+# results = client.run_sequence_evaluation(
+#     eval_run_name="sequence-run1",
+#     project_name="jnpr-demo-sequence",
+#     scorers=[DerailmentScorer(threshold=1)],
+#     sequences=[top_level_sequence, other_sequence],
+#     model="gpt-4o",
+#     log_results=True,
+#     override=True,
+# )

judgeval-0.0.35/src/demo/demo2.py

@@ -0,0 +1,144 @@
+import os
+import asyncio
+from openai import OpenAI, AsyncOpenAI
+from dotenv import load_dotenv
+from judgeval.common.tracer import Tracer, wrap
+from judgeval.scorers import AnswerRelevancyScorer, FaithfulnessScorer, GroundednessScorer
+
+# Load environment variables
+load_dotenv()
+
+# Initialize OpenAI client and Judgment tracer
+client = wrap(OpenAI())
+async_client = wrap(AsyncOpenAI())
+judgment = Tracer(project_name="music-bot-demo")
+
+@judgment.observe(span_type="tool")
+async def search_tavily(query):
+    """Search for information using Tavily."""
+    from tavily import TavilyClient
+    
+    tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY"))
+    search_result = tavily_client.search(
+        query=query,
+        search_depth="advanced",
+        max_results=5
+    )
+    
+    return search_result
+
+@judgment.observe(span_type="function")
+async def ask_user_preferences():
+    """Ask the user a series of questions about their music preferences."""
+    questions = [
+        "What are some of your favorite artists or bands?",
+        "What genres of music do you enjoy the most?",
+        "Do you have any favorite songs currently?",
+        "Are there any moods or themes you're looking for in new music?",
+        "Do you prefer newer releases or classic songs?"
+    ]
+    
+    preferences = {}
+    for question in questions:
+        print(f"\n{question}")
+        answer = input("> ")
+        preferences[question] = answer
+    
+    return preferences
+
+@judgment.observe(span_type="function")
+async def search_music_recommendations(preferences):
+    """Search for music recommendations based on user preferences."""
+    # Construct search queries based on preferences
+    search_results = {}
+    
+    # Search for artist recommendations
+    if preferences.get("What are some of your favorite artists or bands?"):
+        artists_query = f"Music similar to {preferences['What are some of your favorite artists or bands?']}"
+        search_results["artist_based"] = await search_tavily(artists_query)
+    
+    # Search for genre recommendations
+    if preferences.get("What genres of music do you enjoy the most?"):
+        genre_query = f"Best {preferences['What genres of music do you enjoy the most?']} songs"
+        search_results["genre_based"] = await search_tavily(genre_query)
+    
+    # Search for mood-based recommendations
+    if preferences.get("Are there any moods or themes you're looking for in new music?"):
+        mood_query = f"""{preferences["Are there any moods or themes you're looking for in new music?"]} music recommendations"""
+        search_results["mood_based"] = await search_tavily(mood_query)
+    
+    return search_results
+
+@judgment.observe(span_type="function")
+async def generate_recommendations(preferences, search_results):
+    """Generate personalized music recommendations using the search results."""
+    # Prepare context from search results
+    context = ""
+    for category, results in search_results.items():
+        context += f"\n{category.replace('_', ' ').title()} Search Results:\n"
+        for result in results.get("results", []):
+            context += f"- {result.get('title')}: {result.get('content')[:200]}...\n"
+    
+    # Create a prompt for the LLM
+    prompt = f"""
+    Suggest 5-7 songs they could enjoy. Be creative and suggest whatever feels right. You should only recommend songs that are from the user's favorite artists/bands.
+    For each song, include the artist name, song title, and a brief explanation of why they might like it.
+    
+    User Preferences:
+    {preferences}
+    
+    Search Results:
+    {context}
+    
+    Provide recommendations in a clear, organized format. Focus on specific songs rather than just artists.
+    """
+
+    
+    # Generate recommendations using OpenAI
+    response = await async_client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a music recommendation expert with deep knowledge of various genres, artists, and songs. Your goal is to suggest songs that match the user's preferences; recommend songs from their favorite artists/bands."},
+            {"role": "user", "content": prompt}
+        ]
+    )
+    
+    recommendations = response.choices[0].message.content
+    
+    # Evaluate the recommendations
+    judgment.get_current_trace().async_evaluate(
+        scorers=[
+            AnswerRelevancyScorer(threshold=1.0),
+            GroundednessScorer(threshold=1.0)
+        ],
+        input=prompt,
+        actual_output=recommendations,
+        retrieval_context=[str(search_results)],
+        model="gpt-4o"
+    )
+    
+    return recommendations
+
+@judgment.observe(span_type="Main Function")
+async def music_recommendation_bot():
+    """Main function to run the music recommendation bot."""
+    print("🎵 Welcome to the Music Recommendation Bot! 🎵")
+    print("I'll ask you a few questions to understand your music taste, then suggest some songs you might enjoy.")
+    
+    # Get user preferences
+    preferences = await ask_user_preferences()
+    
+    print("\nSearching for music recommendations based on your preferences...")
+    search_results = await search_music_recommendations(preferences)
+    
+    print("\nGenerating personalized recommendations...")
+    recommendations = await generate_recommendations(preferences, search_results)
+    
+    print("\n🎧 Your Personalized Music Recommendations 🎧")
+    print(recommendations)
+    
+    return recommendations
+
+if __name__ == "__main__":
+    asyncio.run(music_recommendation_bot())
+    

judgeval-0.0.35/src/test.py

@@ -0,0 +1,143 @@
+import os
+import asyncio
+from openai import OpenAI
+from dotenv import load_dotenv
+from judgeval.common.tracer import Tracer, wrap
+from judgeval.scorers import AnswerRelevancyScorer, FaithfulnessScorer, GroundednessScorer
+
+# Load environment variables
+load_dotenv()
+
+# Initialize OpenAI client and Judgment tracer
+client = wrap(OpenAI())
+judgment = Tracer(project_name="music-bot-demo")
+
+@judgment.observe(span_type="tool")
+async def search_tavily(query):
+    """Search for information using Tavily."""
+    from tavily import TavilyClient
+    
+    tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY"))
+    search_result = tavily_client.search(
+        query=query,
+        search_depth="advanced",
+        max_results=5
+    )
+    
+    return search_result
+
+@judgment.observe(span_type="function")
+async def ask_user_preferences():
+    """Ask the user a series of questions about their music preferences."""
+    questions = [
+        "What are some of your favorite artists or bands?",
+        "What genres of music do you enjoy the most?",
+        "Do you have any favorite songs currently?",
+        "Are there any moods or themes you're looking for in new music?",
+        "Do you prefer newer releases or classic songs?"
+    ]
+    
+    preferences = {}
+    for question in questions:
+        print(f"\n{question}")
+        answer = input("> ")
+        preferences[question] = answer
+    
+    return preferences
+
+@judgment.observe(span_type="function")
+async def search_music_recommendations(preferences):
+    """Search for music recommendations based on user preferences."""
+    # Construct search queries based on preferences
+    search_results = {}
+    
+    # Search for artist recommendations
+    if preferences.get("What are some of your favorite artists or bands?"):
+        artists_query = f"Music similar to {preferences['What are some of your favorite artists or bands?']}"
+        search_results["artist_based"] = await search_tavily(artists_query)
+    
+    # Search for genre recommendations
+    if preferences.get("What genres of music do you enjoy the most?"):
+        genre_query = f"Best {preferences['What genres of music do you enjoy the most?']} songs"
+        search_results["genre_based"] = await search_tavily(genre_query)
+    
+    # Search for mood-based recommendations
+    if preferences.get("Are there any moods or themes you're looking for in new music?"):
+        mood_query = f"""{preferences["Are there any moods or themes you're looking for in new music?"]} music recommendations"""
+        search_results["mood_based"] = await search_tavily(mood_query)
+    
+    return search_results
+
+@judgment.observe(span_type="function")
+async def generate_recommendations(preferences, search_results):
+    """Generate personalized music recommendations using the search results."""
+    # Prepare context from search results
+    context = ""
+    for category, results in search_results.items():
+        context += f"\n{category.replace('_', ' ').title()} Search Results:\n"
+        for result in results.get("results", []):
+            context += f"- {result.get('title')}: {result.get('content')[:200]}...\n"
+    
+    # Create a prompt for the LLM
+    prompt = f"""
+    Suggest 5-7 songs they could enjoy. Be creative and suggest whatever feels right. You should only recommend songs that are from the user's favorite artists/bands.
+    For each song, include the artist name, song title, and a brief explanation of why they might like it.
+    
+    User Preferences:
+    {preferences}
+    
+    Search Results:
+    {context}
+    
+    Provide recommendations in a clear, organized format. Focus on specific songs rather than just artists.
+    """
+
+    
+    # Generate recommendations using OpenAI
+    response = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a music recommendation expert with deep knowledge of various genres, artists, and songs. Your goal is to suggest songs that match the user's preferences; recommend songs from their favorite artists/bands."},
+            {"role": "user", "content": prompt}
+        ]
+    )
+    
+    recommendations = response.choices[0].message.content
+    
+    # Evaluate the recommendations
+    judgment.get_current_trace().async_evaluate(
+        scorers=[
+            AnswerRelevancyScorer(threshold=1.0),
+            GroundednessScorer(threshold=1.0)
+        ],
+        input=prompt,
+        actual_output=recommendations,
+        retrieval_context=[str(search_results)],
+        model="gpt-4o"
+    )
+    
+    return recommendations
+
+@judgment.observe(span_type="Main Function")
+async def music_recommendation_bot():
+    """Main function to run the music recommendation bot."""
+    print("🎵 Welcome to the Music Recommendation Bot! 🎵")
+    print("I'll ask you a few questions to understand your music taste, then suggest some songs you might enjoy.")
+    
+    # Get user preferences
+    preferences = await ask_user_preferences()
+    
+    print("\nSearching for music recommendations based on your preferences...")
+    search_results = await search_music_recommendations(preferences)
+    
+    print("\nGenerating personalized recommendations...")
+    recommendations = await generate_recommendations(preferences, search_results)
+    
+    print("\n🎧 Your Personalized Music Recommendations 🎧")
+    print(recommendations)
+    
+    return recommendations
+
+if __name__ == "__main__":
+    asyncio.run(music_recommendation_bot())
+    

judgeval-0.0.33/src/demo/demo.py

@@ -1,50 +0,0 @@
-from judgeval import JudgmentClient
-from judgeval.data import Example, Sequence
-from judgeval.scorers import DerailmentScorer
-
-client = JudgmentClient()
-
-airlines_example = Example(
-    input="Which airlines fly to Tokyo?",
-    actual_output="Japan Airlines, All Nippon Airways, and Chinese Airlines offer direct flights."
-)
-weather_example = Example(
-    input="What is the weather like in Japan?",
-    actual_output="It's cloudy with a high of 75°F and a low of 60°F in Japan."
-)
-airline_sequence = Sequence(
-    name="Flight Details",
-    items=[airlines_example, weather_example],
-)
-
-# Level 1: Top-level sequence
-top_example1 = Example(
-    input="I want to plan a trip to Tokyok.",
-    actual_output="That sounds great! When are you planning to go?"
-)
-top_example2 = Example(
-    input="Can you book a flight for me and anything else I need to know?",
-    actual_output="Sure, I'll help you with flights. hotels. and transportation."
-)
-top_level_sequence = Sequence(
-    name="Travel Planning",
-    items=[top_example1, top_example2, airline_sequence],
-)
-
-other_sequence = Sequence(
-    name="Other",
-    items=[Example(
-        input="What is the weather like in Tokyo?",
-        actual_output="It's cloudy with a high of 75°F and a low of 60°F in Tokyo."
-    )]
-)
-
-results = client.run_sequence_evaluation(
-    eval_run_name="sequence-run1",
-    project_name="jnpr-demo-sequence",
-    scorers=[DerailmentScorer(threshold=1)],
-    sequences=[top_level_sequence, other_sequence],
-    model="gpt-4o",
-    log_results=True,
-    override=True,
-)