judgeval 0.0.11__tar.gz → 0.0.12__tar.gz

{judgeval-0.0.11 → judgeval-0.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: judgeval
-Version: 0.0.11
+Version: 0.0.12
 Summary: Judgeval Package
 Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
 Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues

{judgeval-0.0.11 → judgeval-0.0.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgeval"
-version = "0.0.11"
+version = "0.0.12"
 authors = [
     { name="Andrew Li", email="andrew@judgmentlabs.ai" },
     { name="Alex Shan", email="alex@judgmentlabs.ai" },

judgeval-0.0.12/src/demo/cookbooks/new_bot/basic_bot.py

@@ -0,0 +1,106 @@
+import os
+import asyncio
+from typing import Dict, List
+from openai import OpenAI
+from uuid import uuid4
+from dotenv import load_dotenv
+
+from judgeval.tracer import Tracer, wrap
+from judgeval.scorers import AnswerRelevancyScorer, FaithfulnessScorer
+
+# Initialize clients
+load_dotenv()
+judgment = Tracer(api_key=os.getenv("JUDGMENT_API_KEY"), project_name="restaurant_bot")
+client = wrap(OpenAI())
+
+@judgment.observe(span_type="Research")
+async def search_restaurants(cuisine: str, location: str = "nearby") -> List[Dict]:
+    """Search for restaurants matching the cuisine type."""
+    # Simulate API call to restaurant database
+    prompt = f"Find 3 popular {cuisine} restaurants {location}. Return ONLY a JSON array of objects with 'name', 'rating', and 'price_range' fields. No other text."
+    
+    response = client.chat.completions.create(
+        model="gpt-4",
+        messages=[
+            {"role": "system", "content": """You are a restaurant search expert. 
+             Return ONLY valid JSON arrays containing restaurant objects.
+             Example format: [{"name": "Restaurant Name", "rating": 4.5, "price_range": "$$"}]
+             Do not include any other text or explanations."""},
+            {"role": "user", "content": prompt}
+        ]
+    )
+    
+    try:
+        import json
+        return json.loads(response.choices[0].message.content)
+    except json.JSONDecodeError as e:
+        print(f"Error parsing JSON response: {response.choices[0].message.content}")
+        return [{"name": "Error fetching restaurants", "rating": 0, "price_range": "N/A"}]
+
+@judgment.observe(span_type="Research") 
+async def get_menu_highlights(restaurant_name: str) -> List[str]:
+    """Get popular menu items for a restaurant."""
+    prompt = f"What are 3 must-try dishes at {restaurant_name}?"
+    
+    response = client.chat.completions.create(
+        model="gpt-4",
+        messages=[
+            {"role": "system", "content": "You are a food critic. List only the dish names."},
+            {"role": "user", "content": prompt}
+        ]
+    )
+
+    judgment.get_current_trace().async_evaluate(
+        scorers=[AnswerRelevancyScorer(threshold=0.5)],
+        input=prompt,
+        actual_output=response.choices[0].message.content,
+        model="gpt-4",
+    )
+
+    return response.choices[0].message.content.split("\n")
+
+@judgment.observe(span_type="function")
+async def generate_recommendation(cuisine: str, restaurants: List[Dict], menu_items: Dict[str, List[str]]) -> str:
+    """Generate a natural language recommendation."""
+    context = f"""
+    Cuisine: {cuisine}
+    Restaurants: {restaurants}
+    Popular Items: {menu_items}
+    """
+    
+    response = client.chat.completions.create(
+        model="gpt-4",
+        messages=[
+            {"role": "system", "content": "You are a helpful food recommendation bot. Provide a natural recommendation based on the data."},
+            {"role": "user", "content": context}
+        ]
+    )
+    return response.choices[0].message.content
+
+@judgment.observe(span_type="Research")
+async def get_food_recommendations(cuisine: str) -> str:
+    """Main function to get restaurant recommendations."""
+    # Search for restaurants
+    restaurants = await search_restaurants(cuisine)
+    
+    # Get menu highlights for each restaurant
+    menu_items = {}
+    for restaurant in restaurants:
+        menu_items[restaurant['name']] = await get_menu_highlights(restaurant['name'])
+        
+    # Generate final recommendation
+    recommendation = await generate_recommendation(cuisine, restaurants, menu_items)
+    judgment.get_current_trace().async_evaluate(
+        scorers=[AnswerRelevancyScorer(threshold=0.5), FaithfulnessScorer(threshold=1.0)],
+        input=f"Create a recommendation for a restaurant and dishes based on the desired cuisine: {cuisine}",
+        actual_output=recommendation,
+        retrieval_context=[str(restaurants), str(menu_items)],
+        model="gpt-4",
+    )
+    return recommendation
+
+if __name__ == "__main__":
+    cuisine = input("What kind of food would you like to eat? ")
+    recommendation = asyncio.run(get_food_recommendations(cuisine))
+    print("\nHere are my recommendations:\n")
+    print(recommendation)

{judgeval-0.0.11 → judgeval-0.0.12}/src/demo/cookbooks/openai_travel_agent/agent.py

@@ -10,9 +10,11 @@ from chromadb.utils import embedding_functions
 from judgeval.common.tracer import Tracer, wrap
 from demo.cookbooks.openai_travel_agent.populate_db import destinations_data
 from demo.cookbooks.openai_travel_agent.tools import search_tavily
+from judgeval.scorers import AnswerRelevancyScorer, FaithfulnessScorer
+
 
 client = wrap(openai.Client(api_key=os.getenv("OPENAI_API_KEY")))
-judgment = Tracer()
+judgment = Tracer(api_key=os.getenv("JUDGMENT_API_KEY"), project_name="travel_agent_demo")
 
 def populate_vector_db(collection, destinations_data):
     """
@@ -45,6 +47,12 @@ async def get_flights(destination):
     """Search for flights to the destination."""
     prompt = f"Flights to {destination} from major cities"
     flights_search = search_tavily(prompt)
+    judgment.get_current_trace().async_evaluate(
+        scorers=[AnswerRelevancyScorer(threshold=0.5)],
+        input=prompt,
+        actual_output=flights_search,
+        model="gpt-4",
+    )
     return flights_search
 
 @judgment.observe(span_type="tool")
@@ -52,6 +60,12 @@ async def get_weather(destination, start_date, end_date):
     """Search for weather information."""
     prompt = f"Weather forecast for {destination} from {start_date} to {end_date}"
     weather_search = search_tavily(prompt)
+    judgment.get_current_trace().async_evaluate(
+        scorers=[AnswerRelevancyScorer(threshold=0.5)],
+        input=prompt,
+        actual_output=weather_search,
+        model="gpt-4",
+    )
     return weather_search
 
 def initialize_vector_db():
@@ -125,21 +139,23 @@ async def create_travel_plan(destination, start_date, end_date, research_data):
             {"role": "user", "content": prompt}
         ]
     ).choices[0].message.content
+
+    judgment.get_current_trace().async_evaluate(
+        scorers=[FaithfulnessScorer(threshold=0.5)],
+        input=prompt,
+        actual_output=response,
+        retrieval_context=[str(vector_db_context), str(research_data)],
+        model="gpt-4",
+    )
     
     return response
 
-
+@judgment.observe(span_type="Main Function", overwrite=True)
 async def generate_itinerary(destination, start_date, end_date):
     """Main function to generate a travel itinerary."""
-    
-    with judgment.trace(
-        f"generate_itinerary_demo_{uuid4()}",
-        project_name="travel_agent_demo"    
-    ) as trace:
-        research_data = await research_destination(destination, start_date, end_date)
-        res = await create_travel_plan(destination, start_date, end_date, research_data)
-        trace.save()
-        return res
+    research_data = await research_destination(destination, start_date, end_date)
+    res = await create_travel_plan(destination, start_date, end_date, research_data)
+    return res
 
 
 if __name__ == "__main__":

{judgeval-0.0.11 → judgeval-0.0.12}/src/demo/cookbooks/openai_travel_agent/tools.py

@@ -5,7 +5,7 @@ from tavily import TavilyClient
 
 from judgeval.common.tracer import Tracer
 
-judgment = Tracer()
+judgment = Tracer(project_name="travel_agent_demo")
 
 @judgment.observe(span_type="search_tool")
 def search_tavily(query):

{judgeval-0.0.11 → judgeval-0.0.12}/src/demo/customer_use/cstone/faithfulness_testing.py

@@ -53,10 +53,10 @@ def run_judgment_evaluation(examples: List[Example]):
     scorer = FaithfulnessScorer(threshold=1.0)
     
     output = client.run_evaluation(
-        model="osiris-mini",
+        model="osiris-large",
         examples=examples,
         scorers=[scorer],
-        eval_run_name="cstone-basic-test-osiris-mini-2", 
+        eval_run_name="cstone-basic-test-osiris-large-1", 
         project_name="cstone_faithfulness_testing",
         override=True,
     )
@@ -66,7 +66,7 @@ def run_judgment_evaluation(examples: List[Example]):
         score = result.scorers_data[0].score
         scores.append(score)
         
-    return [score < 0.95 for score in scores]
+    return [score < 1 for score in scores]
 
 def run_patronus_evaluation(examples: List[Example]):
     """
@@ -94,7 +94,7 @@ def run_patronus_evaluation(examples: List[Example]):
 
     print(f"patronus scores: {scores}")
         
-    return [score < 0.95 for score in scores]
+    return [score < 0.9 for score in scores]
 
 def evaluate_predictions(predictions):
     """Calculate metrics comparing predictions to gold labels"""

{judgeval-0.0.11 → judgeval-0.0.12}/src/judgeval/common/tracer.py

@@ -557,7 +557,8 @@ class TraceClient:
             "overwrite": overwrite
         }
         
-        if not empty_save:
+        # Execute asynchrous evaluation in the background
+        if not empty_save:  # Only send to RabbitMQ if the trace is not empty
             connection = pika.BlockingConnection(
                 pika.ConnectionParameters(host=RABBITMQ_HOST, port=RABBITMQ_PORT))
             channel = connection.channel()
@@ -588,23 +589,31 @@ class Tracer:
             cls._instance = super(Tracer, cls).__new__(cls)
         return cls._instance
 
-    def __init__(self, api_key: str = os.getenv("JUDGMENT_API_KEY")):
+    def __init__(self, api_key: str = os.getenv("JUDGMENT_API_KEY"), project_name: str = "default_project"):
         if not hasattr(self, 'initialized'):
-
             if not api_key:
                 raise ValueError("Tracer must be configured with a Judgment API key")
             
             self.api_key: str = api_key
+            self.project_name: str = project_name
             self.client: JudgmentClient = JudgmentClient(judgment_api_key=api_key)
             self.depth: int = 0
             self._current_trace: Optional[str] = None
             self.initialized: bool = True
+        elif hasattr(self, 'project_name') and self.project_name != project_name:
+            warnings.warn(
+                f"Attempting to initialize Tracer with project_name='{project_name}' but it was already initialized with "
+                f"project_name='{self.project_name}'. Due to the singleton pattern, the original project_name will be used. "
+                "To use a different project name, ensure the first Tracer initialization uses the desired project name.",
+                RuntimeWarning
+            )
         
     @contextmanager
-    def trace(self, name: str, project_name: str = "default_project", overwrite: bool = False) -> Generator[TraceClient, None, None]:
+    def trace(self, name: str, project_name: str = None, overwrite: bool = False) -> Generator[TraceClient, None, None]:
         """Start a new trace context using a context manager"""
         trace_id = str(uuid.uuid4())
-        trace = TraceClient(self, trace_id, name, project_name=project_name, overwrite=overwrite)
+        project = project_name if project_name is not None else self.project_name
+        trace = TraceClient(self, trace_id, name, project_name=project, overwrite=overwrite)
         prev_trace = self._current_trace
         self._current_trace = trace
         
@@ -623,28 +632,40 @@ class Tracer:
         """
         return self._current_trace    
 
-    def observe(self, func=None, *, name=None, span_type: SpanType = "span"):
+    def observe(self, func=None, *, name=None, span_type: SpanType = "span", project_name: str = None, overwrite: bool = False):
         """
         Decorator to trace function execution with detailed entry/exit information.
         
         Args:
-            func: The function to trace
-            name: Optional custom name for the function
-            span_type: The type of span to use for this observation (default: "span")
+            func: The function to decorate
+            name: Optional custom name for the span (defaults to function name)
+            span_type: Type of span (default "span")
+            project_name: Optional project name override
+            overwrite: Whether to overwrite existing traces
         """
         if func is None:
-            return lambda f: self.observe(f, name=name, span_type=span_type)
+            return lambda f: self.observe(f, name=name, span_type=span_type, project_name=project_name, overwrite=overwrite)
+        
+        # Use provided name or fall back to function name
+        span_name = name or func.__name__
         
         if asyncio.iscoroutinefunction(func):
             @functools.wraps(func)
             async def async_wrapper(*args, **kwargs):
+                # If there's already a trace, use it. Otherwise create a new one
                 if self._current_trace:
-                    span_name = name or func.__name__
-                    
-                    with self._current_trace.span(span_name, span_type=span_type) as span:
-                        # Set the span type
-                        span.span_type = span_type
-                        
+                    trace = self._current_trace
+                else:
+                    trace_id = str(uuid.uuid4())
+                    trace_name = str(uuid.uuid4())
+                    project = project_name if project_name is not None else self.project_name
+                    trace = TraceClient(self, trace_id, trace_name, project_name=project, overwrite=overwrite)
+                    self._current_trace = trace
+                    # Only save empty trace for the root call
+                    trace.save(empty_save=True, overwrite=overwrite)
+
+                try:
+                    with trace.span(span_name, span_type=span_type) as span:
                         # Record inputs
                         span.record_input({
                             'args': list(args),
@@ -658,19 +679,30 @@ class Tracer:
                         span.record_output(result)
                         
                         return result
-                
-                return await func(*args, **kwargs)
+                finally:
+                    # Only save and cleanup if this is the root observe call
+                    if self.depth == 0:
+                        trace.save(empty_save=False, overwrite=overwrite)
+                        self._current_trace = None
+                    
             return async_wrapper
         else:
             @functools.wraps(func)
             def wrapper(*args, **kwargs):
+                # If there's already a trace, use it. Otherwise create a new one
                 if self._current_trace:
-                    span_name = name or func.__name__
-                    
-                    with self._current_trace.span(span_name, span_type=span_type) as span:
-                        # Set the span type
-                        span.span_type = span_type
-                        
+                    trace = self._current_trace
+                else:
+                    trace_id = str(uuid.uuid4())
+                    trace_name = str(uuid.uuid4())
+                    project = project_name if project_name is not None else self.project_name
+                    trace = TraceClient(self, trace_id, trace_name, project_name=project, overwrite=overwrite)
+                    self._current_trace = trace
+                    # Only save empty trace for the root call
+                    trace.save(empty_save=True, overwrite=overwrite)
+                
+                try:
+                    with trace.span(span_name, span_type=span_type) as span:
                         # Record inputs
                         span.record_input({
                             'args': list(args),
@@ -684,8 +716,12 @@ class Tracer:
                         span.record_output(result)
                         
                         return result
-                
-                return func(*args, **kwargs)
+                finally:
+                    # Only save and cleanup if this is the root observe call
+                    if self.depth == 0:
+                        trace.save(empty_save=False, overwrite=overwrite)
+                        self._current_trace = None
+                    
             return wrapper
 
 def wrap(client: Any) -> Any: