judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/v1/trainers/trainable_model.py

@@ -0,0 +1,252 @@
+import time
+from fireworks import LLM  # type: ignore[import-not-found,import-untyped]
+from .config import TrainerConfig, ModelConfig
+from typing import Optional, Dict, Any, Callable
+from .console import _model_spinner_progress, _print_model_progress
+from judgeval.exceptions import JudgmentRuntimeError
+
+
+class TrainableModel:
+    """
+    A wrapper class for managing model snapshots during training.
+
+    This class automatically handles model snapshot creation and management
+    during the RFT (Reinforcement Fine-Tuning) process,
+    abstracting away manual snapshot management from users.
+    """
+
+    config: TrainerConfig
+    current_step: int
+    _current_model: LLM
+    _tracer_wrapper_func: Optional[Callable]
+    _base_model: LLM
+
+    def __init__(self, config: TrainerConfig):
+        """
+        Initialize the TrainableModel.
+
+        Args:
+            config: TrainerConfig instance with model configuration
+        """
+        try:
+            self.config = config
+            self.current_step = 0
+            self._tracer_wrapper_func = None
+
+            self._base_model = self._create_base_model()
+            self._current_model = self._base_model
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"Failed to initialize TrainableModel: {str(e)}"
+            ) from e
+
+    @classmethod
+    def from_model_config(cls, model_config: ModelConfig) -> "TrainableModel":
+        """
+        Create a TrainableModel from a saved ModelConfig.
+
+        Args:
+            model_config: ModelConfig instance with saved model state
+
+        Returns:
+            TrainableModel instance configured to use the saved model
+        """
+        # Create a TrainerConfig from the ModelConfig
+        trainer_config = TrainerConfig(
+            base_model_name=model_config.base_model_name,
+            deployment_id=model_config.deployment_id,
+            user_id=model_config.user_id,
+            model_id=model_config.model_id,
+            enable_addons=model_config.enable_addons,
+        )
+
+        instance = cls(trainer_config)
+        instance.current_step = model_config.current_step
+
+        if model_config.is_trained and model_config.current_model_name:
+            instance._load_trained_model(model_config.current_model_name)
+
+        return instance
+
+    def _create_base_model(self):
+        """Create and configure the base model."""
+        try:
+            with _model_spinner_progress(
+                "Creating and deploying base model..."
+            ) as update_progress:
+                update_progress("Creating base model instance...")
+                base_model = LLM(
+                    model=self.config.base_model_name,
+                    deployment_type="on-demand",
+                    id=self.config.deployment_id,
+                    enable_addons=self.config.enable_addons,
+                )
+                update_progress("Applying deployment configuration...")
+                base_model.apply()
+            _print_model_progress("Base model deployment ready")
+            return base_model
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"Failed to create and deploy base model '{self.config.base_model_name}': {str(e)}"
+            ) from e
+
+    def _load_trained_model(self, model_name: str):
+        """Load a trained model by name."""
+        try:
+            with _model_spinner_progress(
+                f"Loading and deploying trained model: {model_name}"
+            ) as update_progress:
+                update_progress("Creating trained model instance...")
+                self._current_model = LLM(
+                    model=model_name,
+                    deployment_type="on-demand-lora",
+                    base_id=self.config.deployment_id,
+                )
+                update_progress("Applying deployment configuration...")
+                self._current_model.apply()
+            _print_model_progress("Trained model deployment ready")
+
+            if self._tracer_wrapper_func:
+                self._tracer_wrapper_func(self._current_model)
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"Failed to load and deploy trained model '{model_name}': {str(e)}"
+            ) from e
+
+    def get_current_model(self):
+        return self._current_model
+
+    @property
+    def chat(self):
+        """OpenAI-compatible chat interface."""
+        return self._current_model.chat
+
+    @property
+    def completions(self):
+        """OpenAI-compatible completions interface."""
+        return self._current_model.completions
+
+    def advance_to_next_step(self, step: int):
+        """
+        Advance to the next training step and update the current model snapshot.
+
+        Args:
+            step: The current training step number
+        """
+        try:
+            self.current_step = step
+
+            if step == 0:
+                self._current_model = self._base_model
+            else:
+                model_name = f"accounts/{self.config.user_id}/models/{self.config.model_id}-v{step}"
+                with _model_spinner_progress(
+                    f"Creating and deploying model snapshot: {model_name}"
+                ) as update_progress:
+                    update_progress("Creating model snapshot instance...")
+                    self._current_model = LLM(
+                        model=model_name,
+                        deployment_type="on-demand-lora",
+                        base_id=self.config.deployment_id,
+                    )
+                    update_progress("Applying deployment configuration...")
+                    self._current_model.apply()
+                _print_model_progress("Model snapshot deployment ready")
+
+                if self._tracer_wrapper_func:
+                    self._tracer_wrapper_func(self._current_model)
+        except Exception as e:
+            raise JudgmentRuntimeError(
+                f"Failed to advance to training step {step}: {str(e)}"
+            ) from e
+
+    def perform_reinforcement_step(
+        self, dataset, step: int, max_retries: int = 3, initial_backoff: float = 1.0
+    ):
+        """
+        Perform a reinforcement learning step using the current model.
+
+        Args:
+            dataset: Training dataset for the reinforcement step
+            step: Current step number for output model naming
+            max_retries: Maximum number of retry attempts (default: 3)
+            initial_backoff: Initial backoff time in seconds for exponential backoff (default: 1.0)
+
+        Returns:
+            Training job object
+        """
+        model_name = f"{self.config.model_id}-v{step + 1}"
+
+        for attempt in range(max_retries):
+            try:
+                return self._current_model.reinforcement_step(
+                    dataset=dataset,
+                    output_model=model_name,
+                    epochs=self.config.epochs,
+                    learning_rate=self.config.learning_rate,
+                )
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    backoff_time = initial_backoff * (2**attempt)
+                    time.sleep(backoff_time)
+                else:
+                    raise JudgmentRuntimeError(
+                        f"Failed to start reinforcement learning step {step + 1} after {max_retries} attempts: {str(e)}"
+                    ) from e
+
+    def get_model_config(
+        self, training_params: Optional[Dict[str, Any]] = None
+    ) -> ModelConfig:
+        """
+        Get the current model configuration for persistence.
+
+        Args:
+            training_params: Optional training parameters to include in config
+
+        Returns:
+            ModelConfig instance with current model state
+        """
+        current_model_name = None
+        is_trained = False
+
+        if self.current_step > 0:
+            current_model_name = f"accounts/{self.config.user_id}/models/{self.config.model_id}-v{self.current_step}"
+            is_trained = True
+
+        return ModelConfig(
+            base_model_name=self.config.base_model_name,
+            deployment_id=self.config.deployment_id,
+            user_id=self.config.user_id,
+            model_id=self.config.model_id,
+            enable_addons=self.config.enable_addons,
+            current_step=self.current_step,
+            total_steps=self.config.num_steps,
+            current_model_name=current_model_name,
+            is_trained=is_trained,
+            training_params=training_params,
+        )
+
+    def save_model_config(
+        self, filepath: str, training_params: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Save the current model configuration to a file.
+
+        Args:
+            filepath: Path to save the configuration file
+            training_params: Optional training parameters to include in config
+        """
+        model_config = self.get_model_config(training_params)
+        model_config.save_to_file(filepath)
+
+    def _register_tracer_wrapper(self, wrapper_func: Callable):
+        """
+        Register a tracer wrapper function to be reapplied when models change.
+
+        This is called internally by the tracer's wrap() function to ensure
+        that new model instances created during training are automatically wrapped.
+
+        Args:
+            wrapper_func: Function that wraps a model instance with tracing
+        """
+        self._tracer_wrapper_func = wrapper_func

judgeval/v1/trainers/trainers_factory.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from judgeval.v1.internal.api import JudgmentSyncClient
+
+if TYPE_CHECKING:
+    from judgeval.v1.trainers.config import TrainerConfig
+    from judgeval.v1.trainers.trainable_model import TrainableModel
+    from judgeval.v1.tracer.tracer import Tracer
+
+
+class TrainersFactory:
+    __slots__ = "_client"
+
+    def __init__(
+        self,
+        client: JudgmentSyncClient,
+    ):
+        self._client = client
+
+    def fireworks(
+        self,
+        config: TrainerConfig,
+        trainable_model: TrainableModel,
+        tracer: Tracer,
+        project_name: str | None = None,
+    ):
+        from judgeval.v1.trainers.fireworks_trainer import FireworksTrainer
+
+        return FireworksTrainer(
+            config=config,
+            trainable_model=trainable_model,
+            tracer=tracer,
+            project_name=project_name,
+            client=self._client,
+        )

judgeval/v1/utils.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Optional
+
+from judgeval.logger import judgeval_logger
+from judgeval.v1.internal.api import JudgmentSyncClient
+
+
+@lru_cache(maxsize=128)
+def resolve_project_id(client: JudgmentSyncClient, project_name: str) -> Optional[str]:
+    try:
+        response = client.projects_resolve({"project_name": project_name})
+        project_id = response.get("project_id")
+        return str(project_id) if project_id else None
+    except Exception as e:
+        judgeval_logger.error(f"Failed to resolve project '{project_name}': {str(e)}")
+        return None

judgeval/version.py

@@ -0,0 +1,5 @@
+__version__ = "0.23.0"
+
+
+def get_version() -> str:
+    return __version__

judgeval/warnings.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+
+class JudgmentWarning(Warning): ...

judgeval-0.23.0.dist-info/METADATA

@@ -0,0 +1,266 @@
+Metadata-Version: 2.4
+Name: judgeval
+Version: 0.23.0
+Summary: The open source post-building layer for Agent Behavior Monitoring.
+Project-URL: Homepage, https://github.com/JudgmentLabs/judgeval
+Project-URL: Issues, https://github.com/JudgmentLabs/judgeval/issues
+Author-email: Andrew Li <andrew@judgmentlabs.ai>, Alex Shan <alex@judgmentlabs.ai>, Joseph Camyre <joseph@judgmentlabs.ai>
+Maintainer-email: Judgment Labs <contact@judgmentlabs.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.md
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: boto3>=1.40.11
+Requires-Dist: click<8.2.0
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: litellm>=1.75.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.36.0
+Requires-Dist: opentelemetry-sdk>=1.36.0
+Requires-Dist: orjson>=3.9.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: s3
+Requires-Dist: boto3>=1.40.11; extra == 's3'
+Provides-Extra: trainer
+Requires-Dist: fireworks-ai>=0.19.18; extra == 'trainer'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+<a href="https://judgmentlabs.ai/">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="assets/logo_darkmode.svg">
+    <img src="assets/logo_lightmode.svg" alt="Judgment Logo" width="400" />
+  </picture>
+</a>
+
+<br>
+
+## Agent Behavior Monitoring (ABM)
+
+Track and judge any agent behavior in online and offline setups. Set up Sentry-style alerts and analyze agent behaviors / topic patterns at scale! 
+
+[![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.judgmentlabs.ai/documentation)
+[![Judgment Cloud](https://img.shields.io/badge/Judgment%20Cloud-brightgreen)](https://app.judgmentlabs.ai/register)
+[![Self-Host](https://img.shields.io/badge/Self--Host-orange)](https://docs.judgmentlabs.ai/documentation/self-hosting/get-started)
+
+
+[![X](https://img.shields.io/badge/-X/Twitter-000?logo=x&logoColor=white)](https://x.com/JudgmentLabs)
+[![LinkedIn](https://custom-icon-badges.demolab.com/badge/LinkedIn%20-0A66C2?logo=linkedin-white&logoColor=fff)](https://www.linkedin.com/company/judgmentlabs)
+
+</div>
+
+
+</table>
+
+## [NEW] 🎆 Agent Reinforcement Learning
+
+Train your agents with multi-turn reinforcement learning using judgeval and [Fireworks AI](https://fireworks.ai/)! Judgeval's ABM now integrates with Fireworks' Reinforcement Fine-Tuning (RFT) endpoint, supporting gpt-oss, qwen3, Kimi2, DeepSeek, and more.
+
+Judgeval's agent monitoring infra provides a simple harness for integrating GRPO into any Python agent, giving builders a quick method to **try RL with minimal code changes** to their existing agents!
+
+```python
+await trainer.train(
+    agent_function=your_agent_function,  # entry point to your agent
+    scorers=[RewardScorer()],  # Custom scorer you define based on task criteria, acts as reward
+    prompts=training_prompts  # Tasks
+)
+```
+
+**That's it!** Judgeval automatically manages trajectory collection and reward tagging - your agent can learn from production data with minimal code changes. 
+
+👉 Check out the [Wikipedia Racer notebook](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/rl/WikiRacingAgent_RL.ipynb), where an agent learns to navigate Wikipedia using RL, to see Judgeval in action.
+
+
+You can view and monitor training progress for free via the [Judgment Dashboard](https://app.judgmentlabs.ai/).
+
+
+## Judgeval Overview
+
+Judgeval is an open-source framework for agent behavior monitoring. Judgeval offers a toolkit to track and judge agent behavior in online and offline setups, enabling you to convert interaction data from production/test environments into improved agents. To get started, try running one of the notebooks below or dive deeper in our [docs](https://docs.judgmentlabs.ai/documentation).
+
+Our mission is to unlock the power of production data for agent development, enabling teams to improve their apps by catching real-time failures and optimizing over their users' preferences.
+
+## 📚 Cookbooks
+
+| Try Out | Notebook | Description |
+|:---------|:-----|:------------|
+| RL | [Wikipedia Racer](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/rl/WikiRacingAgent_RL.ipynb) | Train agents with reinforcement learning |
+| Online ABM | [Research Agent](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/monitoring/Research_Agent_Online_Monitoring.ipynb) | Monitor agent behavior in production |
+| Custom Scorers | [HumanEval](https://colab.research.google.com/github/JudgmentLabs/judgment-cookbook/blob/main/custom_scorers/HumanEval_Custom_Scorer.ipynb) | Build custom evaluators for your agents |
+| Offline Testing | [Get Started For Free] | Compare how different prompts, models, or agent configs affect performance across ANY metric |
+
+You can access our [repo of cookbooks](https://github.com/JudgmentLabs/judgment-cookbook).
+
+You can find a list of [video tutorials for Judgeval use cases](https://www.youtube.com/@Alexshander-JL).
+
+## Why Judgeval?
+
+🤖 **Simple to run multi-turn RL**: Optimize your agents with multi-turn RL without managing compute infrastructure or data pipelines. Just add a few lines of code to your existing agent code and train!
+
+⚙️ **Custom Evaluators**: No restriction to only monitoring with prefab scorers. Judgeval provides simple abstractions for custom Python scorers, supporting any LLM-as-a-judge rubrics/models and code-based scorers that integrate to our live agent-tracking infrastructure. [Learn more](https://docs.judgmentlabs.ai/documentation/evaluation/custom-scorers)
+
+🚨 **Production Monitoring**: Run any custom scorer in a hosted, virtualized secure container to flag agent behaviors online in production. Get Slack alerts for failures and add custom hooks to address regressions before they impact users. [Learn more](https://docs.judgmentlabs.ai/documentation/performance/online-evals)
+
+📊 **Behavior/Topic Grouping**: Group agent runs by behavior type or topic for deeper analysis. Drill down into subsets of users, agents, or use cases to reveal patterns of agent behavior.
+<!-- Add link to Bucketing docs once we have it -->
+<!-- 
+TODO: Once we have trainer code docs, plug in here
+-->
+
+🧪 **Run experiments on your agents**: Compare test different prompts, models, or agent configs across customer segments. Measure which changes improve agent performance and decrease bad agent behaviors.
+
+<!-- 
+Use this once we have AI PM features:
+
+**Run experiments on your agents**: A/B test different prompts, models, or agent configs across customer segments. Measure which changes improve agent performance and decrease bad agent behaviors. [Learn more]
+
+-->
+
+## 🛠️ Quickstart
+
+Get started with Judgeval by installing our SDK using pip:
+
+```bash
+pip install judgeval
+```
+
+Ensure you have your `JUDGMENT_API_KEY` and `JUDGMENT_ORG_ID` environment variables set to connect to the [Judgment Platform](https://app.judgmentlabs.ai/).
+
+```bash
+export JUDGMENT_API_KEY=...
+export JUDGMENT_ORG_ID=...
+```
+
+**If you don't have keys, [create an account for free](https://app.judgmentlabs.ai/register) on the platform!**
+
+### Start monitoring with Judgeval
+
+```python
+from judgeval.tracer import Tracer, wrap
+from judgeval.data import Example
+from judgeval.scorers import AnswerRelevancyScorer
+from openai import OpenAI
+
+
+judgment = Tracer(project_name="default_project")
+client = wrap(OpenAI())  # tracks all LLM calls
+
+@judgment.observe(span_type="tool")
+def format_question(question: str) -> str:
+    # dummy tool
+    return f"Question : {question}"
+
+@judgment.observe(span_type="function")
+def run_agent(prompt: str) -> str:
+    task = format_question(prompt)
+    response = client.chat.completions.create(
+        model="gpt-5-mini",
+        messages=[{"role": "user", "content": task}]
+    )
+
+    judgment.async_evaluate(  # trigger online monitoring
+        scorer=AnswerRelevancyScorer(threshold=0.5),  # swap with any scorer
+        example=Example(input=task, actual_output=response),  # customize to your data
+        model="gpt-5",
+    )
+    return response.choices[0].message.content
+
+run_agent("What is the capital of the United States?")
+```
+
+Running this code will deliver monitoring results to your [free platform account](https://app.judgmentlabs.ai/register) and should look like this:
+
+![Judgment Platform Trajectory View](assets/quickstart_trajectory_ss.png)
+
+
+### Customizable Scorers Over Agent Behavior
+
+Judgeval's strongest suit is the full customization over the types of scorers you can run online monitoring with. No restrictions to only single-prompt LLM judges or prefab scorers - if you can express your scorer
+in python code, judgeval can monitor it! Under the hood, judgeval hosts your scorer in a virtualized secure container, enabling online monitoring for any scorer.
+
+
+First, create a behavior scorer in a file called `helpfulness_scorer.py`:
+
+```python
+from judgeval.data import Example
+from judgeval.scorers.example_scorer import ExampleScorer
+
+# Define custom example class
+class QuestionAnswer(Example):
+    question: str
+    answer: str
+
+# Define a server-hosted custom scorer
+class HelpfulnessScorer(ExampleScorer):
+    name: str = "Helpfulness Scorer"
+    server_hosted: bool = True  # Enable server hosting
+    async def a_score_example(self, example: QuestionAnswer):
+        # Custom scoring logic for agent behavior
+        # Can be an arbitrary combination of code and LLM calls
+        if len(example.answer) > 10 and "?" not in example.answer:
+            self.reason = "Answer is detailed and provides helpful information"
+            return 1.0
+        else:
+            self.reason = "Answer is too brief or unclear"
+            return 0.0
+```
+
+Then deploy your scorer to Judgment's infrastructure:
+
+```bash
+echo "pydantic" > requirements.txt
+uv run judgeval upload_scorer helpfulness_scorer.py requirements.txt
+```
+
+Now you can instrument your agent with monitoring and online evaluation:
+
+```python
+from judgeval.tracer import Tracer, wrap
+from helpfulness_scorer import HelpfulnessScorer, QuestionAnswer
+from openai import OpenAI
+
+judgment = Tracer(project_name="default_project")
+client = wrap(OpenAI())  # tracks all LLM calls
+
+@judgment.observe(span_type="tool")
+def format_task(question: str) -> str:  # replace with your prompt engineering
+    return f"Please answer the following question: {question}"
+
+@judgment.observe(span_type="tool")
+def answer_question(prompt: str) -> str:  # replace with your LLM system calls
+    response = client.chat.completions.create(
+        model="gpt-5-mini",
+        messages=[{"role": "user", "content": prompt}]
+    )
+    return response.choices[0].message.content
+
+@judgment.observe(span_type="function")
+def run_agent(question: str) -> str:
+    task = format_task(question)
+    answer = answer_question(task)
+
+    # Add online evaluation with server-hosted scorer
+    judgment.async_evaluate(
+        scorer=HelpfulnessScorer(),
+        example=QuestionAnswer(question=question, answer=answer),
+        sampling_rate=0.9  # Evaluate 90% of agent runs
+    )
+
+    return answer
+
+if __name__ == "__main__":
+    result = run_agent("What is the capital of the United States?")
+    print(result)
+```
+
+Congratulations! Your online eval result should look like this:
+
+![Custom Scorer Online ABM](assets/custom_scorer_online_abm.png)
+
+You can now run any online scorer in a secure Firecracker microVMs with no latency impact on your applications.
+
+---
+
+Judgeval is created and maintained by [Judgment Labs](https://judgmentlabs.ai/).