langgraph-agent-toolkit 0.1.0__py3-none-any.whl

langgraph_agent_toolkit/core/models/fake.py

@@ -0,0 +1,25 @@
+from typing import Any, Callable, Dict, Literal, Optional, Sequence, Union
+
+from langchain_community.chat_models import FakeListChatModel
+from langchain_core.language_models.base import LanguageModelInput
+from langchain_core.messages import BaseMessage
+from langchain_core.runnables import Runnable
+from langchain_core.tools import BaseTool
+
+
+class FakeToolModel(FakeListChatModel):
+    """A fake model that returns a fixed response for testing purposes."""
+
+    def __init__(self, responses: list[str]):
+        super().__init__(responses=responses)
+
+    def bind_tools(
+        self,
+        tools: Sequence[
+            Union[Dict[str, Any], type, Callable, BaseTool]  # noqa: UP006
+        ],
+        *,
+        tool_choice: Optional[Union[str, Literal["any"]]] = None,
+        **kwargs: Any,
+    ) -> Runnable[LanguageModelInput, BaseMessage]:
+        return self

langgraph_agent_toolkit/core/observability/__init__.py

@@ -0,0 +1,10 @@
+from langgraph_agent_toolkit.core.observability.base import BaseObservabilityPlatform
+from langgraph_agent_toolkit.core.observability.empty import EmptyObservability
+from langgraph_agent_toolkit.core.observability.factory import ObservabilityFactory
+from langgraph_agent_toolkit.core.observability.langfuse import LangfuseObservability
+from langgraph_agent_toolkit.core.observability.langsmith import LangsmithObservability
+from langgraph_agent_toolkit.core.observability.types import ObservabilityBackend
+
+
+# For backward compatibility
+create_observability_platform = ObservabilityFactory.create

langgraph_agent_toolkit/core/observability/base.py

@@ -0,0 +1,331 @@
+import functools
+import os
+import tempfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, TypeVar, cast
+
+import joblib
+from jinja2 import Template
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.prompts.chat import (
+    AIMessagePromptTemplate,
+    BaseMessage,
+    HumanMessagePromptTemplate,
+    MessagesPlaceholder,
+    SystemMessagePromptTemplate,
+)
+
+from langgraph_agent_toolkit.core.observability.types import MessageRole, PromptReturnType, PromptTemplateType
+from langgraph_agent_toolkit.helper.logging import logger
+
+
+T = TypeVar("T")
+
+
+class BaseObservabilityPlatform(ABC):
+    """Base class for observability platforms."""
+
+    __default_required_vars = []
+
+    def __init__(self, prompts_dir: Optional[str] = None):
+        self._required_vars = self.__default_required_vars.copy()
+
+        if prompts_dir:
+            self._prompts_dir = Path(prompts_dir)
+        else:
+            temp_base = Path(tempfile.gettempdir())
+            self._prompts_dir = temp_base / "langgraph_prompts"
+
+        self._prompts_dir.mkdir(exist_ok=True, parents=True)
+
+    @property
+    def prompts_dir(self) -> Path:
+        return self._prompts_dir
+
+    @prompts_dir.setter
+    def prompts_dir(self, path: str) -> None:
+        self._prompts_dir = Path(path)
+        self._prompts_dir.mkdir(exist_ok=True, parents=True)
+
+    @property
+    def required_vars(self) -> List[str]:
+        return self._required_vars
+
+    @required_vars.setter
+    def required_vars(self, value: List[str]) -> None:
+        self._required_vars = value
+
+    def validate_environment(self) -> bool:
+        missing_vars = [var for var in self._required_vars if not os.environ.get(var)]
+
+        if missing_vars:
+            raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}")
+
+        return True
+
+    @staticmethod
+    def requires_env_vars(func: Callable[..., T]) -> Callable[..., T]:
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            self.validate_environment()
+            return func(self, *args, **kwargs)
+
+        return wrapper
+
+    @abstractmethod
+    def get_callback_handler(self, **kwargs) -> Any:
+        pass
+
+    @abstractmethod
+    def before_shutdown(self) -> None:
+        pass
+
+    @abstractmethod
+    def record_feedback(self, run_id: str, key: str, score: float, **kwargs) -> None:
+        pass
+
+    def _handle_existing_prompt(
+        self,
+        name: str,
+        create_new_version: bool = True,
+        client: Any = None,
+        client_pull_method: Optional[str] = None,
+        client_delete_method: Optional[str] = None,
+    ) -> Tuple[Any, Any]:
+        existing_prompt = None
+        url = None
+
+        if not client or not client_pull_method or not client_delete_method:
+            return (existing_prompt, url)
+
+        pull_method = getattr(client, client_pull_method, None)
+        delete_method = getattr(client, client_delete_method, None)
+
+        if not pull_method or not delete_method:
+            return (existing_prompt, url)
+
+        if not create_new_version:
+            try:
+                existing_prompt = pull_method(name=name)
+                url = getattr(existing_prompt, "url", None)
+                logger.debug(f"Using existing prompt '{name}' as create_new_version is False")
+            except Exception:
+                logger.debug(f"Existing prompt '{name}' not found, will create a new one")
+        else:
+            try:
+                pull_method(name=name)
+                delete_method(name=name)
+                logger.debug(f"Deleted existing prompt '{name}' to create new version")
+            except Exception:
+                pass
+
+        return (existing_prompt, url)
+
+    def _convert_to_chat_prompt(self, prompt_template: PromptTemplateType) -> ChatPromptTemplate:
+        if isinstance(prompt_template, str):
+            return ChatPromptTemplate.from_template(prompt_template)
+        elif isinstance(prompt_template, list) and all(isinstance(msg, dict) for msg in prompt_template):
+            messages = []
+            for msg in prompt_template:
+                role = msg.get("role", "")
+                content = msg.get("content", "")
+
+                match role.lower():
+                    case MessageRole.SYSTEM:
+                        messages.append(SystemMessage(content=content))
+                    case MessageRole.HUMAN | MessageRole.USER:
+                        messages.append(HumanMessage(content=content))
+                    case MessageRole.AI | MessageRole.ASSISTANT:
+                        messages.append(AIMessage(content=content))
+                    case MessageRole.PLACEHOLDER | MessageRole.MESSAGES_PLACEHOLDER:
+                        messages.append(MessagesPlaceholder(variable_name=content))
+                    case _:
+                        raise ValueError(f"Unknown message role: {role}")
+
+            return ChatPromptTemplate.from_messages(messages)
+        else:
+            return cast(ChatPromptTemplate, prompt_template)
+
+    def _process_messages_from_prompt(
+        self, messages: List[Any], template_format: Literal["f-string", "mustache", "jinja2"] = "f-string"
+    ) -> List[Any]:
+        MESSAGE_TYPE_MAP = {
+            MessageRole.SYSTEM: SystemMessagePromptTemplate,
+            MessageRole.HUMAN: HumanMessagePromptTemplate,
+            MessageRole.USER: HumanMessagePromptTemplate,
+            MessageRole.AI: AIMessagePromptTemplate,
+            MessageRole.ASSISTANT: AIMessagePromptTemplate,
+        }
+
+        processed_messages = []
+
+        for msg in messages:
+            if isinstance(msg, MessagesPlaceholder):
+                processed_messages.append(msg)
+                continue
+
+            if isinstance(msg, BaseMessage):
+                msg_type = MessageRole(msg.type)
+                if msg_type in MESSAGE_TYPE_MAP:
+                    template_class = MESSAGE_TYPE_MAP[msg_type]
+                    processed_messages.append(
+                        template_class.from_template(msg.content, template_format=template_format)
+                    )
+                continue
+
+            if isinstance(msg, dict) and "role" in msg and "content" in msg:
+                role, content = msg["role"], msg["content"]
+
+                if role in MESSAGE_TYPE_MAP:
+                    template_class = MESSAGE_TYPE_MAP[role]
+                    processed_messages.append(template_class.from_template(content, template_format=template_format))
+                    continue
+
+                if role.lower() in (MessageRole.PLACEHOLDER, MessageRole.MESSAGES_PLACEHOLDER):
+                    processed_messages.append(MessagesPlaceholder(variable_name=content))
+                    continue
+
+            if isinstance(msg, tuple) and len(msg) == 2:
+                role, content = msg
+                if role in MESSAGE_TYPE_MAP:
+                    template_class = MESSAGE_TYPE_MAP[role]
+                    processed_messages.append(template_class.from_template(content, template_format=template_format))
+                    continue
+
+                if role.lower() in (MessageRole.PLACEHOLDER, MessageRole.MESSAGES_PLACEHOLDER):
+                    processed_messages.append(MessagesPlaceholder(variable_name=content))
+                    continue
+
+            processed_messages.append(msg)
+
+        return processed_messages
+
+    def _process_prompt_object(
+        self, prompt_obj: Any, template_format: Literal["f-string", "mustache", "jinja2"] = "f-string"
+    ) -> ChatPromptTemplate:
+        if isinstance(prompt_obj, ChatPromptTemplate):
+            return prompt_obj
+
+        if hasattr(prompt_obj, "messages") and isinstance(prompt_obj.messages, list):
+            processed_messages = self._process_messages_from_prompt(
+                prompt_obj.messages, template_format=template_format
+            )
+            if processed_messages:
+                return ChatPromptTemplate.from_messages(processed_messages)
+
+        elif isinstance(prompt_obj, list):
+            if all(isinstance(item, dict) and "role" in item and "content" in item for item in prompt_obj):
+                processed_messages = self._process_messages_from_prompt(prompt_obj, template_format=template_format)
+                if processed_messages:
+                    return ChatPromptTemplate.from_messages(processed_messages)
+
+        elif isinstance(prompt_obj, str):
+            return ChatPromptTemplate.from_template(prompt_obj, template_format=template_format)
+
+        else:
+            raise ValueError(f"Could not process prompt object of type {type(prompt_obj)}")
+
+    def _extract_template_string(self, prompt_template: PromptTemplateType, prompt_obj: Any) -> str:
+        if isinstance(prompt_template, str):
+            return prompt_template
+        elif isinstance(prompt_template, list) and all(isinstance(msg, dict) for msg in prompt_template):
+            template_str = ""
+            for msg in prompt_template:
+                template_str += f"[{msg['role']}]: {msg['content']}\n\n"
+            return template_str
+        else:
+            if hasattr(prompt_obj, "template"):
+                return prompt_obj.template
+            return str(prompt_obj)
+
+    def _local_pull_prompt(self, name: str, template_format: str = "f-string", **kwargs) -> PromptReturnType:
+        """Local implementation of pull_prompt that reads from the file system."""
+        file_path = self._prompts_dir / f"{name}.jinja2"
+
+        if not file_path.exists():
+            raise ValueError(f"Prompt '{name}' not found at {file_path}")
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            template_content = f.read()
+
+        metadata_path = self._prompts_dir / f"{name}.metadata.joblib"
+
+        if metadata_path.exists():
+            try:
+                metadata = joblib.load(metadata_path)
+                original_prompt = metadata.get("original_prompt")
+                if original_prompt:
+                    return original_prompt
+            except Exception:
+                pass
+
+        return ChatPromptTemplate.from_template(template_content, template_format=template_format)
+
+    def pull_prompt(
+        self, name: str, template_format: Literal["f-string", "mustache", "jinja2"] = "f-string", **kwargs
+    ) -> PromptReturnType:
+        """Pull a prompt from the observability platform."""
+        # Use the local implementation
+        return self._local_pull_prompt(name, template_format=template_format, **kwargs)
+
+    def push_prompt(
+        self,
+        name: str,
+        prompt_template: PromptTemplateType,
+        metadata: Optional[Dict[str, Any]] = None,
+        create_new_version: bool = True,
+    ) -> None:
+        self._prompts_dir.mkdir(exist_ok=True, parents=True)
+
+        file_path = self._prompts_dir / f"{name}.jinja2"
+        metadata_path = self._prompts_dir / f"{name}.metadata.joblib"
+
+        if create_new_version:
+            if file_path.exists():
+                file_path.unlink()
+            if metadata_path.exists():
+                metadata_path.unlink()
+
+        chat_prompt = self._convert_to_chat_prompt(prompt_template)
+        template_str = self._extract_template_string(prompt_template, chat_prompt)
+
+        with open(file_path, "w", encoding="utf-8") as f:
+            f.write(str(template_str))
+
+        full_metadata = metadata.copy() if metadata else {}
+        if not isinstance(prompt_template, str):
+            full_metadata["original_prompt"] = chat_prompt
+            full_metadata["original_format"] = "chat_message_dict" if isinstance(prompt_template, list) else "other"
+            joblib.dump(full_metadata, metadata_path)
+        elif metadata:
+            joblib.dump(full_metadata, metadata_path)
+
+    def get_template(self, name: str) -> str:
+        file_path = self._prompts_dir / f"{name}.jinja2"
+
+        if not file_path.exists():
+            raise ValueError(f"Prompt '{name}' not found at {file_path}")
+
+        with open(file_path, "r", encoding="utf-8") as f:
+            return f.read()
+
+    def render_prompt(self, prompt_name: str, **variables) -> str:
+        template_content = self.get_template(prompt_name)
+        template = Template(template_content)
+        return template.render(**variables)
+
+    def delete_prompt(self, name: str) -> None:
+        file_path = self._prompts_dir / f"{name}.jinja2"
+        metadata_path = self._prompts_dir / f"{name}.metadata.joblib"
+        json_metadata_path = self._prompts_dir / f"{name}.metadata.json"
+
+        if file_path.exists():
+            file_path.unlink()
+
+        if metadata_path.exists():
+            metadata_path.unlink()
+
+        if json_metadata_path.exists():
+            json_metadata_path.unlink()

langgraph_agent_toolkit/core/observability/empty.py

@@ -0,0 +1,67 @@
+from typing import Any, Dict, Literal, Optional
+
+from langgraph_agent_toolkit.core.observability.base import BaseObservabilityPlatform
+from langgraph_agent_toolkit.core.observability.types import PromptReturnType, PromptTemplateType
+
+
+class EmptyObservability(BaseObservabilityPlatform):
+    """Empty implementation of observability platform."""
+
+    __default_required_vars = []
+
+    def __init__(self, prompts_dir: Optional[str] = None):
+        """Initialize EmptyObservability.
+
+        Args:
+            prompts_dir: Optional directory to store prompts locally. If None, a system temp directory is used.
+
+        """
+        super().__init__(prompts_dir)
+
+    def get_callback_handler(self, **kwargs) -> None:
+        """Get the callback handler for the observability platform."""
+        return None
+
+    def before_shutdown(self) -> None:
+        """Perform any necessary cleanup before shutdown."""
+        pass
+
+    def record_feedback(self, run_id: str, key: str, score: float, **kwargs) -> None:
+        """Record feedback for a run with Empty observability platform."""
+        raise ValueError("Cannot record feedback: No observability platform is configured.")
+
+    def push_prompt(
+        self,
+        name: str,
+        prompt_template: PromptTemplateType,
+        metadata: Optional[Dict[str, Any]] = None,
+        create_new_version: bool = True,
+    ) -> None:
+        """Push a prompt using local storage.
+
+        Args:
+            name: Name of the prompt
+            prompt_template: String template, list of message dicts, or prompt object
+            metadata: Additional metadata for the prompt
+            create_new_version: If True, overwrite existing prompt with new version
+
+        """
+        super().push_prompt(name, prompt_template, metadata, create_new_version)
+
+    def pull_prompt(
+        self,
+        name: str,
+        template_format: Literal["f-string", "mustache", "jinja2"] = "f-string",
+        **kwargs,
+    ) -> PromptReturnType:
+        """Pull a prompt from local storage."""
+        return super().pull_prompt(name, template_format=template_format, **kwargs)
+
+    def delete_prompt(self, name: str) -> None:
+        """Delete a prompt from local storage.
+
+        Args:
+            name: Name of the prompt to delete
+
+        """
+        super().delete_prompt(name)

langgraph_agent_toolkit/core/observability/factory.py

@@ -0,0 +1,43 @@
+from typing import Optional, Union
+
+from langgraph_agent_toolkit.core.observability.base import BaseObservabilityPlatform
+from langgraph_agent_toolkit.core.observability.empty import EmptyObservability
+from langgraph_agent_toolkit.core.observability.langfuse import LangfuseObservability
+from langgraph_agent_toolkit.core.observability.langsmith import LangsmithObservability
+from langgraph_agent_toolkit.core.observability.types import ObservabilityBackend
+
+
+class ObservabilityFactory:
+    """Factory for creating observability platform instances."""
+
+    @staticmethod
+    def create(
+        platform: Union[ObservabilityBackend, str], prompts_dir: Optional[str] = None
+    ) -> BaseObservabilityPlatform:
+        """Create and return an observability platform instance.
+
+        Args:
+            platform: The observability platform to create
+            prompts_dir: Optional directory to store prompts locally
+
+        Returns:
+            An instance of the requested observability platform
+
+        Raises:
+            ValueError: If the requested platform is not supported
+
+        """
+        platform = ObservabilityBackend(platform)
+
+        match platform:
+            case ObservabilityBackend.LANGFUSE:
+                return LangfuseObservability(prompts_dir=prompts_dir)
+
+            case ObservabilityBackend.LANGSMITH:
+                return LangsmithObservability(prompts_dir=prompts_dir)
+
+            case ObservabilityBackend.EMPTY:
+                return EmptyObservability(prompts_dir=prompts_dir)
+
+            case _:
+                raise ValueError(f"Unsupported ObservabilityBackend: {platform}")

langgraph_agent_toolkit/core/observability/langfuse.py

@@ -0,0 +1,118 @@
+from typing import Any, Dict, Literal, Optional, Tuple, Union
+
+from langfuse import Langfuse
+from langfuse.callback import CallbackHandler
+
+from langgraph_agent_toolkit.core.observability.base import BaseObservabilityPlatform
+from langgraph_agent_toolkit.core.observability.types import PromptReturnType, PromptTemplateType
+from langgraph_agent_toolkit.helper.constants import DEFAULT_CACHE_TTL_SECOND
+from langgraph_agent_toolkit.helper.logging import logger
+
+
+class LangfuseObservability(BaseObservabilityPlatform):
+    """Langfuse implementation of observability platform."""
+
+    def __init__(self, prompts_dir: Optional[str] = None):
+        super().__init__(prompts_dir)
+        self.required_vars = ["LANGFUSE_SECRET_KEY", "LANGFUSE_PUBLIC_KEY", "LANGFUSE_HOST"]
+
+    @BaseObservabilityPlatform.requires_env_vars
+    def get_callback_handler(self, **kwargs) -> CallbackHandler:
+        return CallbackHandler(**kwargs)
+
+    def before_shutdown(self) -> None:
+        Langfuse().flush()
+
+    @BaseObservabilityPlatform.requires_env_vars
+    def record_feedback(self, run_id: str, key: str, score: float, **kwargs) -> None:
+        Langfuse().score(
+            trace_id=run_id,
+            name=key,
+            value=score,
+            **kwargs,
+        )
+
+    @BaseObservabilityPlatform.requires_env_vars
+    def push_prompt(
+        self,
+        name: str,
+        prompt_template: PromptTemplateType,
+        metadata: Optional[Dict[str, Any]] = None,
+        create_new_version: bool = True,
+    ) -> None:
+        langfuse = Langfuse()
+        labels = metadata.get("labels", ["production"]) if metadata else ["production"]
+
+        # Handle existing prompt versions - custom implementation for Langfuse
+        existing_prompt = None
+        if not create_new_version:
+            try:
+                existing_prompt = langfuse.get_prompt(name=name)
+                logger.debug(f"Using existing prompt '{name}' as create_new_version is False")
+            except Exception:
+                logger.debug(f"Existing prompt '{name}' not found, will create a new one")
+
+        prompt_obj = self._convert_to_chat_prompt(prompt_template)
+        type_prompt = "text" if isinstance(prompt_template, str) else "chat"
+
+        if existing_prompt:
+            langfuse_prompt = existing_prompt
+        else:
+            langfuse_prompt = langfuse.create_prompt(
+                name=name,
+                prompt=prompt_template,
+                labels=labels,
+                type=type_prompt,
+            )
+
+        full_metadata = metadata.copy() if metadata else {}
+        full_metadata["langfuse_prompt"] = langfuse_prompt
+        full_metadata["original_prompt"] = prompt_obj
+
+        super().push_prompt(name, prompt_template, full_metadata)
+
+    @BaseObservabilityPlatform.requires_env_vars
+    def pull_prompt(
+        self,
+        name: str,
+        return_with_prompt_object: bool = False,
+        cache_ttl_seconds: Optional[int] = DEFAULT_CACHE_TTL_SECOND,
+        template_format: Literal["f-string", "mustache", "jinja2"] = "f-string",
+        label: Optional[str] = None,
+        version: Optional[int] = None,
+        **kwargs,
+    ) -> Union[PromptReturnType, Tuple[PromptReturnType, Any]]:
+        try:
+            langfuse = Langfuse()
+            get_prompt_kwargs = {"name": name, "cache_ttl_seconds": cache_ttl_seconds}
+
+            if label:
+                get_prompt_kwargs["label"] = label
+            elif kwargs.get("prompt_label"):
+                get_prompt_kwargs["label"] = kwargs.get("prompt_label")
+
+            if version is not None:
+                get_prompt_kwargs["version"] = version
+            elif kwargs.get("prompt_version"):
+                get_prompt_kwargs["version"] = kwargs.get("prompt_version")
+
+            try:
+                langfuse_prompt = langfuse.get_prompt(**get_prompt_kwargs)
+            except Exception as e:
+                logger.debug(f"Prompt not found with parameters: {e}")
+                langfuse_prompt = langfuse.get_prompt(name=name, cache_ttl_seconds=cache_ttl_seconds)
+
+            # Process the prompt object using the base class helper
+            prompt = self._process_prompt_object(langfuse_prompt.prompt, template_format=template_format)
+
+            return (prompt, langfuse_prompt) if return_with_prompt_object else prompt
+
+        except Exception as e:
+            logger.warning(f"Failed to pull prompt from Langfuse: {e}")
+            local_prompt = super().pull_prompt(name, template_format=template_format, **kwargs)
+            return (local_prompt, None) if return_with_prompt_object else local_prompt
+
+    @BaseObservabilityPlatform.requires_env_vars
+    def delete_prompt(self, name: str) -> None:
+        logger.warning(f"Skipping deletion of prompt '{name}' from Langfuse")
+        super().delete_prompt(name)