freeplay 0.3.16__tar.gz → 0.3.18__tar.gz

{freeplay-0.3.16 → freeplay-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.16
+Version: 0.3.18
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.16 → freeplay-0.3.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.3.16"
+version = "0.3.18"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"

{freeplay-0.3.16 → freeplay-0.3.18}/src/freeplay/__init__.py

@@ -2,9 +2,11 @@ from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
 from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
+from .support import CustomMetadata
 
 __all__ = [
     'CallInfo',
+    'CustomMetadata',
     'Freeplay',
     'PromptInfo',
     'RecordPayload',

{freeplay-0.3.16 → freeplay-0.3.18}/src/freeplay/resources/prompts.py

@@ -1,16 +1,37 @@
 import copy
 import json
 import logging
+import warnings
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Dict, Optional, List, Sequence, cast, Any, Union, runtime_checkable, Protocol
-
-from freeplay.errors import FreeplayConfigurationError, FreeplayClientError, log_freeplay_client_warning
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Protocol,
+    Sequence,
+    TypedDict,
+    Union,
+    cast,
+    runtime_checkable,
+)
+
+from freeplay.errors import (
+    FreeplayClientError,
+    FreeplayConfigurationError,
+    log_freeplay_client_warning,
+)
 from freeplay.llm_parameters import LLMParameters
 from freeplay.model import InputVariables
-from freeplay.support import CallSupport, ToolSchema
-from freeplay.support import PromptTemplate, PromptTemplates, PromptTemplateMetadata
+from freeplay.support import (
+    CallSupport,
+    PromptTemplate,
+    PromptTemplateMetadata,
+    PromptTemplates,
+    ToolSchema,
+)
 from freeplay.utils import bind_template_variables, convert_provider_message_to_dict
 
 logger = logging.getLogger(__name__)
@@ -33,16 +54,30 @@ class UnsupportedToolSchemaError(FreeplayConfigurationError):
 
 # Models ==
 
-# A content block a la OpenAI or Anthropic. Intentionally over-permissive to allow schema evolution by the providers.
+# A content block compatible with stainless generated SDKs (such as Anthropic
+# and OpenAI). This lets us generate a dictionary from the stainless classes
+# correctly. Intentionally over-permissive to allow schema evolution by the
+# providers.
 @runtime_checkable
-class ProviderMessageContentBlock(Protocol):
+class ProviderMessageProtocol(Protocol):
     def model_dump(self) -> Dict[str, Any]:
         pass
 
 
-# A content/role pair with a type-safe content for common provider recording. If not using a common provider,
-# use {'content': str, 'role': str} to record. If using a common provider, this is usually the `.content` field.
-GenericProviderMessage = Union[Dict[str, Any], ProviderMessageContentBlock]
+class MessageDict(TypedDict):
+    role: str
+    content: Any
+
+
+# This type represents a struct or dict containing a role and content. The role
+#  should be one of user, assistant or system. This type should be compatible
+#  with OpenAI and Anthropic's message format, as well as most other SDKs. If
+#  not using a common provider, use {'content': str, 'role': str} to record. If
+#  using a common provider, this is usually the `.content` field.
+ProviderMessage = Union[MessageDict, Dict[str, Any], ProviderMessageProtocol]
+
+# DEPRECATED: Use ProviderMessage instead
+GenericProviderMessage = ProviderMessage
 
 
 # SDK-Exposed Classes
@@ -69,7 +104,7 @@ class FormattedPrompt:
             formatted_prompt_text: Optional[str] = None,
             tool_schema: Optional[List[Dict[str, Any]]] = None
     ):
-        # These two definitions allow us to operate on typed fields unitl we expose them as Any for client use.
+        # These two definitions allow us to operate on typed fields until we expose them as Any for client use.
         self._llm_prompt = formatted_prompt
         self._tool_schema = tool_schema
 
@@ -81,11 +116,18 @@ class FormattedPrompt:
             (message['content'] for message in messages if message['role'] == 'system'), None)
         self.system_content = maybe_system_content
 
-        # Note: messages are **not formatted** for the provider.
-        self.messages = messages
+        self._messages = messages
+
+    @property
+    def messages(self) -> List[Dict[str, str]]:
+        warnings.warn(
+            "The 'messages' attribute is deprecated and will be removed in a future version. It is not formatted for the provider. Use 'llm_prompt' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._messages
 
     @property
-    # We know this is a list of dict[str,str], but we use Any to avoid typing issues with client SDK libraries, which require strict TypedDict.
     def llm_prompt(self) -> Any:
         return self._llm_prompt
 
@@ -93,12 +135,9 @@ class FormattedPrompt:
     def tool_schema(self) -> Any:
         return self._tool_schema
 
-    def all_messages(
-            self,
-            new_message: GenericProviderMessage
-    ) -> List[Dict[str, Any]]:
+    def all_messages(self, new_message: ProviderMessage) -> List[Dict[str, Any]]:
         converted_message = convert_provider_message_to_dict(new_message)
-        return self.messages + [converted_message]
+        return self._messages + [converted_message]
 
 
 class BoundPrompt:
@@ -117,7 +156,13 @@ class BoundPrompt:
             flavor_name: str,
             messages: List[Dict[str, str]]
     ) -> Union[str, List[Dict[str, str]]]:
-        if flavor_name in ['azure_openai_chat', 'openai_chat', 'baseten_mistral_chat', 'mistral_chat']:
+        if flavor_name in [
+            'azure_openai_chat',
+            'openai_chat',
+            'baseten_mistral_chat',
+            'mistral_chat',
+            'perplexity_chat'
+        ]:
             # We need a deepcopy here to avoid referential equality with the llm_prompt
             return copy.deepcopy(messages)
         elif flavor_name == 'anthropic_chat':
@@ -211,7 +256,11 @@ class TemplatePrompt:
         self.tool_schema = tool_schema
         self.messages = messages
 
-    def bind(self, variables: InputVariables, history: Optional[Sequence[GenericProviderMessage]] = None) -> BoundPrompt:
+    def bind(
+        self,
+        variables: InputVariables,
+        history: Optional[Sequence[ProviderMessage]] = None,
+    ) -> BoundPrompt:
         # check history for a system message
         history_clean = []
         if history:
@@ -521,13 +570,13 @@ class Prompts:
         return TemplatePrompt(prompt_info, prompt.content, prompt.tool_schema)
 
     def get_formatted(
-            self,
-            project_id: str,
-            template_name: str,
-            environment: str,
-            variables: InputVariables,
-            history: Optional[Sequence[GenericProviderMessage]] = None,
-            flavor_name: Optional[str] = None
+        self,
+        project_id: str,
+        template_name: str,
+        environment: str,
+        variables: InputVariables,
+        history: Optional[Sequence[ProviderMessage]] = None,
+        flavor_name: Optional[str] = None,
     ) -> FormattedPrompt:
         bound_prompt = self.get(
             project_id=project_id,

{freeplay-0.3.16 → freeplay-0.3.18}/src/freeplay/resources/sessions.py

@@ -3,9 +3,7 @@ from dataclasses import dataclass
 from typing import Optional, Dict, Union
 
 from freeplay.errors import FreeplayClientError
-from freeplay.support import CallSupport
-
-CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
+from freeplay.support import CallSupport, CustomMetadata
 
 
 @dataclass
@@ -18,6 +16,8 @@ class TraceInfo:
     session_id: str
     trace_id: str
     input: Optional[str] = None
+    agent_name: Optional[str] = None
+    custom_metadata: CustomMetadata = None
     _call_support: CallSupport
 
     def __init__(
@@ -26,16 +26,34 @@ class TraceInfo:
             session_id: str,
             _call_support: CallSupport,
             input: Optional[str] = None,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None,
     ):
         self.trace_id = trace_id
         self.session_id = session_id
         self.input = input
+        self.agent_name = agent_name
+        self.custom_metadata = custom_metadata
         self._call_support = _call_support
 
-    def record_output(self, project_id: str, output: str) -> None:
+    def record_output(
+            self,
+            project_id: str,
+            output: str,
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+    ) -> None:
         if self.input is None:
             raise FreeplayClientError("Input must be set before recording output")
-        self._call_support.record_trace(project_id, self.session_id, self.trace_id, self.input, output)
+        self._call_support.record_trace(
+            project_id,
+            self.session_id,
+            self.trace_id,
+            self.input,
+            output,
+            agent_name=self.agent_name,
+            custom_metadata=self.custom_metadata,
+            eval_results=eval_results
+        )
 
 
 @dataclass
@@ -53,19 +71,34 @@ class Session:
     def session_info(self) -> SessionInfo:
         return self._session_info
 
-    def create_trace(self, input: str) -> TraceInfo:
+    def create_trace(
+            self,
+            input: str,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None
+    ) -> TraceInfo:
         return TraceInfo(
             trace_id=str(uuid.uuid4()),
             session_id=self.session_id,
             input=input,
+            agent_name=agent_name,
+            custom_metadata=custom_metadata,
             _call_support=self._call_support
         )
 
-    def restore_trace(self, trace_id: uuid.UUID, input: Optional[str]) -> TraceInfo:
+    def restore_trace(
+            self,
+            trace_id: uuid.UUID,
+            input: Optional[str],
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None
+    ) -> TraceInfo:
         return TraceInfo(
             trace_id=str(trace_id),
             session_id=self.session_id,
             input=input,
+            agent_name=agent_name,
+            custom_metadata=custom_metadata,
             _call_support=self._call_support
         )
 

{freeplay-0.3.16 → freeplay-0.3.18}/src/freeplay/support.py

@@ -1,4 +1,3 @@
-import json
 from dataclasses import dataclass
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union
@@ -8,6 +7,8 @@ from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
 from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
 
+CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
+
 
 @dataclass
 class PromptTemplateMetadata:
@@ -17,12 +18,14 @@ class PromptTemplateMetadata:
     params: Optional[Dict[str, Any]] = None
     provider_info: Optional[Dict[str, Any]] = None
 
+
 @dataclass
 class ToolSchema:
     name: str
     description: str
     parameters: Dict[str, Any]
 
+
 @dataclass
 class PromptTemplate:
     prompt_template_id: str
@@ -40,6 +43,7 @@ class PromptTemplate:
 class PromptTemplates:
     prompt_templates: List[PromptTemplate]
 
+
 @dataclass
 class SummaryStatistics:
     auto_evaluation: Dict[str, Any]
@@ -88,8 +92,10 @@ class TestRunRetrievalResponse:
             human_evaluation=summary_statistics['human_evaluation']
         )
 
+
 class DatasetTestCaseRequest:
-    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables, metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
+    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables,
+                 metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
         self.history: Optional[List[NormalizedMessage]] = history
         self.inputs: InputVariables = inputs
         self.metadata: Optional[Dict[str, str]] = metadata
@@ -104,6 +110,7 @@ class DatasetTestCaseResponse:
         self.history: Optional[List[NormalizedMessage]] = test_case.get('history')
         self.metadata: Optional[Dict[str, str]] = test_case.get('metadata')
 
+
 class DatasetTestCasesRetrievalResponse:
     def __init__(self, test_cases: List[Dict[str, Any]]) -> None:
         self.test_cases = [
@@ -111,6 +118,7 @@ class DatasetTestCasesRetrievalResponse:
             for test_case in test_cases
         ]
 
+
 class CallSupport:
     def __init__(
             self,
@@ -256,13 +264,26 @@ class CallSupport:
             summary_statistics=json_dom['summary_statistics']
         )
 
-    def record_trace(self, project_id: str, session_id: str, trace_id: str, input: str, output: str) -> None:
+    def record_trace(
+            self,
+            project_id: str,
+            session_id: str,
+            trace_id: str,
+            input: str,
+            output: str,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None,
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+    ) -> None:
         response = api_support.post_raw(
             self.freeplay_api_key,
             f'{self.api_base}/v2/projects/{project_id}/sessions/{session_id}/traces/id/{trace_id}',
             {
+                'agent_name': agent_name,
                 'input': input,
-                'output': output
+                'output': output,
+                'custom_metadata': custom_metadata,
+                'eval_results': eval_results,
             }
         )
         if response.status_code != 201:
@@ -277,7 +298,13 @@ class CallSupport:
             raise freeplay_response_error('Error while deleting session.', response)
 
     def create_test_cases(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCaseRequest]) -> None:
-        examples = [{"history": test_case.history, "output": test_case.output, "metadata": test_case.metadata, "inputs": test_case.inputs} for test_case in test_cases]
+        examples = [
+            {
+                "history": test_case.history,
+                "output": test_case.output,
+                "metadata": test_case.metadata,
+                "inputs": test_case.inputs
+            } for test_case in test_cases]
         payload: Dict[str, Any] = {"examples": examples}
         url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
 
@@ -295,5 +322,13 @@ class CallSupport:
         json_dom = response.json()
 
         return DatasetTestCasesRetrievalResponse(
-            test_cases=[{"history": jsn["history"], "id": jsn["id"], "output": jsn["output"], "values": jsn["values"], "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None} for jsn in json_dom]
+            test_cases=[
+                {
+                    "history": jsn["history"],
+                    "id": jsn["id"],
+                    "output": jsn["output"],
+                    "values": jsn["values"],
+                    "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None
+                } for jsn in json_dom
+            ]
         )