freeplay 0.3.15__py3-none-any.whl → 0.3.17__py3-none-any.whl

freeplay/__init__.py

@@ -2,9 +2,11 @@ from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
 from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
+from .support import CustomMetadata
 
 __all__ = [
     'CallInfo',
+    'CustomMetadata',
     'Freeplay',
     'PromptInfo',
     'RecordPayload',

freeplay/resources/prompts.py

@@ -1,16 +1,37 @@
 import copy
 import json
 import logging
+import warnings
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Dict, Optional, List, Sequence, cast, Any, Union, runtime_checkable, Protocol
-
-from freeplay.errors import FreeplayConfigurationError, FreeplayClientError, log_freeplay_client_warning
+from typing import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Protocol,
+    Sequence,
+    TypedDict,
+    Union,
+    cast,
+    runtime_checkable,
+)
+
+from freeplay.errors import (
+    FreeplayClientError,
+    FreeplayConfigurationError,
+    log_freeplay_client_warning,
+)
 from freeplay.llm_parameters import LLMParameters
 from freeplay.model import InputVariables
-from freeplay.support import CallSupport, ToolSchema
-from freeplay.support import PromptTemplate, PromptTemplates, PromptTemplateMetadata
+from freeplay.support import (
+    CallSupport,
+    PromptTemplate,
+    PromptTemplateMetadata,
+    PromptTemplates,
+    ToolSchema,
+)
 from freeplay.utils import bind_template_variables, convert_provider_message_to_dict
 
 logger = logging.getLogger(__name__)
@@ -33,16 +54,30 @@ class UnsupportedToolSchemaError(FreeplayConfigurationError):
 
 # Models ==
 
-# A content block a la OpenAI or Anthropic. Intentionally over-permissive to allow schema evolution by the providers.
+# A content block compatible with stainless generated SDKs (such as Anthropic
+# and OpenAI). This lets us generate a dictionary from the stainless classes
+# correctly. Intentionally over-permissive to allow schema evolution by the
+# providers.
 @runtime_checkable
-class ProviderMessageContentBlock(Protocol):
+class ProviderMessageProtocol(Protocol):
     def model_dump(self) -> Dict[str, Any]:
         pass
 
 
-# A content/role pair with a type-safe content for common provider recording. If not using a common provider,
-# use {'content': str, 'role': str} to record. If using a common provider, this is usually the `.content` field.
-GenericProviderMessage = Union[Dict[str, Any], ProviderMessageContentBlock]
+class MessageDict(TypedDict):
+    role: str
+    content: Any
+
+
+# This type represents a struct or dict containing a role and content. The role
+#  should be one of user, assistant or system. This type should be compatible
+#  with OpenAI and Anthropic's message format, as well as most other SDKs. If
+#  not using a common provider, use {'content': str, 'role': str} to record. If
+#  using a common provider, this is usually the `.content` field.
+ProviderMessage = Union[MessageDict, Dict[str, Any], ProviderMessageProtocol]
+
+# DEPRECATED: Use ProviderMessage instead
+GenericProviderMessage = ProviderMessage
 
 
 # SDK-Exposed Classes
@@ -69,7 +104,7 @@ class FormattedPrompt:
             formatted_prompt_text: Optional[str] = None,
             tool_schema: Optional[List[Dict[str, Any]]] = None
     ):
-        # These two definitions allow us to operate on typed fields unitl we expose them as Any for client use.
+        # These two definitions allow us to operate on typed fields until we expose them as Any for client use.
         self._llm_prompt = formatted_prompt
         self._tool_schema = tool_schema
 
@@ -81,11 +116,18 @@ class FormattedPrompt:
             (message['content'] for message in messages if message['role'] == 'system'), None)
         self.system_content = maybe_system_content
 
-        # Note: messages are **not formatted** for the provider.
-        self.messages = messages
+        self._messages = messages
+
+    @property
+    def messages(self) -> List[Dict[str, str]]:
+        warnings.warn(
+            "The 'messages' attribute is deprecated and will be removed in a future version. It is not formatted for the provider. Use 'llm_prompt' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._messages
 
     @property
-    # We know this is a list of dict[str,str], but we use Any to avoid typing issues with client SDK libraries, which require strict TypedDict.
     def llm_prompt(self) -> Any:
         return self._llm_prompt
 
@@ -93,12 +135,9 @@ class FormattedPrompt:
     def tool_schema(self) -> Any:
         return self._tool_schema
 
-    def all_messages(
-            self,
-            new_message: GenericProviderMessage
-    ) -> List[Dict[str, Any]]:
+    def all_messages(self, new_message: ProviderMessage) -> List[Dict[str, Any]]:
         converted_message = convert_provider_message_to_dict(new_message)
-        return self.messages + [converted_message]
+        return self._messages + [converted_message]
 
 
 class BoundPrompt:
@@ -211,7 +250,11 @@ class TemplatePrompt:
         self.tool_schema = tool_schema
         self.messages = messages
 
-    def bind(self, variables: InputVariables, history: Optional[Sequence[GenericProviderMessage]] = None) -> BoundPrompt:
+    def bind(
+        self,
+        variables: InputVariables,
+        history: Optional[Sequence[ProviderMessage]] = None,
+    ) -> BoundPrompt:
         # check history for a system message
         history_clean = []
         if history:
@@ -521,13 +564,13 @@ class Prompts:
         return TemplatePrompt(prompt_info, prompt.content, prompt.tool_schema)
 
     def get_formatted(
-            self,
-            project_id: str,
-            template_name: str,
-            environment: str,
-            variables: InputVariables,
-            history: Optional[Sequence[GenericProviderMessage]] = None,
-            flavor_name: Optional[str] = None
+        self,
+        project_id: str,
+        template_name: str,
+        environment: str,
+        variables: InputVariables,
+        history: Optional[Sequence[ProviderMessage]] = None,
+        flavor_name: Optional[str] = None,
     ) -> FormattedPrompt:
         bound_prompt = self.get(
             project_id=project_id,

freeplay/resources/recordings.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Literal
 from uuid import UUID
 
 from requests import HTTPError
@@ -23,6 +23,9 @@ class UsageTokens:
     completion_tokens: int
 
 
+ApiStyle = Union[Literal['batch'], Literal['default']]
+
+
 @dataclass
 class CallInfo:
     provider: str
@@ -32,13 +35,15 @@ class CallInfo:
     model_parameters: LLMParameters
     provider_info: Optional[Dict[str, Any]] = None
     usage: Optional[UsageTokens] = None
+    api_style: Optional[ApiStyle] = None
 
     @staticmethod
     def from_prompt_info(
             prompt_info: PromptInfo,
             start_time: float,
             end_time: float,
-            usage: Optional[UsageTokens] = None
+            usage: Optional[UsageTokens] = None,
+            api_style: Optional[ApiStyle] = None
     ) -> 'CallInfo':
         return CallInfo(
             provider=prompt_info.provider,
@@ -47,7 +52,8 @@ class CallInfo:
             end_time=end_time,
             model_parameters=prompt_info.model_parameters,
             provider_info=prompt_info.provider_info,
-            usage=usage
+            usage=usage,
+            api_style=api_style
         )
 
 
@@ -157,6 +163,9 @@ class Recordings:
                 "completion_tokens": record_payload.call_info.usage.completion_tokens,
             }
 
+        if record_payload.call_info.api_style is not None:
+            record_api_payload['call_info']['api_style'] = record_payload.call_info.api_style
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.call_support.freeplay_api_key,

freeplay/resources/sessions.py

@@ -3,9 +3,7 @@ from dataclasses import dataclass
 from typing import Optional, Dict, Union
 
 from freeplay.errors import FreeplayClientError
-from freeplay.support import CallSupport
-
-CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
+from freeplay.support import CallSupport, CustomMetadata
 
 
 @dataclass
@@ -18,6 +16,8 @@ class TraceInfo:
     session_id: str
     trace_id: str
     input: Optional[str] = None
+    agent_name: Optional[str] = None
+    custom_metadata: CustomMetadata = None
     _call_support: CallSupport
 
     def __init__(
@@ -26,16 +26,34 @@ class TraceInfo:
             session_id: str,
             _call_support: CallSupport,
             input: Optional[str] = None,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None,
     ):
         self.trace_id = trace_id
         self.session_id = session_id
         self.input = input
+        self.agent_name = agent_name
+        self.custom_metadata = custom_metadata
         self._call_support = _call_support
 
-    def record_output(self, project_id: str, output: str) -> None:
+    def record_output(
+            self,
+            project_id: str,
+            output: str,
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+    ) -> None:
         if self.input is None:
             raise FreeplayClientError("Input must be set before recording output")
-        self._call_support.record_trace(project_id, self.session_id, self.trace_id, self.input, output)
+        self._call_support.record_trace(
+            project_id,
+            self.session_id,
+            self.trace_id,
+            self.input,
+            output,
+            agent_name=self.agent_name,
+            custom_metadata=self.custom_metadata,
+            eval_results=eval_results
+        )
 
 
 @dataclass
@@ -53,19 +71,34 @@ class Session:
     def session_info(self) -> SessionInfo:
         return self._session_info
 
-    def create_trace(self, input: str) -> TraceInfo:
+    def create_trace(
+            self,
+            input: str,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None
+    ) -> TraceInfo:
         return TraceInfo(
             trace_id=str(uuid.uuid4()),
             session_id=self.session_id,
             input=input,
+            agent_name=agent_name,
+            custom_metadata=custom_metadata,
             _call_support=self._call_support
         )
 
-    def restore_trace(self, trace_id: uuid.UUID, input: Optional[str]) -> TraceInfo:
+    def restore_trace(
+            self,
+            trace_id: uuid.UUID,
+            input: Optional[str],
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None
+    ) -> TraceInfo:
         return TraceInfo(
             trace_id=str(trace_id),
             session_id=self.session_id,
             input=input,
+            agent_name=agent_name,
+            custom_metadata=custom_metadata,
             _call_support=self._call_support
         )
 

freeplay/support.py

@@ -1,4 +1,3 @@
-import json
 from dataclasses import dataclass
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union
@@ -8,6 +7,8 @@ from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
 from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
 
+CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
+
 
 @dataclass
 class PromptTemplateMetadata:
@@ -17,12 +18,14 @@ class PromptTemplateMetadata:
     params: Optional[Dict[str, Any]] = None
     provider_info: Optional[Dict[str, Any]] = None
 
+
 @dataclass
 class ToolSchema:
     name: str
     description: str
     parameters: Dict[str, Any]
 
+
 @dataclass
 class PromptTemplate:
     prompt_template_id: str
@@ -40,6 +43,7 @@ class PromptTemplate:
 class PromptTemplates:
     prompt_templates: List[PromptTemplate]
 
+
 @dataclass
 class SummaryStatistics:
     auto_evaluation: Dict[str, Any]
@@ -88,8 +92,10 @@ class TestRunRetrievalResponse:
             human_evaluation=summary_statistics['human_evaluation']
         )
 
+
 class DatasetTestCaseRequest:
-    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables, metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
+    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables,
+                 metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
         self.history: Optional[List[NormalizedMessage]] = history
         self.inputs: InputVariables = inputs
         self.metadata: Optional[Dict[str, str]] = metadata
@@ -104,6 +110,7 @@ class DatasetTestCaseResponse:
         self.history: Optional[List[NormalizedMessage]] = test_case.get('history')
         self.metadata: Optional[Dict[str, str]] = test_case.get('metadata')
 
+
 class DatasetTestCasesRetrievalResponse:
     def __init__(self, test_cases: List[Dict[str, Any]]) -> None:
         self.test_cases = [
@@ -111,6 +118,7 @@ class DatasetTestCasesRetrievalResponse:
             for test_case in test_cases
         ]
 
+
 class CallSupport:
     def __init__(
             self,
@@ -256,13 +264,26 @@ class CallSupport:
             summary_statistics=json_dom['summary_statistics']
         )
 
-    def record_trace(self, project_id: str, session_id: str, trace_id: str, input: str, output: str) -> None:
+    def record_trace(
+            self,
+            project_id: str,
+            session_id: str,
+            trace_id: str,
+            input: str,
+            output: str,
+            agent_name: Optional[str] = None,
+            custom_metadata: CustomMetadata = None,
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+    ) -> None:
         response = api_support.post_raw(
             self.freeplay_api_key,
             f'{self.api_base}/v2/projects/{project_id}/sessions/{session_id}/traces/id/{trace_id}',
             {
+                'agent_name': agent_name,
                 'input': input,
-                'output': output
+                'output': output,
+                'custom_metadata': custom_metadata,
+                'eval_results': eval_results,
             }
         )
         if response.status_code != 201:
@@ -277,7 +298,13 @@ class CallSupport:
             raise freeplay_response_error('Error while deleting session.', response)
 
     def create_test_cases(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCaseRequest]) -> None:
-        examples = [{"history": test_case.history, "output": test_case.output, "metadata": test_case.metadata, "inputs": test_case.inputs} for test_case in test_cases]
+        examples = [
+            {
+                "history": test_case.history,
+                "output": test_case.output,
+                "metadata": test_case.metadata,
+                "inputs": test_case.inputs
+            } for test_case in test_cases]
         payload: Dict[str, Any] = {"examples": examples}
         url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
 
@@ -295,5 +322,13 @@ class CallSupport:
         json_dom = response.json()
 
         return DatasetTestCasesRetrievalResponse(
-            test_cases=[{"history": jsn["history"], "id": jsn["id"], "output": jsn["output"], "values": jsn["values"], "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None} for jsn in json_dom]
+            test_cases=[
+                {
+                    "history": jsn["history"],
+                    "id": jsn["id"],
+                    "output": jsn["output"],
+                    "values": jsn["values"],
+                    "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None
+                } for jsn in json_dom
+            ]
         )

{freeplay-0.3.15.dist-info → freeplay-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.15
+Version: 0.3.17
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.15.dist-info → freeplay-0.3.17.dist-info}/RECORD

@@ -1,4 +1,4 @@
-freeplay/__init__.py,sha256=fipXoYuxUtsy6mxshut2ujWJh4lVbQ5v_xQ-33TPC-g,406
+freeplay/__init__.py,sha256=tzEFgRyriPBWPo04jTgR6MxzXUqIHBlUJZvuBj5kFiE,464
 freeplay/api_support.py,sha256=Kn2x3g6yloHQl3NwFRjbZE9BnIh7d1sgwGwC0mHuvw4,2483
 freeplay/errors.py,sha256=vwotUBldxDzREZOmLUeoiDoZjcvDwgH1AMwKBLhLooE,807
 freeplay/freeplay.py,sha256=J04-erDD6rI2SAje_Nsf3x5Qx-Z6p8gQvGrMRHFWoD4,1602
@@ -8,15 +8,15 @@ freeplay/model.py,sha256=o0de_RZ2WTJ4m5OJw1ZVfC2xG6zBq_XShBrRt1laEjc,1405
 freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 freeplay/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 freeplay/resources/customer_feedback.py,sha256=bw8MfEOKbGgn4FOyvcADrcs9GhcpNXNTgxKjBjIzywE,899
-freeplay/resources/prompts.py,sha256=NZi4K6oGnbSgw_i0NFssSqRNonl6Ov8eGPFFbZ6O5aI,22185
-freeplay/resources/recordings.py,sha256=PhwdVJaEhc_-xrtu5CPXoKRVRSBD6uqwIhTJrDCOBro,8144
-freeplay/resources/sessions.py,sha256=Qz5v7VOf1DmQTd1wCOFXnrizlW5WFJT5V8-pq22Ifvg,2793
+freeplay/resources/prompts.py,sha256=fkNY5mg2Iib36e0sUVr3g-btr7bYdgQL9Ug6_f-OGi0,22790
+freeplay/resources/recordings.py,sha256=uslhoKWsXq51sAbF1lBs2uGL14qvBGc3dM8ZL6-H_04,8488
+freeplay/resources/sessions.py,sha256=J5A3CjiV2MFqQyxN3TWTvJaa9jmMza58mRFRq2v9iAk,3746
 freeplay/resources/test_cases.py,sha256=nXL_976RwSJDT6OWDM4GEzbcOzcGkJ9ulvb0XOzCRDM,2240
 freeplay/resources/test_runs.py,sha256=Tp2N-odInT5XEEWrEsVhdgfnsclOE8n92_C8gTwO2MI,2623
-freeplay/support.py,sha256=we_FEtxcqc-8R0uOWy8p0nX0pHUbs-ulw7TC5NarlX4,11091
+freeplay/support.py,sha256=royOK31X-TtluQQn_XG2xFoINV8GZAy3FRRO_P763NU,11800
 freeplay/utils.py,sha256=Xvt4mNLXLL7E6MI2hTuDLV5cl5Y83DgdjCZSyDGMjR0,3187
-freeplay-0.3.15.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.3.15.dist-info/METADATA,sha256=YVeTKAO32MXniQ9UVVigz4Yiyu09xFLHvEAXCISnjNs,1654
-freeplay-0.3.15.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-freeplay-0.3.15.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.3.15.dist-info/RECORD,,
+freeplay-0.3.17.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.3.17.dist-info/METADATA,sha256=lIB2GCeusLys0Dza7RxasOgbpxIRJiZAPpx5DeKMXKs,1654
+freeplay-0.3.17.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+freeplay-0.3.17.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.3.17.dist-info/RECORD,,