freeplay 0.3.12__py3-none-any.whl → 0.3.15__py3-none-any.whl

freeplay/__init__.py

@@ -1,6 +1,6 @@
 from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
-from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo
+from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
 
 __all__ = [
@@ -12,4 +12,5 @@ __all__ = [
     'SessionInfo',
     'TestRunInfo',
     'TraceInfo',
+    'UsageTokens',
 ]

freeplay/freeplay.py

@@ -5,6 +5,7 @@ from freeplay.resources.customer_feedback import CustomerFeedback
 from freeplay.resources.prompts import Prompts, APITemplateResolver, TemplateResolver
 from freeplay.resources.recordings import Recordings
 from freeplay.resources.sessions import Sessions
+from freeplay.resources.test_cases import TestCases
 from freeplay.resources.test_runs import TestRuns
 from freeplay.support import CallSupport
 
@@ -38,3 +39,4 @@ class Freeplay:
         self.recordings = Recordings(self.call_support)
         self.sessions = Sessions(self.call_support)
         self.test_runs = TestRuns(self.call_support)
+        self.test_cases = TestCases(self.call_support)

freeplay/model.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Union, Any, Dict, Mapping, TypedDict
+from typing import List, Union, Any, Dict, Mapping, TypedDict, Literal
 
 InputValue = Union[str, int, bool, float, Dict[str, Any], List[Any]]
 InputVariables = Mapping[str, InputValue]
@@ -16,3 +16,49 @@ class TestRun:
 class OpenAIFunctionCall(TypedDict):
     name: str
     arguments: str
+
+
+@dataclass
+class TextBlock:
+    text: str
+    type: Literal["text"] = "text"
+
+
+@dataclass
+class ToolResultBlock:
+    # AKA tool_use_id -- the ID of the tool call that this message is responding to.
+    tool_call_id: str
+    content: Union[str, List[TextBlock]]
+    type: Literal["tool_result"] = "tool_result"
+
+
+@dataclass
+class ToolCallBlock:
+    id: str
+    name: str
+    arguments: Any
+    type: Literal["tool_call"] = "tool_call"
+
+
+ContentBlock = Union[TextBlock, ToolResultBlock, ToolCallBlock]
+
+
+@dataclass
+class UserMessage:
+    content: Union[str, List[ContentBlock]]
+    role: Literal["user"] = "user"
+
+
+@dataclass
+class SystemMessage:
+    content: str
+    role: Literal["system"] = "system"
+
+
+@dataclass
+class AssistantMessage:
+    content: Union[str, List[ContentBlock]]
+    role: Literal["assistant"] = "assistant"
+
+# Largely used for history in dataset test cases presently
+NormalizedMessage = Union[UserMessage, SystemMessage, AssistantMessage]

freeplay/resources/prompts.py

@@ -35,14 +35,14 @@ class UnsupportedToolSchemaError(FreeplayConfigurationError):
 
 # A content block a la OpenAI or Anthropic. Intentionally over-permissive to allow schema evolution by the providers.
 @runtime_checkable
-class ContentBlock(Protocol):
+class ProviderMessageContentBlock(Protocol):
     def model_dump(self) -> Dict[str, Any]:
         pass
 
 
 # A content/role pair with a type-safe content for common provider recording. If not using a common provider,
 # use {'content': str, 'role': str} to record. If using a common provider, this is usually the `.content` field.
-GenericProviderMessage = Union[Dict[str, Any], ContentBlock]
+GenericProviderMessage = Union[Dict[str, Any], ProviderMessageContentBlock]
 
 
 # SDK-Exposed Classes

freeplay/resources/recordings.py

@@ -17,6 +17,12 @@ from freeplay.support import CallSupport
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class UsageTokens:
+    prompt_tokens: int
+    completion_tokens: int
+
+
 @dataclass
 class CallInfo:
     provider: str
@@ -25,9 +31,15 @@ class CallInfo:
     end_time: float
     model_parameters: LLMParameters
     provider_info: Optional[Dict[str, Any]] = None
+    usage: Optional[UsageTokens] = None
 
     @staticmethod
-    def from_prompt_info(prompt_info: PromptInfo, start_time: float, end_time: float) -> 'CallInfo':
+    def from_prompt_info(
+            prompt_info: PromptInfo,
+            start_time: float,
+            end_time: float,
+            usage: Optional[UsageTokens] = None
+    ) -> 'CallInfo':
         return CallInfo(
             provider=prompt_info.provider,
             model=prompt_info.model,
@@ -35,6 +47,7 @@ class CallInfo:
             end_time=end_time,
             model_parameters=prompt_info.model_parameters,
             provider_info=prompt_info.provider_info,
+            usage=usage
         )
 
 
@@ -68,6 +81,14 @@ class RecordPayload:
     completion_id: Optional[UUID] = None
 
 
+@dataclass
+class RecordUpdatePayload:
+    project_id: str
+    completion_id: str
+    new_messages: Optional[List[Dict[str, Any]]] = None
+    eval_results: Optional[Dict[str, Union[bool, float]]] = None
+
+
 @dataclass
 class RecordResponse:
     completion_id: str
@@ -77,7 +98,7 @@ class Recordings:
     def __init__(self, call_support: CallSupport):
         self.call_support = call_support
 
-    def create(self, record_payload: RecordPayload) -> RecordResponse:
+    def create(self, record_payload: RecordPayload) -> RecordResponse:  # type: ignore
         if len(record_payload.all_messages) < 1:
             raise FreeplayClientError("Messages list must have at least one message. "
                                       "The last message should be the current response.")
@@ -130,6 +151,12 @@ class Recordings:
                 "trace_id": record_payload.trace_info.trace_id
             }
 
+        if record_payload.call_info.usage is not None:
+            record_api_payload['call_info']['usage'] = {
+                "prompt_tokens": record_payload.call_info.usage.prompt_tokens,
+                "completion_tokens": record_payload.call_info.usage.completion_tokens,
+            }
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.call_support.freeplay_api_key,
@@ -143,18 +170,7 @@ class Recordings:
             message = f'There was an error recording to Freeplay. Call will not be logged. ' \
                       f'Status: {e.response.status_code}. '
 
-            if e.response.content:
-                try:
-                    content = e.response.content
-                    json_body = json.loads(content)
-                    if 'message' in json_body:
-                        message += json_body['message']
-                except:
-                    pass
-            else:
-                message += f'{e.__class__}'
-
-            raise FreeplayError(message) from e
+            self.__handle_and_raise_api_error(e, message)
 
         except Exception as e:
             status_code = -1
@@ -165,3 +181,36 @@ class Recordings:
                       f'Status: {status_code}. {e.__class__}'
 
             raise FreeplayError(message) from e
+
+    def update(self, record_update_payload: RecordUpdatePayload) -> RecordResponse:  # type: ignore
+        record_update_api_payload: Dict[str, Any] = {
+            "new_messages": record_update_payload.new_messages,
+            "eval_results": record_update_payload.eval_results,
+        }
+
+        try:
+            record_update_response = api_support.post_raw(
+                api_key=self.call_support.freeplay_api_key,
+                url=f'{self.call_support.api_base}/v2/projects/{record_update_payload.project_id}/completions/{record_update_payload.completion_id}',
+                payload=record_update_api_payload
+            )
+            record_update_response.raise_for_status()
+            json_dom = record_update_response.json()
+            return RecordResponse(completion_id=str(json_dom['completion_id']))
+        except HTTPError as e:
+            message = f'There was an error updating the completion. Status: {e.response.status_code}.'
+            self.__handle_and_raise_api_error(e, message)
+
+    @staticmethod
+    def __handle_and_raise_api_error(e: HTTPError, messages: str) -> None:
+        if e.response.content:
+            try:
+                content = e.response.content
+                json_body = json.loads(content)
+                if 'message' in json_body:
+                    messages += json_body['message']
+            except:
+                pass
+        else:
+            messages += f'{e.__class__}'
+        raise FreeplayError(messages) from e

freeplay/resources/test_cases.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from typing import List, Optional, Dict, Any
+
+from freeplay.model import InputVariables, NormalizedMessage
+from freeplay.support import CallSupport, DatasetTestCaseRequest, DatasetTestCasesRetrievalResponse
+
+
+@dataclass
+class DatasetTestCase:
+    def __init__(
+            self,
+            inputs: InputVariables,
+            output: Optional[str],
+            history: Optional[List[NormalizedMessage]] = None,
+            metadata: Optional[Dict[str, str]] = None,
+            id: Optional[str] = None, # Only set on retrieval
+    ):
+        self.inputs = inputs
+        self.output = output
+        self.history = history
+        self.metadata = metadata
+        self.id = id
+
+
+
+@dataclass
+class Dataset:
+    def __init__(self, dataset_id: str, test_cases: List[DatasetTestCase]):
+        self.dataset_id = dataset_id
+        self.test_cases = test_cases
+
+
+@dataclass
+class DatasetResults:
+    def __init__(self, dataset_id: str, test_cases: List[DatasetTestCase]) -> None:
+        self.dataset_id = dataset_id
+        self.test_cases = test_cases
+
+class TestCases:
+    def __init__(self, call_support: CallSupport) -> None:
+        self.call_support = call_support
+
+    def create(self, project_id: str, dataset_id: str, test_case: DatasetTestCase) -> Dataset:
+        return self.create_many(project_id, dataset_id, [test_case])
+
+    def create_many(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCase]) -> Dataset:
+        dataset_test_cases = [DatasetTestCaseRequest(test_case.history, test_case.inputs, test_case.metadata, test_case.output) for test_case in test_cases]
+        self.call_support.create_test_cases(project_id, dataset_id, dataset_test_cases)
+        return Dataset(dataset_id, test_cases)
+
+    def get(self, project_id: str, dataset_id: str) -> DatasetResults:
+        test_case_results: DatasetTestCasesRetrievalResponse = self.call_support.get_test_cases(project_id, dataset_id)
+        dataset_test_cases = test_case_results.test_cases
+
+        return DatasetResults(dataset_id, [DatasetTestCase(id=test_case.id, history=test_case.history, output=test_case.output, inputs=test_case.values, metadata=test_case.metadata) for test_case in dataset_test_cases])

freeplay/support.py

@@ -1,3 +1,4 @@
+import json
 from dataclasses import dataclass
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union
@@ -5,7 +6,7 @@ from typing import Optional, Dict, Any, List, Union
 from freeplay import api_support
 from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
-from freeplay.model import InputVariables, FeedbackValue
+from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
 
 
 @dataclass
@@ -87,6 +88,28 @@ class TestRunRetrievalResponse:
             human_evaluation=summary_statistics['human_evaluation']
         )
 
+class DatasetTestCaseRequest:
+    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables, metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
+        self.history: Optional[List[NormalizedMessage]] = history
+        self.inputs: InputVariables = inputs
+        self.metadata: Optional[Dict[str, str]] = metadata
+        self.output: Optional[str] = output
+
+
+class DatasetTestCaseResponse:
+    def __init__(self, test_case: Dict[str, Any]):
+        self.values: InputVariables = test_case['values']
+        self.id: str = test_case['id']
+        self.output: Optional[str] = test_case.get('output')
+        self.history: Optional[List[NormalizedMessage]] = test_case.get('history')
+        self.metadata: Optional[Dict[str, str]] = test_case.get('metadata')
+
+class DatasetTestCasesRetrievalResponse:
+    def __init__(self, test_cases: List[Dict[str, Any]]) -> None:
+        self.test_cases = [
+            DatasetTestCaseResponse(test_case)
+            for test_case in test_cases
+        ]
 
 class CallSupport:
     def __init__(
@@ -253,3 +276,24 @@ class CallSupport:
         if response.status_code != 201:
             raise freeplay_response_error('Error while deleting session.', response)
 
+    def create_test_cases(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCaseRequest]) -> None:
+        examples = [{"history": test_case.history, "output": test_case.output, "metadata": test_case.metadata, "inputs": test_case.inputs} for test_case in test_cases]
+        payload: Dict[str, Any] = {"examples": examples}
+        url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
+
+        response = api_support.post_raw(self.freeplay_api_key, url, payload)
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while creating test cases.', response)
+
+    def get_test_cases(self, project_id: str, dataset_id: str) -> DatasetTestCasesRetrievalResponse:
+        url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
+        response = api_support.get_raw(self.freeplay_api_key, url)
+
+        if response.status_code != 200:
+            raise freeplay_response_error('Error while getting test cases.', response)
+
+        json_dom = response.json()
+
+        return DatasetTestCasesRetrievalResponse(
+            test_cases=[{"history": jsn["history"], "id": jsn["id"], "output": jsn["output"], "values": jsn["values"], "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None} for jsn in json_dom]
+        )

{freeplay-0.3.12.dist-info → freeplay-0.3.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.12
+Version: 0.3.15
 Summary: 
 License: MIT
 Author: FreePlay Engineering

freeplay-0.3.15.dist-info/RECORD

@@ -0,0 +1,22 @@
+freeplay/__init__.py,sha256=fipXoYuxUtsy6mxshut2ujWJh4lVbQ5v_xQ-33TPC-g,406
+freeplay/api_support.py,sha256=Kn2x3g6yloHQl3NwFRjbZE9BnIh7d1sgwGwC0mHuvw4,2483
+freeplay/errors.py,sha256=vwotUBldxDzREZOmLUeoiDoZjcvDwgH1AMwKBLhLooE,807
+freeplay/freeplay.py,sha256=J04-erDD6rI2SAje_Nsf3x5Qx-Z6p8gQvGrMRHFWoD4,1602
+freeplay/freeplay_cli.py,sha256=lmdsYwzdpWmUKHz_ieCzB-e6j1EnDHlVw3XIEyP_NEk,3460
+freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
+freeplay/model.py,sha256=o0de_RZ2WTJ4m5OJw1ZVfC2xG6zBq_XShBrRt1laEjc,1405
+freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+freeplay/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+freeplay/resources/customer_feedback.py,sha256=bw8MfEOKbGgn4FOyvcADrcs9GhcpNXNTgxKjBjIzywE,899
+freeplay/resources/prompts.py,sha256=NZi4K6oGnbSgw_i0NFssSqRNonl6Ov8eGPFFbZ6O5aI,22185
+freeplay/resources/recordings.py,sha256=PhwdVJaEhc_-xrtu5CPXoKRVRSBD6uqwIhTJrDCOBro,8144
+freeplay/resources/sessions.py,sha256=Qz5v7VOf1DmQTd1wCOFXnrizlW5WFJT5V8-pq22Ifvg,2793
+freeplay/resources/test_cases.py,sha256=nXL_976RwSJDT6OWDM4GEzbcOzcGkJ9ulvb0XOzCRDM,2240
+freeplay/resources/test_runs.py,sha256=Tp2N-odInT5XEEWrEsVhdgfnsclOE8n92_C8gTwO2MI,2623
+freeplay/support.py,sha256=we_FEtxcqc-8R0uOWy8p0nX0pHUbs-ulw7TC5NarlX4,11091
+freeplay/utils.py,sha256=Xvt4mNLXLL7E6MI2hTuDLV5cl5Y83DgdjCZSyDGMjR0,3187
+freeplay-0.3.15.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.3.15.dist-info/METADATA,sha256=YVeTKAO32MXniQ9UVVigz4Yiyu09xFLHvEAXCISnjNs,1654
+freeplay-0.3.15.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+freeplay-0.3.15.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.3.15.dist-info/RECORD,,

{freeplay-0.3.12.dist-info → freeplay-0.3.15.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.1.2
 Root-Is-Purelib: true
 Tag: py3-none-any

freeplay-0.3.12.dist-info/RECORD

@@ -1,21 +0,0 @@
-freeplay/__init__.py,sha256=oseuUqIVAi-2a_ns4ZbbFqkZez6KGGwI6fPkA0AKt6I,374
-freeplay/api_support.py,sha256=Kn2x3g6yloHQl3NwFRjbZE9BnIh7d1sgwGwC0mHuvw4,2483
-freeplay/errors.py,sha256=vwotUBldxDzREZOmLUeoiDoZjcvDwgH1AMwKBLhLooE,807
-freeplay/freeplay.py,sha256=cj0TGxIziS5tEL12czMJrrKrCKRoYR_Qxsipg3ClpsU,1496
-freeplay/freeplay_cli.py,sha256=lmdsYwzdpWmUKHz_ieCzB-e6j1EnDHlVw3XIEyP_NEk,3460
-freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
-freeplay/model.py,sha256=bh3TmINOxvKFxeVO8Uz7ybX28eD1tmO0XLewwLOtS7I,436
-freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-freeplay/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-freeplay/resources/customer_feedback.py,sha256=bw8MfEOKbGgn4FOyvcADrcs9GhcpNXNTgxKjBjIzywE,899
-freeplay/resources/prompts.py,sha256=-N8djt8VzqGqGNZbG23a9e_dFQfK1RTd6oDyt7Lfgn0,22155
-freeplay/resources/recordings.py,sha256=1JLR82SdwkI7foB4rTGx_Mi6d4vtr5voFLlABALEKxs,6265
-freeplay/resources/sessions.py,sha256=Qz5v7VOf1DmQTd1wCOFXnrizlW5WFJT5V8-pq22Ifvg,2793
-freeplay/resources/test_runs.py,sha256=Tp2N-odInT5XEEWrEsVhdgfnsclOE8n92_C8gTwO2MI,2623
-freeplay/support.py,sha256=RgC-EDMdxKu7iQEHQ16gxt9VGmjHLUbaKi_k0U5YR1I,8686
-freeplay/utils.py,sha256=Xvt4mNLXLL7E6MI2hTuDLV5cl5Y83DgdjCZSyDGMjR0,3187
-freeplay-0.3.12.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.3.12.dist-info/METADATA,sha256=St91ojvaCRRtbnPnXk_XjWuBiZ1LWyzoo97OI3iymTM,1654
-freeplay-0.3.12.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-freeplay-0.3.12.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.3.12.dist-info/RECORD,,