freeplay 0.3.12__tar.gz → 0.3.15__tar.gz

{freeplay-0.3.12 → freeplay-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.12
+Version: 0.3.15
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.12 → freeplay-0.3.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.3.12"
+version = "0.3.15"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"

{freeplay-0.3.12 → freeplay-0.3.15}/src/freeplay/__init__.py

@@ -1,6 +1,6 @@
 from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
-from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo
+from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
 
 __all__ = [
@@ -12,4 +12,5 @@ __all__ = [
     'SessionInfo',
     'TestRunInfo',
     'TraceInfo',
+    'UsageTokens',
 ]

{freeplay-0.3.12 → freeplay-0.3.15}/src/freeplay/freeplay.py

@@ -5,6 +5,7 @@ from freeplay.resources.customer_feedback import CustomerFeedback
 from freeplay.resources.prompts import Prompts, APITemplateResolver, TemplateResolver
 from freeplay.resources.recordings import Recordings
 from freeplay.resources.sessions import Sessions
+from freeplay.resources.test_cases import TestCases
 from freeplay.resources.test_runs import TestRuns
 from freeplay.support import CallSupport
 
@@ -38,3 +39,4 @@ class Freeplay:
         self.recordings = Recordings(self.call_support)
         self.sessions = Sessions(self.call_support)
         self.test_runs = TestRuns(self.call_support)
+        self.test_cases = TestCases(self.call_support)

freeplay-0.3.15/src/freeplay/model.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+from typing import List, Union, Any, Dict, Mapping, TypedDict, Literal
+
+InputValue = Union[str, int, bool, float, Dict[str, Any], List[Any]]
+InputVariables = Mapping[str, InputValue]
+TestRunInput = Mapping[str, InputValue]
+FeedbackValue = Union[bool, str, int, float]
+
+
+@dataclass
+class TestRun:
+    id: str
+    inputs: List[TestRunInput]
+
+
+class OpenAIFunctionCall(TypedDict):
+    name: str
+    arguments: str
+
+
+@dataclass
+class TextBlock:
+    text: str
+    type: Literal["text"] = "text"
+
+
+@dataclass
+class ToolResultBlock:
+    # AKA tool_use_id -- the ID of the tool call that this message is responding to.
+    tool_call_id: str
+    content: Union[str, List[TextBlock]]
+    type: Literal["tool_result"] = "tool_result"
+
+
+@dataclass
+class ToolCallBlock:
+    id: str
+    name: str
+    arguments: Any
+    type: Literal["tool_call"] = "tool_call"
+
+
+ContentBlock = Union[TextBlock, ToolResultBlock, ToolCallBlock]
+
+
+@dataclass
+class UserMessage:
+    content: Union[str, List[ContentBlock]]
+    role: Literal["user"] = "user"
+
+
+@dataclass
+class SystemMessage:
+    content: str
+    role: Literal["system"] = "system"
+
+
+@dataclass
+class AssistantMessage:
+    content: Union[str, List[ContentBlock]]
+    role: Literal["assistant"] = "assistant"
+
+# Largely used for history in dataset test cases presently
+NormalizedMessage = Union[UserMessage, SystemMessage, AssistantMessage]

{freeplay-0.3.12 → freeplay-0.3.15}/src/freeplay/resources/prompts.py

@@ -35,14 +35,14 @@ class UnsupportedToolSchemaError(FreeplayConfigurationError):
 
 # A content block a la OpenAI or Anthropic. Intentionally over-permissive to allow schema evolution by the providers.
 @runtime_checkable
-class ContentBlock(Protocol):
+class ProviderMessageContentBlock(Protocol):
     def model_dump(self) -> Dict[str, Any]:
         pass
 
 
 # A content/role pair with a type-safe content for common provider recording. If not using a common provider,
 # use {'content': str, 'role': str} to record. If using a common provider, this is usually the `.content` field.
-GenericProviderMessage = Union[Dict[str, Any], ContentBlock]
+GenericProviderMessage = Union[Dict[str, Any], ProviderMessageContentBlock]
 
 
 # SDK-Exposed Classes

{freeplay-0.3.12 → freeplay-0.3.15}/src/freeplay/resources/recordings.py

@@ -17,6 +17,12 @@ from freeplay.support import CallSupport
 logger = logging.getLogger(__name__)
 
 
+@dataclass
+class UsageTokens:
+    prompt_tokens: int
+    completion_tokens: int
+
+
 @dataclass
 class CallInfo:
     provider: str
@@ -25,9 +31,15 @@ class CallInfo:
     end_time: float
     model_parameters: LLMParameters
     provider_info: Optional[Dict[str, Any]] = None
+    usage: Optional[UsageTokens] = None
 
     @staticmethod
-    def from_prompt_info(prompt_info: PromptInfo, start_time: float, end_time: float) -> 'CallInfo':
+    def from_prompt_info(
+            prompt_info: PromptInfo,
+            start_time: float,
+            end_time: float,
+            usage: Optional[UsageTokens] = None
+    ) -> 'CallInfo':
         return CallInfo(
             provider=prompt_info.provider,
             model=prompt_info.model,
@@ -35,6 +47,7 @@ class CallInfo:
             end_time=end_time,
             model_parameters=prompt_info.model_parameters,
             provider_info=prompt_info.provider_info,
+            usage=usage
         )
 
 
@@ -68,6 +81,14 @@ class RecordPayload:
     completion_id: Optional[UUID] = None
 
 
+@dataclass
+class RecordUpdatePayload:
+    project_id: str
+    completion_id: str
+    new_messages: Optional[List[Dict[str, Any]]] = None
+    eval_results: Optional[Dict[str, Union[bool, float]]] = None
+
+
 @dataclass
 class RecordResponse:
     completion_id: str
@@ -77,7 +98,7 @@ class Recordings:
     def __init__(self, call_support: CallSupport):
         self.call_support = call_support
 
-    def create(self, record_payload: RecordPayload) -> RecordResponse:
+    def create(self, record_payload: RecordPayload) -> RecordResponse:  # type: ignore
         if len(record_payload.all_messages) < 1:
             raise FreeplayClientError("Messages list must have at least one message. "
                                       "The last message should be the current response.")
@@ -130,6 +151,12 @@ class Recordings:
                 "trace_id": record_payload.trace_info.trace_id
             }
 
+        if record_payload.call_info.usage is not None:
+            record_api_payload['call_info']['usage'] = {
+                "prompt_tokens": record_payload.call_info.usage.prompt_tokens,
+                "completion_tokens": record_payload.call_info.usage.completion_tokens,
+            }
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.call_support.freeplay_api_key,
@@ -143,18 +170,7 @@ class Recordings:
             message = f'There was an error recording to Freeplay. Call will not be logged. ' \
                       f'Status: {e.response.status_code}. '
 
-            if e.response.content:
-                try:
-                    content = e.response.content
-                    json_body = json.loads(content)
-                    if 'message' in json_body:
-                        message += json_body['message']
-                except:
-                    pass
-            else:
-                message += f'{e.__class__}'
-
-            raise FreeplayError(message) from e
+            self.__handle_and_raise_api_error(e, message)
 
         except Exception as e:
             status_code = -1
@@ -165,3 +181,36 @@ class Recordings:
                       f'Status: {status_code}. {e.__class__}'
 
             raise FreeplayError(message) from e
+
+    def update(self, record_update_payload: RecordUpdatePayload) -> RecordResponse:  # type: ignore
+        record_update_api_payload: Dict[str, Any] = {
+            "new_messages": record_update_payload.new_messages,
+            "eval_results": record_update_payload.eval_results,
+        }
+
+        try:
+            record_update_response = api_support.post_raw(
+                api_key=self.call_support.freeplay_api_key,
+                url=f'{self.call_support.api_base}/v2/projects/{record_update_payload.project_id}/completions/{record_update_payload.completion_id}',
+                payload=record_update_api_payload
+            )
+            record_update_response.raise_for_status()
+            json_dom = record_update_response.json()
+            return RecordResponse(completion_id=str(json_dom['completion_id']))
+        except HTTPError as e:
+            message = f'There was an error updating the completion. Status: {e.response.status_code}.'
+            self.__handle_and_raise_api_error(e, message)
+
+    @staticmethod
+    def __handle_and_raise_api_error(e: HTTPError, messages: str) -> None:
+        if e.response.content:
+            try:
+                content = e.response.content
+                json_body = json.loads(content)
+                if 'message' in json_body:
+                    messages += json_body['message']
+            except:
+                pass
+        else:
+            messages += f'{e.__class__}'
+        raise FreeplayError(messages) from e

freeplay-0.3.15/src/freeplay/resources/test_cases.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from typing import List, Optional, Dict, Any
+
+from freeplay.model import InputVariables, NormalizedMessage
+from freeplay.support import CallSupport, DatasetTestCaseRequest, DatasetTestCasesRetrievalResponse
+
+
+@dataclass
+class DatasetTestCase:
+    def __init__(
+            self,
+            inputs: InputVariables,
+            output: Optional[str],
+            history: Optional[List[NormalizedMessage]] = None,
+            metadata: Optional[Dict[str, str]] = None,
+            id: Optional[str] = None, # Only set on retrieval
+    ):
+        self.inputs = inputs
+        self.output = output
+        self.history = history
+        self.metadata = metadata
+        self.id = id
+
+
+
+@dataclass
+class Dataset:
+    def __init__(self, dataset_id: str, test_cases: List[DatasetTestCase]):
+        self.dataset_id = dataset_id
+        self.test_cases = test_cases
+
+
+@dataclass
+class DatasetResults:
+    def __init__(self, dataset_id: str, test_cases: List[DatasetTestCase]) -> None:
+        self.dataset_id = dataset_id
+        self.test_cases = test_cases
+
+class TestCases:
+    def __init__(self, call_support: CallSupport) -> None:
+        self.call_support = call_support
+
+    def create(self, project_id: str, dataset_id: str, test_case: DatasetTestCase) -> Dataset:
+        return self.create_many(project_id, dataset_id, [test_case])
+
+    def create_many(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCase]) -> Dataset:
+        dataset_test_cases = [DatasetTestCaseRequest(test_case.history, test_case.inputs, test_case.metadata, test_case.output) for test_case in test_cases]
+        self.call_support.create_test_cases(project_id, dataset_id, dataset_test_cases)
+        return Dataset(dataset_id, test_cases)
+
+    def get(self, project_id: str, dataset_id: str) -> DatasetResults:
+        test_case_results: DatasetTestCasesRetrievalResponse = self.call_support.get_test_cases(project_id, dataset_id)
+        dataset_test_cases = test_case_results.test_cases
+
+        return DatasetResults(dataset_id, [DatasetTestCase(id=test_case.id, history=test_case.history, output=test_case.output, inputs=test_case.values, metadata=test_case.metadata) for test_case in dataset_test_cases])

{freeplay-0.3.12 → freeplay-0.3.15}/src/freeplay/support.py

@@ -1,3 +1,4 @@
+import json
 from dataclasses import dataclass
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union
@@ -5,7 +6,7 @@ from typing import Optional, Dict, Any, List, Union
 from freeplay import api_support
 from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
-from freeplay.model import InputVariables, FeedbackValue
+from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
 
 
 @dataclass
@@ -87,6 +88,28 @@ class TestRunRetrievalResponse:
             human_evaluation=summary_statistics['human_evaluation']
         )
 
+class DatasetTestCaseRequest:
+    def __init__(self, history: Optional[List[NormalizedMessage]], inputs: InputVariables, metadata: Optional[Dict[str, str]], output: Optional[str]) -> None:
+        self.history: Optional[List[NormalizedMessage]] = history
+        self.inputs: InputVariables = inputs
+        self.metadata: Optional[Dict[str, str]] = metadata
+        self.output: Optional[str] = output
+
+
+class DatasetTestCaseResponse:
+    def __init__(self, test_case: Dict[str, Any]):
+        self.values: InputVariables = test_case['values']
+        self.id: str = test_case['id']
+        self.output: Optional[str] = test_case.get('output')
+        self.history: Optional[List[NormalizedMessage]] = test_case.get('history')
+        self.metadata: Optional[Dict[str, str]] = test_case.get('metadata')
+
+class DatasetTestCasesRetrievalResponse:
+    def __init__(self, test_cases: List[Dict[str, Any]]) -> None:
+        self.test_cases = [
+            DatasetTestCaseResponse(test_case)
+            for test_case in test_cases
+        ]
 
 class CallSupport:
     def __init__(
@@ -253,3 +276,24 @@ class CallSupport:
         if response.status_code != 201:
             raise freeplay_response_error('Error while deleting session.', response)
 
+    def create_test_cases(self, project_id: str, dataset_id: str, test_cases: List[DatasetTestCaseRequest]) -> None:
+        examples = [{"history": test_case.history, "output": test_case.output, "metadata": test_case.metadata, "inputs": test_case.inputs} for test_case in test_cases]
+        payload: Dict[str, Any] = {"examples": examples}
+        url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
+
+        response = api_support.post_raw(self.freeplay_api_key, url, payload)
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while creating test cases.', response)
+
+    def get_test_cases(self, project_id: str, dataset_id: str) -> DatasetTestCasesRetrievalResponse:
+        url = f'{self.api_base}/v2/projects/{project_id}/datasets/id/{dataset_id}/test-cases'
+        response = api_support.get_raw(self.freeplay_api_key, url)
+
+        if response.status_code != 200:
+            raise freeplay_response_error('Error while getting test cases.', response)
+
+        json_dom = response.json()
+
+        return DatasetTestCasesRetrievalResponse(
+            test_cases=[{"history": jsn["history"], "id": jsn["id"], "output": jsn["output"], "values": jsn["values"], "metadata": jsn["metadata"] if 'metadata' in jsn.keys() else None} for jsn in json_dom]
+        )

freeplay-0.3.12/src/freeplay/model.py

@@ -1,18 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Union, Any, Dict, Mapping, TypedDict
-
-InputValue = Union[str, int, bool, float, Dict[str, Any], List[Any]]
-InputVariables = Mapping[str, InputValue]
-TestRunInput = Mapping[str, InputValue]
-FeedbackValue = Union[bool, str, int, float]
-
-
-@dataclass
-class TestRun:
-    id: str
-    inputs: List[TestRunInput]
-
-
-class OpenAIFunctionCall(TypedDict):
-    name: str
-    arguments: str