freeplay 0.3.22__tar.gz → 0.3.24a0__tar.gz

{freeplay-0.3.22 → freeplay-0.3.24a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.22
+Version: 0.3.24a0
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.22 → freeplay-0.3.24a0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.3.22"
+version = "0.3.24-alpha"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/__init__.py

@@ -1,6 +1,7 @@
 from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
-from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
+from .model import TestRunInfo
+from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
 from .support import CustomMetadata
 

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/model.py

@@ -13,6 +13,12 @@ class TestRun:
     inputs: List[TestRunInput]
 
 
+@dataclass
+class TestRunInfo:
+    test_run_id: str
+    test_case_id: str
+
+
 class OpenAIFunctionCall(TypedDict):
     name: str
     arguments: str

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/resources/adapters.py

@@ -35,6 +35,7 @@ class MissingFlavorError(FreeplayConfigurationError):
 
 
 class LLMAdapter(Protocol):
+    # This method must handle BOTH prompt template messages and provider specific messages.
     def to_llm_syntax(self, messages: List[Dict[str, Any]]) -> Union[str, List[Dict[str, Any]]]:
         pass
 
@@ -187,11 +188,13 @@ class GeminiAdapter(LLMAdapter):
                     "role": self.__translate_role(message["role"]),
                     "parts": [self.__map_content(content) for content in message['content']]
                 })
-            else:
+            elif "content" in message:
                 gemini_messages.append({
                     "role": self.__translate_role(message["role"]),
                     "parts": [{"text": message["content"]}]
                 })
+            else:
+                gemini_messages.append(copy.deepcopy(message))
 
         return gemini_messages
 

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/resources/recordings.py

@@ -9,11 +9,12 @@ from requests import HTTPError
 from freeplay import api_support
 from freeplay.errors import FreeplayClientError, FreeplayError
 from freeplay.llm_parameters import LLMParameters
-from freeplay.model import InputVariables, OpenAIFunctionCall
+from freeplay.model import InputVariables, OpenAIFunctionCall, TestRunInfo
 from freeplay.resources.prompts import PromptInfo, MediaInputMap, MediaInput, MediaInputUrl
 from freeplay.resources.sessions import SessionInfo, TraceInfo
 from freeplay.support import CallSupport
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -65,11 +66,6 @@ class ResponseInfo:
     response_tokens: Optional[int] = None
 
 
-@dataclass
-class TestRunInfo:
-    test_run_id: str
-    test_case_id: str
-
 
 @dataclass
 class RecordPayload:

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/resources/sessions.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import Optional, Dict, Union
 
 from freeplay.errors import FreeplayClientError
+from freeplay.model import TestRunInfo
 from freeplay.support import CallSupport, CustomMetadata
 
 
@@ -40,7 +41,8 @@ class TraceInfo:
             self,
             project_id: str,
             output: str,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
         if self.input is None:
             raise FreeplayClientError("Input must be set before recording output")
@@ -52,7 +54,8 @@ class TraceInfo:
             output,
             agent_name=self.agent_name,
             custom_metadata=self.custom_metadata,
-            eval_results=eval_results
+            eval_results=eval_results,
+            test_run_info=test_run_info
         )
 
 

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/resources/test_runs.py

@@ -1,8 +1,7 @@
 from dataclasses import dataclass
 from typing import List, Optional, Dict
 
-from freeplay.model import InputVariables
-from freeplay.resources.recordings import TestRunInfo
+from freeplay.model import InputVariables, TestRunInfo
 from freeplay.support import CallSupport, SummaryStatistics
 
 
@@ -20,20 +19,45 @@ class TestCase:
         self.output = output
         self.history = history
 
+class TraceTestCase:
+    def __init__(
+            self,
+            test_case_id: str,
+            input: str,
+            output: Optional[str],
+    ):
+        self.id = test_case_id
+        self.input = input
+        self.output = output
 
 @dataclass
 class TestRun:
     def __init__(
             self,
             test_run_id: str,
-            test_cases: List[TestCase]
+            test_cases: List[TestCase] = [],
+            trace_test_cases: List[TraceTestCase] = []
     ):
         self.test_run_id = test_run_id
         self.test_cases = test_cases
+        self.trace_test_cases = trace_test_cases
+
+    def __must_not_be_both_trace_and_completion(self) -> None:
+        if self.test_cases and len(self.test_cases) > 0 and self.trace_test_cases and len(self.trace_test_cases) > 0:
+            raise ValueError("Test case and trace test case cannot both be present")
 
     def get_test_cases(self) -> List[TestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.trace_test_cases) > 0:
+            raise ValueError("Completion test cases are not present. Please use get_trace_test_cases() instead.")
         return self.test_cases
 
+    def get_trace_test_cases(self) -> List[TraceTestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.test_cases) > 0:
+            raise ValueError("Trace test cases are not present. Please use get_test_cases() instead.")
+        return self.trace_test_cases
+
     def get_test_run_info(self, test_case_id: str) -> TestRunInfo:
         return TestRunInfo(self.test_run_id, test_case_id)
 
@@ -75,8 +99,14 @@ class TestRuns:
                      history=test_case.history)
             for test_case in test_run.test_cases
         ]
+        trace_test_cases = [
+            TraceTestCase(test_case_id=test_case.id,
+                          input=test_case.input,
+                          output=test_case.output)
+            for test_case in test_run.trace_test_cases
+        ]
 
-        return TestRun(test_run.test_run_id, test_cases)
+        return TestRun(test_run.test_run_id, test_cases, trace_test_cases)
 
     def get(self, project_id: str, test_run_id: str) -> TestRunResults:
         test_run_results = self.call_support.get_test_run_results(project_id, test_run_id)

{freeplay-0.3.22 → freeplay-0.3.24a0}/src/freeplay/support.py

@@ -1,11 +1,11 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, asdict
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union, Literal
 
 from freeplay import api_support
 from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
-from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
+from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage, TestRunInfo
 
 CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
 
@@ -83,22 +83,38 @@ class PromptTemplateEncoder(JSONEncoder):
 
 class TestCaseTestRunResponse:
     def __init__(self, test_case: Dict[str, Any]):
-        self.variables: InputVariables = test_case['variables']
         self.id: str = test_case['test_case_id']
+        self.variables: InputVariables = test_case['variables']
         self.output: Optional[str] = test_case.get('output')
         self.history: Optional[List[Dict[str, Any]]] = test_case.get('history')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
+class TraceTestCaseTestRunResponse:
+    def __init__(self, test_case: Dict[str, Any]):
+        self.id: str = test_case['test_case_id']
+        self.input: str = test_case['input']
+        self.output: Optional[str] = test_case.get('output')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
 class TestRunResponse:
     def __init__(
             self,
             test_run_id: str,
-            test_cases: List[Dict[str, Any]]
+            test_cases: Optional[List[Dict[str, Any]]],
+            trace_test_cases: Optional[List[Dict[str, Any]]]
     ):
+        if test_cases and trace_test_cases:
+            raise ValueError("Test cases and trace test cases cannot both be present.")
+
         self.test_cases = [
             TestCaseTestRunResponse(test_case)
-            for test_case in test_cases
+            for test_case in (test_cases or []) if test_case is not None
+        ]
+        self.trace_test_cases = [
+            TraceTestCaseTestRunResponse(test_case)
+            for test_case in (trace_test_cases or []) if test_case is not None
         ]
+
         self.test_run_id = test_run_id
 
 
@@ -267,7 +283,7 @@ class CallSupport:
 
         json_dom = response.json()
 
-        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'], json_dom['trace_test_cases'])
 
     def get_test_run_results(
             self,
@@ -299,18 +315,21 @@ class CallSupport:
             output: str,
             agent_name: Optional[str] = None,
             custom_metadata: CustomMetadata = None,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
+        payload = {
+            'agent_name': agent_name,
+            'input': input,
+            'output': output,
+            'custom_metadata': custom_metadata,
+            'eval_results': eval_results,
+            'test_run_info': asdict(test_run_info) if test_run_info else None
+        }
         response = api_support.post_raw(
             self.freeplay_api_key,
             f'{self.api_base}/v2/projects/{project_id}/sessions/{session_id}/traces/id/{trace_id}',
-            {
-                'agent_name': agent_name,
-                'input': input,
-                'output': output,
-                'custom_metadata': custom_metadata,
-                'eval_results': eval_results,
-            }
+            payload
         )
         if response.status_code != 201:
             raise freeplay_response_error('Error while recording trace.', response)