freeplay 0.3.22__tar.gz → 0.3.24__tar.gz

{freeplay-0.3.22 → freeplay-0.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.22
+Version: 0.3.24
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.22 → freeplay-0.3.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.3.22"
+version = "0.3.24"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/__init__.py

@@ -1,6 +1,7 @@
 from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
-from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
+from .model import TestRunInfo
+from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
 from .support import CustomMetadata
 

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/model.py

@@ -13,6 +13,12 @@ class TestRun:
     inputs: List[TestRunInput]
 
 
+@dataclass
+class TestRunInfo:
+    test_run_id: str
+    test_case_id: str
+
+
 class OpenAIFunctionCall(TypedDict):
     name: str
     arguments: str

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/resources/adapters.py

@@ -35,6 +35,7 @@ class MissingFlavorError(FreeplayConfigurationError):
 
 
 class LLMAdapter(Protocol):
+    # This method must handle BOTH prompt template messages and provider specific messages.
     def to_llm_syntax(self, messages: List[Dict[str, Any]]) -> Union[str, List[Dict[str, Any]]]:
         pass
 
@@ -187,11 +188,13 @@ class GeminiAdapter(LLMAdapter):
                     "role": self.__translate_role(message["role"]),
                     "parts": [self.__map_content(content) for content in message['content']]
                 })
-            else:
+            elif "content" in message:
                 gemini_messages.append({
                     "role": self.__translate_role(message["role"]),
                     "parts": [{"text": message["content"]}]
                 })
+            else:
+                gemini_messages.append(copy.deepcopy(message))
 
         return gemini_messages
 

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/resources/recordings.py

@@ -9,11 +9,12 @@ from requests import HTTPError
 from freeplay import api_support
 from freeplay.errors import FreeplayClientError, FreeplayError
 from freeplay.llm_parameters import LLMParameters
-from freeplay.model import InputVariables, OpenAIFunctionCall
+from freeplay.model import InputVariables, OpenAIFunctionCall, TestRunInfo
 from freeplay.resources.prompts import PromptInfo, MediaInputMap, MediaInput, MediaInputUrl
 from freeplay.resources.sessions import SessionInfo, TraceInfo
 from freeplay.support import CallSupport
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -65,11 +66,6 @@ class ResponseInfo:
     response_tokens: Optional[int] = None
 
 
-@dataclass
-class TestRunInfo:
-    test_run_id: str
-    test_case_id: str
-
 
 @dataclass
 class RecordPayload:

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/resources/sessions.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import Optional, Dict, Union
 
 from freeplay.errors import FreeplayClientError
+from freeplay.model import TestRunInfo
 from freeplay.support import CallSupport, CustomMetadata
 
 
@@ -40,7 +41,8 @@ class TraceInfo:
             self,
             project_id: str,
             output: str,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
         if self.input is None:
             raise FreeplayClientError("Input must be set before recording output")
@@ -52,7 +54,8 @@ class TraceInfo:
             output,
             agent_name=self.agent_name,
             custom_metadata=self.custom_metadata,
-            eval_results=eval_results
+            eval_results=eval_results,
+            test_run_info=test_run_info
         )
 
 

freeplay-0.3.24/src/freeplay/resources/test_runs.py

@@ -0,0 +1,132 @@
+from dataclasses import dataclass
+from typing import List, Optional, Dict, Any
+import warnings
+
+from freeplay.model import InputVariables, TestRunInfo
+from freeplay.support import CallSupport, SummaryStatistics
+
+@dataclass
+class CompletionTestCase:
+    def __init__(
+            self,
+            test_case_id: str,
+            variables: InputVariables,
+            output: Optional[str],
+            history: Optional[List[Dict[str, str]]],
+            custom_metadata: Optional[Dict[str, str]]
+    ):
+        self.id = test_case_id
+        self.variables = variables
+        self.output = output
+        self.history = history
+        self.custom_metadata = custom_metadata
+
+class TestCase(CompletionTestCase):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        warnings.warn(
+            "'TestCase' is deprecated; use 'CompletionTestCase' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
+
+class TraceTestCase:
+    def __init__(
+            self,
+            test_case_id: str,
+            input: str,
+            output: Optional[str],
+            custom_metadata: Optional[Dict[str, str]]
+    ):
+        self.id = test_case_id
+        self.input = input
+        self.output = output
+        self.custom_metadata = custom_metadata
+@dataclass
+class TestRun:
+    def __init__(
+            self,
+            test_run_id: str,
+            test_cases: List[CompletionTestCase] = [],
+            trace_test_cases: List[TraceTestCase] = []
+    ):
+        self.test_run_id = test_run_id
+        self.test_cases = test_cases
+        self.trace_test_cases = trace_test_cases
+
+    def __must_not_be_both_trace_and_completion(self) -> None:
+        if self.test_cases and len(self.test_cases) > 0 and self.trace_test_cases and len(self.trace_test_cases) > 0:
+            raise ValueError("Test case and trace test case cannot both be present")
+
+    def get_test_cases(self) -> List[CompletionTestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.trace_test_cases) > 0:
+            raise ValueError("Completion test cases are not present. Please use get_trace_test_cases() instead.")
+        return self.test_cases
+
+    def get_trace_test_cases(self) -> List[TraceTestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.test_cases) > 0:
+            raise ValueError("Trace test cases are not present. Please use get_test_cases() instead.")
+        return self.trace_test_cases
+
+    def get_test_run_info(self, test_case_id: str) -> TestRunInfo:
+        return TestRunInfo(self.test_run_id, test_case_id)
+
+
+@dataclass
+class TestRunResults:
+    def __init__(
+            self,
+            name: str,
+            description: str,
+            test_run_id: str,
+            summary_statistics: SummaryStatistics,
+    ):
+        self.name = name
+        self.description = description
+        self.test_run_id = test_run_id
+        self.summary_statistics = summary_statistics
+
+
+class TestRuns:
+    def __init__(self, call_support: CallSupport) -> None:
+        self.call_support = call_support
+
+    def create(
+            self,
+            project_id: str,
+            testlist: str,
+            include_outputs: bool = False,
+            name: Optional[str] = None,
+            description: Optional[str] = None,
+            flavor_name: Optional[str] = None
+    ) -> TestRun:
+        test_run = self.call_support.create_test_run(
+            project_id, testlist, include_outputs, name, description, flavor_name)
+        test_cases = [
+            CompletionTestCase(test_case_id=test_case.id,
+                     variables=test_case.variables,
+                     output=test_case.output,
+                     history=test_case.history,
+                     custom_metadata=test_case.custom_metadata)
+            for test_case in test_run.test_cases
+        ]
+        trace_test_cases = [
+            TraceTestCase(test_case_id=test_case.id,
+                          input=test_case.input,
+                          output=test_case.output,
+                          custom_metadata=test_case.custom_metadata)
+            for test_case in test_run.trace_test_cases
+        ]
+
+        return TestRun(test_run.test_run_id, test_cases, trace_test_cases)
+
+    def get(self, project_id: str, test_run_id: str) -> TestRunResults:
+        test_run_results = self.call_support.get_test_run_results(project_id, test_run_id)
+        return TestRunResults(
+            test_run_results.name,
+            test_run_results.description,
+            test_run_results.test_run_id,
+            test_run_results.summary_statistics
+        )

{freeplay-0.3.22 → freeplay-0.3.24}/src/freeplay/support.py

@@ -1,11 +1,11 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, asdict
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union, Literal
 
 from freeplay import api_support
 from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
-from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
+from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage, TestRunInfo
 
 CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
 
@@ -83,22 +83,38 @@ class PromptTemplateEncoder(JSONEncoder):
 
 class TestCaseTestRunResponse:
     def __init__(self, test_case: Dict[str, Any]):
-        self.variables: InputVariables = test_case['variables']
         self.id: str = test_case['test_case_id']
+        self.variables: InputVariables = test_case['variables']
         self.output: Optional[str] = test_case.get('output')
         self.history: Optional[List[Dict[str, Any]]] = test_case.get('history')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
+class TraceTestCaseTestRunResponse:
+    def __init__(self, test_case: Dict[str, Any]):
+        self.id: str = test_case['test_case_id']
+        self.input: str = test_case['input']
+        self.output: Optional[str] = test_case.get('output')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
 class TestRunResponse:
     def __init__(
             self,
             test_run_id: str,
-            test_cases: List[Dict[str, Any]]
+            test_cases: Optional[List[Dict[str, Any]]],
+            trace_test_cases: Optional[List[Dict[str, Any]]]
     ):
+        if test_cases and trace_test_cases:
+            raise ValueError("Test cases and trace test cases cannot both be present.")
+
         self.test_cases = [
             TestCaseTestRunResponse(test_case)
-            for test_case in test_cases
+            for test_case in (test_cases or []) if test_case is not None
+        ]
+        self.trace_test_cases = [
+            TraceTestCaseTestRunResponse(test_case)
+            for test_case in (trace_test_cases or []) if test_case is not None
         ]
+
         self.test_run_id = test_run_id
 
 
@@ -267,7 +283,7 @@ class CallSupport:
 
         json_dom = response.json()
 
-        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'], json_dom['trace_test_cases'])
 
     def get_test_run_results(
             self,
@@ -299,18 +315,21 @@ class CallSupport:
             output: str,
             agent_name: Optional[str] = None,
             custom_metadata: CustomMetadata = None,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
+        payload = {
+            'agent_name': agent_name,
+            'input': input,
+            'output': output,
+            'custom_metadata': custom_metadata,
+            'eval_results': eval_results,
+            'test_run_info': asdict(test_run_info) if test_run_info else None
+        }
         response = api_support.post_raw(
             self.freeplay_api_key,
             f'{self.api_base}/v2/projects/{project_id}/sessions/{session_id}/traces/id/{trace_id}',
-            {
-                'agent_name': agent_name,
-                'input': input,
-                'output': output,
-                'custom_metadata': custom_metadata,
-                'eval_results': eval_results,
-            }
+            payload
         )
         if response.status_code != 201:
             raise freeplay_response_error('Error while recording trace.', response)

freeplay-0.3.22/src/freeplay/resources/test_runs.py

@@ -1,88 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional, Dict
-
-from freeplay.model import InputVariables
-from freeplay.resources.recordings import TestRunInfo
-from freeplay.support import CallSupport, SummaryStatistics
-
-
-@dataclass
-class TestCase:
-    def __init__(
-            self,
-            test_case_id: str,
-            variables: InputVariables,
-            output: Optional[str],
-            history: Optional[List[Dict[str, str]]]
-    ):
-        self.id = test_case_id
-        self.variables = variables
-        self.output = output
-        self.history = history
-
-
-@dataclass
-class TestRun:
-    def __init__(
-            self,
-            test_run_id: str,
-            test_cases: List[TestCase]
-    ):
-        self.test_run_id = test_run_id
-        self.test_cases = test_cases
-
-    def get_test_cases(self) -> List[TestCase]:
-        return self.test_cases
-
-    def get_test_run_info(self, test_case_id: str) -> TestRunInfo:
-        return TestRunInfo(self.test_run_id, test_case_id)
-
-
-@dataclass
-class TestRunResults:
-    def __init__(
-            self,
-            name: str,
-            description: str,
-            test_run_id: str,
-            summary_statistics: SummaryStatistics,
-    ):
-        self.name = name
-        self.description = description
-        self.test_run_id = test_run_id
-        self.summary_statistics = summary_statistics
-
-
-class TestRuns:
-    def __init__(self, call_support: CallSupport) -> None:
-        self.call_support = call_support
-
-    def create(
-            self,
-            project_id: str,
-            testlist: str,
-            include_outputs: bool = False,
-            name: Optional[str] = None,
-            description: Optional[str] = None,
-            flavor_name: Optional[str] = None
-    ) -> TestRun:
-        test_run = self.call_support.create_test_run(
-            project_id, testlist, include_outputs, name, description, flavor_name)
-        test_cases = [
-            TestCase(test_case_id=test_case.id,
-                     variables=test_case.variables,
-                     output=test_case.output,
-                     history=test_case.history)
-            for test_case in test_run.test_cases
-        ]
-
-        return TestRun(test_run.test_run_id, test_cases)
-
-    def get(self, project_id: str, test_run_id: str) -> TestRunResults:
-        test_run_results = self.call_support.get_test_run_results(project_id, test_run_id)
-        return TestRunResults(
-            test_run_results.name,
-            test_run_results.description,
-            test_run_results.test_run_id,
-            test_run_results.summary_statistics
-        )