freeplay 0.3.23__py3-none-any.whl → 0.3.24__py3-none-any.whl

freeplay/__init__.py

@@ -1,6 +1,7 @@
 from .freeplay import Freeplay
 from .resources.prompts import PromptInfo
-from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo, UsageTokens
+from .model import TestRunInfo
+from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, UsageTokens
 from .resources.sessions import SessionInfo, TraceInfo
 from .support import CustomMetadata
 

freeplay/model.py

@@ -13,6 +13,12 @@ class TestRun:
     inputs: List[TestRunInput]
 
 
+@dataclass
+class TestRunInfo:
+    test_run_id: str
+    test_case_id: str
+
+
 class OpenAIFunctionCall(TypedDict):
     name: str
     arguments: str

freeplay/resources/recordings.py

@@ -9,11 +9,12 @@ from requests import HTTPError
 from freeplay import api_support
 from freeplay.errors import FreeplayClientError, FreeplayError
 from freeplay.llm_parameters import LLMParameters
-from freeplay.model import InputVariables, OpenAIFunctionCall
+from freeplay.model import InputVariables, OpenAIFunctionCall, TestRunInfo
 from freeplay.resources.prompts import PromptInfo, MediaInputMap, MediaInput, MediaInputUrl
 from freeplay.resources.sessions import SessionInfo, TraceInfo
 from freeplay.support import CallSupport
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -65,11 +66,6 @@ class ResponseInfo:
     response_tokens: Optional[int] = None
 
 
-@dataclass
-class TestRunInfo:
-    test_run_id: str
-    test_case_id: str
-
 
 @dataclass
 class RecordPayload:

freeplay/resources/sessions.py

@@ -3,6 +3,7 @@ from dataclasses import dataclass
 from typing import Optional, Dict, Union
 
 from freeplay.errors import FreeplayClientError
+from freeplay.model import TestRunInfo
 from freeplay.support import CallSupport, CustomMetadata
 
 
@@ -40,7 +41,8 @@ class TraceInfo:
             self,
             project_id: str,
             output: str,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
         if self.input is None:
             raise FreeplayClientError("Input must be set before recording output")
@@ -52,7 +54,8 @@ class TraceInfo:
             output,
             agent_name=self.agent_name,
             custom_metadata=self.custom_metadata,
-            eval_results=eval_results
+            eval_results=eval_results,
+            test_run_info=test_run_info
         )
 
 

freeplay/resources/test_runs.py

@@ -1,39 +1,75 @@
 from dataclasses import dataclass
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Any
+import warnings
 
-from freeplay.model import InputVariables
-from freeplay.resources.recordings import TestRunInfo
+from freeplay.model import InputVariables, TestRunInfo
 from freeplay.support import CallSupport, SummaryStatistics
 
-
 @dataclass
-class TestCase:
+class CompletionTestCase:
     def __init__(
             self,
             test_case_id: str,
             variables: InputVariables,
             output: Optional[str],
-            history: Optional[List[Dict[str, str]]]
+            history: Optional[List[Dict[str, str]]],
+            custom_metadata: Optional[Dict[str, str]]
     ):
         self.id = test_case_id
         self.variables = variables
         self.output = output
         self.history = history
+        self.custom_metadata = custom_metadata
 
+class TestCase(CompletionTestCase):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        warnings.warn(
+            "'TestCase' is deprecated; use 'CompletionTestCase' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
 
+class TraceTestCase:
+    def __init__(
+            self,
+            test_case_id: str,
+            input: str,
+            output: Optional[str],
+            custom_metadata: Optional[Dict[str, str]]
+    ):
+        self.id = test_case_id
+        self.input = input
+        self.output = output
+        self.custom_metadata = custom_metadata
 @dataclass
 class TestRun:
     def __init__(
             self,
             test_run_id: str,
-            test_cases: List[TestCase]
+            test_cases: List[CompletionTestCase] = [],
+            trace_test_cases: List[TraceTestCase] = []
     ):
         self.test_run_id = test_run_id
         self.test_cases = test_cases
+        self.trace_test_cases = trace_test_cases
 
-    def get_test_cases(self) -> List[TestCase]:
+    def __must_not_be_both_trace_and_completion(self) -> None:
+        if self.test_cases and len(self.test_cases) > 0 and self.trace_test_cases and len(self.trace_test_cases) > 0:
+            raise ValueError("Test case and trace test case cannot both be present")
+
+    def get_test_cases(self) -> List[CompletionTestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.trace_test_cases) > 0:
+            raise ValueError("Completion test cases are not present. Please use get_trace_test_cases() instead.")
         return self.test_cases
 
+    def get_trace_test_cases(self) -> List[TraceTestCase]:
+        self.__must_not_be_both_trace_and_completion()
+        if len(self.test_cases) > 0:
+            raise ValueError("Trace test cases are not present. Please use get_test_cases() instead.")
+        return self.trace_test_cases
+
     def get_test_run_info(self, test_case_id: str) -> TestRunInfo:
         return TestRunInfo(self.test_run_id, test_case_id)
 
@@ -69,14 +105,22 @@ class TestRuns:
         test_run = self.call_support.create_test_run(
             project_id, testlist, include_outputs, name, description, flavor_name)
         test_cases = [
-            TestCase(test_case_id=test_case.id,
+            CompletionTestCase(test_case_id=test_case.id,
                      variables=test_case.variables,
                      output=test_case.output,
-                     history=test_case.history)
+                     history=test_case.history,
+                     custom_metadata=test_case.custom_metadata)
             for test_case in test_run.test_cases
         ]
+        trace_test_cases = [
+            TraceTestCase(test_case_id=test_case.id,
+                          input=test_case.input,
+                          output=test_case.output,
+                          custom_metadata=test_case.custom_metadata)
+            for test_case in test_run.trace_test_cases
+        ]
 
-        return TestRun(test_run.test_run_id, test_cases)
+        return TestRun(test_run.test_run_id, test_cases, trace_test_cases)
 
     def get(self, project_id: str, test_run_id: str) -> TestRunResults:
         test_run_results = self.call_support.get_test_run_results(project_id, test_run_id)

freeplay/support.py

@@ -1,11 +1,11 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, asdict
 from json import JSONEncoder
 from typing import Optional, Dict, Any, List, Union, Literal
 
 from freeplay import api_support
 from freeplay.api_support import try_decode
 from freeplay.errors import freeplay_response_error, FreeplayServerError
-from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage
+from freeplay.model import InputVariables, FeedbackValue, NormalizedMessage, TestRunInfo
 
 CustomMetadata = Optional[Dict[str, Union[str, int, float, bool]]]
 
@@ -83,22 +83,38 @@ class PromptTemplateEncoder(JSONEncoder):
 
 class TestCaseTestRunResponse:
     def __init__(self, test_case: Dict[str, Any]):
-        self.variables: InputVariables = test_case['variables']
         self.id: str = test_case['test_case_id']
+        self.variables: InputVariables = test_case['variables']
         self.output: Optional[str] = test_case.get('output')
         self.history: Optional[List[Dict[str, Any]]] = test_case.get('history')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
+class TraceTestCaseTestRunResponse:
+    def __init__(self, test_case: Dict[str, Any]):
+        self.id: str = test_case['test_case_id']
+        self.input: str = test_case['input']
+        self.output: Optional[str] = test_case.get('output')
+        self.custom_metadata: Optional[Dict[str, str]] = test_case.get('custom_metadata')
 
 class TestRunResponse:
     def __init__(
             self,
             test_run_id: str,
-            test_cases: List[Dict[str, Any]]
+            test_cases: Optional[List[Dict[str, Any]]],
+            trace_test_cases: Optional[List[Dict[str, Any]]]
     ):
+        if test_cases and trace_test_cases:
+            raise ValueError("Test cases and trace test cases cannot both be present.")
+
         self.test_cases = [
             TestCaseTestRunResponse(test_case)
-            for test_case in test_cases
+            for test_case in (test_cases or []) if test_case is not None
+        ]
+        self.trace_test_cases = [
+            TraceTestCaseTestRunResponse(test_case)
+            for test_case in (trace_test_cases or []) if test_case is not None
         ]
+
         self.test_run_id = test_run_id
 
 
@@ -267,7 +283,7 @@ class CallSupport:
 
         json_dom = response.json()
 
-        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'], json_dom['trace_test_cases'])
 
     def get_test_run_results(
             self,
@@ -299,18 +315,21 @@ class CallSupport:
             output: str,
             agent_name: Optional[str] = None,
             custom_metadata: CustomMetadata = None,
-            eval_results: Optional[Dict[str, Union[bool, float]]] = None
+            eval_results: Optional[Dict[str, Union[bool, float]]] = None,
+            test_run_info: Optional[TestRunInfo] = None
     ) -> None:
+        payload = {
+            'agent_name': agent_name,
+            'input': input,
+            'output': output,
+            'custom_metadata': custom_metadata,
+            'eval_results': eval_results,
+            'test_run_info': asdict(test_run_info) if test_run_info else None
+        }
         response = api_support.post_raw(
             self.freeplay_api_key,
             f'{self.api_base}/v2/projects/{project_id}/sessions/{session_id}/traces/id/{trace_id}',
-            {
-                'agent_name': agent_name,
-                'input': input,
-                'output': output,
-                'custom_metadata': custom_metadata,
-                'eval_results': eval_results,
-            }
+            payload
         )
         if response.status_code != 201:
             raise freeplay_response_error('Error while recording trace.', response)

{freeplay-0.3.23.dist-info → freeplay-0.3.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: freeplay
-Version: 0.3.23
+Version: 0.3.24
 Summary: 
 License: MIT
 Author: FreePlay Engineering

{freeplay-0.3.23.dist-info → freeplay-0.3.24.dist-info}/RECORD

@@ -1,23 +1,23 @@
-freeplay/__init__.py,sha256=tzEFgRyriPBWPo04jTgR6MxzXUqIHBlUJZvuBj5kFiE,464
+freeplay/__init__.py,sha256=ljXF0vHC3oGgk-vLyJFawr7MZKOj_ExYJJsVdrpqZoM,482
 freeplay/api_support.py,sha256=Kn2x3g6yloHQl3NwFRjbZE9BnIh7d1sgwGwC0mHuvw4,2483
 freeplay/errors.py,sha256=vwotUBldxDzREZOmLUeoiDoZjcvDwgH1AMwKBLhLooE,807
 freeplay/freeplay.py,sha256=J04-erDD6rI2SAje_Nsf3x5Qx-Z6p8gQvGrMRHFWoD4,1602
 freeplay/freeplay_cli.py,sha256=lmdsYwzdpWmUKHz_ieCzB-e6j1EnDHlVw3XIEyP_NEk,3460
 freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
-freeplay/model.py,sha256=o0de_RZ2WTJ4m5OJw1ZVfC2xG6zBq_XShBrRt1laEjc,1405
+freeplay/model.py,sha256=GI3qPRouwdrBFOPBrcQFt5O0chtlSNvl8jAstePuYlA,1480
 freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 freeplay/resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 freeplay/resources/adapters.py,sha256=6ZAPpoLeOkUkV1s9VNQNsYrnupV0-sy11zFfKfctM1Y,9296
 freeplay/resources/customer_feedback.py,sha256=bw8MfEOKbGgn4FOyvcADrcs9GhcpNXNTgxKjBjIzywE,899
 freeplay/resources/prompts.py,sha256=mnL1VscMGM7D4ulGf3CBGQKdsSj8I5Wf_6nJEYa7mZI,23353
-freeplay/resources/recordings.py,sha256=z2ARII1jCnmNh1GU3hGnXZUz5IF_KhyayQum71k-h9c,9213
-freeplay/resources/sessions.py,sha256=J5A3CjiV2MFqQyxN3TWTvJaa9jmMza58mRFRq2v9iAk,3746
+freeplay/resources/recordings.py,sha256=V8KAPWnYAQ2-gqwyAJveD8a_AaBPYT32N_xbPU4S27M,9153
+freeplay/resources/sessions.py,sha256=dZtd9nq2nH8pmXxQOJitBnN5Jl3kjggDItDcjC69TYo,3883
 freeplay/resources/test_cases.py,sha256=nXL_976RwSJDT6OWDM4GEzbcOzcGkJ9ulvb0XOzCRDM,2240
-freeplay/resources/test_runs.py,sha256=Tp2N-odInT5XEEWrEsVhdgfnsclOE8n92_C8gTwO2MI,2623
-freeplay/support.py,sha256=kQMItnMGZT5TOdPQsSiKOlBbBqO4AyR91vito6wt4JM,12275
+freeplay/resources/test_runs.py,sha256=u7bBfJ3Ro5DJZQdjCAXj4Xj-3fYmtQIGhy8vJeluJvQ,4668
+freeplay/support.py,sha256=AcaG6vkmI9r6_eNFoX29eTh4MRjdb4To5wIEm78v7IE,13299
 freeplay/utils.py,sha256=Xvt4mNLXLL7E6MI2hTuDLV5cl5Y83DgdjCZSyDGMjR0,3187
-freeplay-0.3.23.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.3.23.dist-info/METADATA,sha256=IW54IuEE9o5_huw03dbWjekNpyWXyRofb_x9AEFUZf0,1661
-freeplay-0.3.23.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-freeplay-0.3.23.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.3.23.dist-info/RECORD,,
+freeplay-0.3.24.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.3.24.dist-info/METADATA,sha256=o8XujEUGJhQzw3ONbL91BPC38d-JcUfL24sIGLaaHZM,1661
+freeplay-0.3.24.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+freeplay-0.3.24.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.3.24.dist-info/RECORD,,