freeplay 0.2.31__py3-none-any.whl → 0.2.32__py3-none-any.whl

freeplay/freeplay.py

@@ -3,7 +3,6 @@ import logging
 from dataclasses import dataclass
 from typing import Any, Dict, Generator, List, Optional, Tuple, Union
 
-from . import api_support
 from .completions import (
     PromptTemplates,
     CompletionResponse,
@@ -11,7 +10,7 @@ from .completions import (
     ChatCompletionResponse,
     ChatMessage
 )
-from .errors import FreeplayConfigurationError, freeplay_response_error
+from .errors import FreeplayConfigurationError
 from .flavors import Flavor, ChatFlavor, require_chat_flavor, get_chat_flavor_from_config
 from .llm_parameters import LLMParameters
 from .model import InputVariables
@@ -236,6 +235,8 @@ class FreeplayTestRun:
 # The simplifications are:
 #  - Always assumes there is a single choice returned, does not support multiple
 #  - Does not support an "escape hatch" to allow use of features we don't explicitly expose
+
+
 class Freeplay:
     def __init__(
             self,
@@ -341,25 +342,6 @@ class Freeplay:
                                                               completion_parameters=LLMParameters(kwargs),
                                                               metadata=metadata)
 
-    def create_test_run(self, project_id: str, testlist: str) -> FreeplayTestRun:
-        response = api_support.post_raw(
-            api_key=self.freeplay_api_key,
-            url=f'{self.api_base}/projects/{project_id}/test-runs',
-            payload={'playlist_name': testlist},
-        )
-
-        if response.status_code != 201:
-            raise freeplay_response_error('Error while creating a test run.', response)
-
-        json_dom = response.json()
-
-        return FreeplayTestRun(
-            self.call_support,
-            self.client_flavor,
-            self.provider_config,
-            json_dom['test_run_id'],
-            json_dom['inputs'])
-
     def start_chat(
             self,
             project_id: str,
@@ -410,6 +392,17 @@ class Freeplay:
         completion_response = session.start_chat_stream(**kwargs)
         return session, completion_response
 
+    def create_test_run(self, project_id: str, testlist: str) -> FreeplayTestRun:
+        test_run_response = self.call_support.create_test_run(project_id=project_id, testlist=testlist)
+
+        return FreeplayTestRun(
+            self.call_support,
+            self.client_flavor,
+            self.provider_config,
+            test_run_response.test_run_id,
+            [test_case.variables for test_case in test_run_response.test_cases]
+        )
+
     def __create_chat_session(
             self,
             project_id: str,

freeplay/freeplay_thin.py

@@ -1,7 +1,7 @@
 from .completions import PromptTemplates
 from .errors import FreeplayConfigurationError
 from .record import DefaultRecordProcessor
-from .support import (CallSupport)
+from .support import CallSupport
 
 
 class FreeplayThin:
@@ -13,7 +13,8 @@ class FreeplayThin:
         if not freeplay_api_key or not freeplay_api_key.strip():
             raise FreeplayConfigurationError("Freeplay API key not set. It must be set to the Freeplay API.")
 
-        self.call_support = CallSupport(freeplay_api_key, api_base, DefaultRecordProcessor(freeplay_api_key, api_base))
+        self.call_support = CallSupport(freeplay_api_key, api_base,
+                                        DefaultRecordProcessor(freeplay_api_key, api_base))
         self.freeplay_api_key = freeplay_api_key
         self.api_base = api_base
 

freeplay/model.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Union, Any, Dict
+from typing import List, Union, Any, Dict, Mapping
 
 from pydantic import RootModel
 
@@ -7,11 +7,11 @@ InputValue = Union[str, int, bool, dict[str, Any], list[Any]]
 InputVariable = RootModel[Union[Dict[str, "InputVariable"], List["InputVariable"], str, int, bool, float]]
 InputVariable.model_rebuild()
 
-InputVariables = Dict[str, InputValue]
+InputVariables = Mapping[str, InputValue]
 
 PydanticInputVariables = RootModel[Dict[str, InputVariable]]
 
-TestRunInput = Dict[str, InputValue]
+TestRunInput = Mapping[str, InputValue]
 
 
 @dataclass

freeplay/record.py

@@ -25,6 +25,7 @@ class RecordCallFields:
     variables: InputVariables
     tag: str
     test_run_id: Optional[str]
+    test_case_id: Optional[str]
     record_format_type: Optional[str]
     model: Optional[str]
     provider: Optional[str]
@@ -87,6 +88,9 @@ class DefaultRecordProcessor(RecordProcessor):
         if record_call.test_run_id is not None:
             record_payload['test_run_id'] = record_call.test_run_id
 
+        if record_call.test_case_id is not None:
+            record_payload['test_case_id'] = record_call.test_case_id
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.freeplay_api_key,

freeplay/support.py

@@ -1,7 +1,8 @@
 import json
 import time
 from copy import copy
-from typing import cast, Dict, Any, Optional, Union, List, Generator
+from typing import Dict, Any, Optional, Union, List, Generator
+from uuid import uuid4
 
 from freeplay import api_support
 from freeplay.api_support import try_decode
@@ -13,12 +14,29 @@ from freeplay.llm_parameters import LLMParameters
 from freeplay.model import InputVariables
 from freeplay.provider_config import ProviderConfig
 from freeplay.record import RecordProcessor, RecordCallFields
-from freeplay.utils import check_all_values_string_or_number
-from uuid import uuid4
 
 JsonDom = Dict[str, Any]
 
 
+class TestCaseTestRunResponse:
+    def __init__(self, test_case: JsonDom):
+        self.variables: InputVariables = test_case['variables']
+        self.id: str = test_case['id']
+
+
+class TestRunResponse:
+    def __init__(
+            self,
+            test_run_id: str,
+            test_cases: list[JsonDom]
+    ):
+        self.test_cases = [
+            TestCaseTestRunResponse(test_case)
+            for test_case in test_cases
+        ]
+        self.test_run_id = test_run_id
+
+
 class CallSupport:
     def __init__(
             self,
@@ -67,6 +85,20 @@ class CallSupport:
 
         return maybe_prompts
 
+    def create_test_run(self, project_id: str, testlist: str) -> TestRunResponse:
+        response = api_support.post_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/projects/{project_id}/test-runs-cases',
+            payload={'testlist_name': testlist},
+        )
+
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while creating a test run.', response)
+
+        json_dom = response.json()
+
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+
     # noinspection PyUnboundLocalVariable
     def prepare_and_make_chat_call(
             self,
@@ -80,7 +112,7 @@ class CallSupport:
             new_messages: Optional[List[ChatMessage]],
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> ChatCompletionResponse:
         # make call
         start = time.time()
@@ -110,6 +142,7 @@ class CallSupport:
             record_format_type=flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=flavor.provider,
             llm_parameters=params,
@@ -131,7 +164,7 @@ class CallSupport:
             message_history: List[ChatMessage],
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> Generator[CompletionChunk, None, None]:
         # make call
         start = time.time()
@@ -164,6 +197,7 @@ class CallSupport:
             record_format_type=flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=flavor.provider,
             llm_parameters=params,
@@ -183,7 +217,7 @@ class CallSupport:
             tag: str,
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> CompletionResponse:
         target_template = self.find_template_by_name(prompts, template_name)
         params = target_template.get_params() \
@@ -215,6 +249,7 @@ class CallSupport:
             record_format_type=final_flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=final_flavor.provider,
             llm_parameters=params,
@@ -235,7 +270,7 @@ class CallSupport:
             tag: str,
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> Generator[CompletionChunk, None, None]:
         target_template = self.find_template_by_name(prompts, template_name)
         params = target_template.get_params() \
@@ -272,6 +307,7 @@ class CallSupport:
             record_format_type=final_flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=final_flavor.provider,
             llm_parameters=params,

{freeplay-0.2.31.dist-info → freeplay-0.2.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freeplay
-Version: 0.2.31
+Version: 0.2.32
 Summary: 
 License: MIT
 Author: FreePlay Engineering
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: anthropic (>=0.7.7,<0.8.0)
 Requires-Dist: click (==8.1.7)
 Requires-Dist: dacite (>=1.8.0,<2.0.0)

{freeplay-0.2.31.dist-info → freeplay-0.2.32.dist-info}/RECORD

@@ -3,18 +3,18 @@ freeplay/api_support.py,sha256=E4Mxa3Lx31TEw_X6o4s9eAR1TSxs0PhFYYaWax6dH2I,2027
 freeplay/completions.py,sha256=F1rMBtQaCtn0rBQqvCurkV25g8gLtwnEod5rRvf-txY,1176
 freeplay/errors.py,sha256=bPqsw32YX-xSr7O-G49M0sSFF7mq-YF1WGq928UV47s,631
 freeplay/flavors.py,sha256=XroBKT8Nf92VTsuC261Nceo9f9stkkpC5CXbi8y4NEs,17236
-freeplay/freeplay.py,sha256=qKBgpBEDl7drJTNRxli2_nPGkC0cjM-L7W_vd8Omhas,17346
+freeplay/freeplay.py,sha256=BLlr4YXh3a624xcM93KvZGsiIGSMrIMFM07lLyVCIao,17089
 freeplay/freeplay_cli.py,sha256=nasbc_ckSr5-YtUKfg_w-6X1geQZ9s5u79VzRULGsbs,3868
-freeplay/freeplay_thin.py,sha256=MY8Ej9zVY8cYB_mh7YpHkyqtOKb3LxWJK1R_Los6XeY,829
+freeplay/freeplay_thin.py,sha256=WHoVCkS30mDM7ghjDadgfj48BNL5EvaXktUsGfeXlIY,867
 freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
-freeplay/model.py,sha256=zmcaE4ZuJeWL7mxZ7hNc-EVMljyn3PHqzSvZbZ3XWZk,519
+freeplay/model.py,sha256=kVARXUEJKbhOdWz7T8eR7npTPXlpaY9wWaEzJvbIvOU,534
 freeplay/provider_config.py,sha256=hruf3Khusrwb76_-hv7ouuxmvJuaRyC1UxIw7XlJx8A,1416
 freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-freeplay/record.py,sha256=PCKZjBtTNTHaHxJDIVIQMDdpdMXyAljgMCWfmbfNlyo,3202
-freeplay/support.py,sha256=Z46_39J3he1Ilbhb1P7-D0_RRfJYSoq5Jug6cchF53g,11525
+freeplay/record.py,sha256=_0uo0rUFZz8kwgWC0JhxFg13JyXgMfAVCkzZsuRLzBA,3354
+freeplay/support.py,sha256=OOnUZOMYnLSEG_3toycWfHb2XZ8meF77uQRllYexXeo,12616
 freeplay/utils.py,sha256=cRCCIzVqWNDKlTI-DDhXGyCkplbd-X4qzDs__aUpvww,1840
-freeplay-0.2.31.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.2.31.dist-info/METADATA,sha256=nejUVtT4OHp01SPHN3N3YNwBgnxDZdPIEUKtcerPRMU,1684
-freeplay-0.2.31.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-freeplay-0.2.31.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.2.31.dist-info/RECORD,,
+freeplay-0.2.32.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.2.32.dist-info/METADATA,sha256=o9MiS7BrkV0M2Erp7bKomeRUBgveiqW6XQf7LLyVA4s,1633
+freeplay-0.2.32.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+freeplay-0.2.32.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.2.32.dist-info/RECORD,,

{freeplay-0.2.31.dist-info → freeplay-0.2.32.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.8.1
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any