freeplay 0.2.31__tar.gz → 0.2.32__tar.gz

{freeplay-0.2.31 → freeplay-0.2.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freeplay
-Version: 0.2.31
+Version: 0.2.32
 Summary: 
 License: MIT
 Author: FreePlay Engineering
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: anthropic (>=0.7.7,<0.8.0)
 Requires-Dist: click (==8.1.7)
 Requires-Dist: dacite (>=1.8.0,<2.0.0)

{freeplay-0.2.31 → freeplay-0.2.32}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.2.31"
+version = "0.2.32"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"

{freeplay-0.2.31 → freeplay-0.2.32}/src/freeplay/freeplay.py

@@ -3,7 +3,6 @@ import logging
 from dataclasses import dataclass
 from typing import Any, Dict, Generator, List, Optional, Tuple, Union
 
-from . import api_support
 from .completions import (
     PromptTemplates,
     CompletionResponse,
@@ -11,7 +10,7 @@ from .completions import (
     ChatCompletionResponse,
     ChatMessage
 )
-from .errors import FreeplayConfigurationError, freeplay_response_error
+from .errors import FreeplayConfigurationError
 from .flavors import Flavor, ChatFlavor, require_chat_flavor, get_chat_flavor_from_config
 from .llm_parameters import LLMParameters
 from .model import InputVariables
@@ -236,6 +235,8 @@ class FreeplayTestRun:
 # The simplifications are:
 #  - Always assumes there is a single choice returned, does not support multiple
 #  - Does not support an "escape hatch" to allow use of features we don't explicitly expose
+
+
 class Freeplay:
     def __init__(
             self,
@@ -341,25 +342,6 @@ class Freeplay:
                                                               completion_parameters=LLMParameters(kwargs),
                                                               metadata=metadata)
 
-    def create_test_run(self, project_id: str, testlist: str) -> FreeplayTestRun:
-        response = api_support.post_raw(
-            api_key=self.freeplay_api_key,
-            url=f'{self.api_base}/projects/{project_id}/test-runs',
-            payload={'playlist_name': testlist},
-        )
-
-        if response.status_code != 201:
-            raise freeplay_response_error('Error while creating a test run.', response)
-
-        json_dom = response.json()
-
-        return FreeplayTestRun(
-            self.call_support,
-            self.client_flavor,
-            self.provider_config,
-            json_dom['test_run_id'],
-            json_dom['inputs'])
-
     def start_chat(
             self,
             project_id: str,
@@ -410,6 +392,17 @@ class Freeplay:
         completion_response = session.start_chat_stream(**kwargs)
         return session, completion_response
 
+    def create_test_run(self, project_id: str, testlist: str) -> FreeplayTestRun:
+        test_run_response = self.call_support.create_test_run(project_id=project_id, testlist=testlist)
+
+        return FreeplayTestRun(
+            self.call_support,
+            self.client_flavor,
+            self.provider_config,
+            test_run_response.test_run_id,
+            [test_case.variables for test_case in test_run_response.test_cases]
+        )
+
     def __create_chat_session(
             self,
             project_id: str,

{freeplay-0.2.31 → freeplay-0.2.32}/src/freeplay/freeplay_thin.py

@@ -1,7 +1,7 @@
 from .completions import PromptTemplates
 from .errors import FreeplayConfigurationError
 from .record import DefaultRecordProcessor
-from .support import (CallSupport)
+from .support import CallSupport
 
 
 class FreeplayThin:
@@ -13,7 +13,8 @@ class FreeplayThin:
         if not freeplay_api_key or not freeplay_api_key.strip():
             raise FreeplayConfigurationError("Freeplay API key not set. It must be set to the Freeplay API.")
 
-        self.call_support = CallSupport(freeplay_api_key, api_base, DefaultRecordProcessor(freeplay_api_key, api_base))
+        self.call_support = CallSupport(freeplay_api_key, api_base,
+                                        DefaultRecordProcessor(freeplay_api_key, api_base))
         self.freeplay_api_key = freeplay_api_key
         self.api_base = api_base
 

{freeplay-0.2.31 → freeplay-0.2.32}/src/freeplay/model.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Union, Any, Dict
+from typing import List, Union, Any, Dict, Mapping
 
 from pydantic import RootModel
 
@@ -7,11 +7,11 @@ InputValue = Union[str, int, bool, dict[str, Any], list[Any]]
 InputVariable = RootModel[Union[Dict[str, "InputVariable"], List["InputVariable"], str, int, bool, float]]
 InputVariable.model_rebuild()
 
-InputVariables = Dict[str, InputValue]
+InputVariables = Mapping[str, InputValue]
 
 PydanticInputVariables = RootModel[Dict[str, InputVariable]]
 
-TestRunInput = Dict[str, InputValue]
+TestRunInput = Mapping[str, InputValue]
 
 
 @dataclass

{freeplay-0.2.31 → freeplay-0.2.32}/src/freeplay/record.py

@@ -25,6 +25,7 @@ class RecordCallFields:
     variables: InputVariables
     tag: str
     test_run_id: Optional[str]
+    test_case_id: Optional[str]
     record_format_type: Optional[str]
     model: Optional[str]
     provider: Optional[str]
@@ -87,6 +88,9 @@ class DefaultRecordProcessor(RecordProcessor):
         if record_call.test_run_id is not None:
             record_payload['test_run_id'] = record_call.test_run_id
 
+        if record_call.test_case_id is not None:
+            record_payload['test_case_id'] = record_call.test_case_id
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.freeplay_api_key,

{freeplay-0.2.31 → freeplay-0.2.32}/src/freeplay/support.py

@@ -1,7 +1,8 @@
 import json
 import time
 from copy import copy
-from typing import cast, Dict, Any, Optional, Union, List, Generator
+from typing import Dict, Any, Optional, Union, List, Generator
+from uuid import uuid4
 
 from freeplay import api_support
 from freeplay.api_support import try_decode
@@ -13,12 +14,29 @@ from freeplay.llm_parameters import LLMParameters
 from freeplay.model import InputVariables
 from freeplay.provider_config import ProviderConfig
 from freeplay.record import RecordProcessor, RecordCallFields
-from freeplay.utils import check_all_values_string_or_number
-from uuid import uuid4
 
 JsonDom = Dict[str, Any]
 
 
+class TestCaseTestRunResponse:
+    def __init__(self, test_case: JsonDom):
+        self.variables: InputVariables = test_case['variables']
+        self.id: str = test_case['id']
+
+
+class TestRunResponse:
+    def __init__(
+            self,
+            test_run_id: str,
+            test_cases: list[JsonDom]
+    ):
+        self.test_cases = [
+            TestCaseTestRunResponse(test_case)
+            for test_case in test_cases
+        ]
+        self.test_run_id = test_run_id
+
+
 class CallSupport:
     def __init__(
             self,
@@ -67,6 +85,20 @@ class CallSupport:
 
         return maybe_prompts
 
+    def create_test_run(self, project_id: str, testlist: str) -> TestRunResponse:
+        response = api_support.post_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/projects/{project_id}/test-runs-cases',
+            payload={'testlist_name': testlist},
+        )
+
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while creating a test run.', response)
+
+        json_dom = response.json()
+
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+
     # noinspection PyUnboundLocalVariable
     def prepare_and_make_chat_call(
             self,
@@ -80,7 +112,7 @@ class CallSupport:
             new_messages: Optional[List[ChatMessage]],
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> ChatCompletionResponse:
         # make call
         start = time.time()
@@ -110,6 +142,7 @@ class CallSupport:
             record_format_type=flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=flavor.provider,
             llm_parameters=params,
@@ -131,7 +164,7 @@ class CallSupport:
             message_history: List[ChatMessage],
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> Generator[CompletionChunk, None, None]:
         # make call
         start = time.time()
@@ -164,6 +197,7 @@ class CallSupport:
             record_format_type=flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=flavor.provider,
             llm_parameters=params,
@@ -183,7 +217,7 @@ class CallSupport:
             tag: str,
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> CompletionResponse:
         target_template = self.find_template_by_name(prompts, template_name)
         params = target_template.get_params() \
@@ -215,6 +249,7 @@ class CallSupport:
             record_format_type=final_flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=final_flavor.provider,
             llm_parameters=params,
@@ -235,7 +270,7 @@ class CallSupport:
             tag: str,
             test_run_id: Optional[str] = None,
             completion_parameters: Optional[LLMParameters] = None,
-            metadata: Optional[Dict[str, Union[str,int,float]]] = None
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
     ) -> Generator[CompletionChunk, None, None]:
         target_template = self.find_template_by_name(prompts, template_name)
         params = target_template.get_params() \
@@ -272,6 +307,7 @@ class CallSupport:
             record_format_type=final_flavor.record_format_type,
             tag=tag,
             test_run_id=test_run_id,
+            test_case_id=None,
             model=model,
             provider=final_flavor.provider,
             llm_parameters=params,