freeplay 0.2.30__py3-none-any.whl → 0.2.32__py3-none-any.whl

freeplay/record.py

@@ -1,14 +1,18 @@
 import logging
 from dataclasses import dataclass
-from typing import Any, Dict, Optional
+from typing import Dict, Optional, Union
 from abc import abstractmethod, ABC
+from dataclasses import dataclass
+from typing import Optional
 
-from .llm_parameters import LLMParameters
-from .completions import PromptTemplateWithMetadata
 from . import api_support
+from .completions import PromptTemplateWithMetadata
+from .llm_parameters import LLMParameters
+from .model import InputVariables
 
 logger = logging.getLogger(__name__)
 
+
 @dataclass
 class RecordCallFields:
     completion_content: str
@@ -18,13 +22,15 @@ class RecordCallFields:
     session_id: str
     start: float
     target_template: PromptTemplateWithMetadata
-    variables: Dict[str, str]
+    variables: InputVariables
     tag: str
     test_run_id: Optional[str]
+    test_case_id: Optional[str]
     record_format_type: Optional[str]
     model: Optional[str]
     provider: Optional[str]
     llm_parameters: Optional[LLMParameters]
+    custom_metadata: Optional[Dict[str, Union[str,int,float]]] = None
 
 
 class RecordProcessor(ABC):
@@ -35,6 +41,7 @@ class RecordProcessor(ABC):
     ) -> None:
         pass
 
+
 class NoOpRecorder(RecordProcessor):
     def record_call(
             self,
@@ -42,8 +49,10 @@ class NoOpRecorder(RecordProcessor):
     ) -> None:
         pass
 
+
 no_op_recorder = NoOpRecorder()
 
+
 class DefaultRecordProcessor(RecordProcessor):
 
     def __init__(
@@ -55,8 +64,8 @@ class DefaultRecordProcessor(RecordProcessor):
         self.freeplay_api_key = freeplay_api_key
 
     def record_call(
-        self,
-        record_call: RecordCallFields
+            self,
+            record_call: RecordCallFields
     ) -> None:
         record_payload = {
             "session_id": record_call.session_id,
@@ -72,12 +81,16 @@ class DefaultRecordProcessor(RecordProcessor):
             "is_complete": record_call.completion_is_complete,
             "model": record_call.model,
             "provider": record_call.provider,
-            "llm_parameters": record_call.llm_parameters
+            "llm_parameters": record_call.llm_parameters,
+            "custom_metadata": record_call.custom_metadata,
         }
 
         if record_call.test_run_id is not None:
             record_payload['test_run_id'] = record_call.test_run_id
 
+        if record_call.test_case_id is not None:
+            record_payload['test_case_id'] = record_call.test_case_id
+
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.freeplay_api_key,

freeplay/support.py

@@ -0,0 +1,316 @@
+import json
+import time
+from copy import copy
+from typing import Dict, Any, Optional, Union, List, Generator
+from uuid import uuid4
+
+from freeplay import api_support
+from freeplay.api_support import try_decode
+from freeplay.completions import PromptTemplates, PromptTemplateWithMetadata, ChatMessage, ChatCompletionResponse, \
+    CompletionChunk, CompletionResponse
+from freeplay.errors import FreeplayConfigurationError, freeplay_response_error, FreeplayServerError
+from freeplay.flavors import ChatFlavor, Flavor, pick_flavor_from_config
+from freeplay.llm_parameters import LLMParameters
+from freeplay.model import InputVariables
+from freeplay.provider_config import ProviderConfig
+from freeplay.record import RecordProcessor, RecordCallFields
+
+JsonDom = Dict[str, Any]
+
+
+class TestCaseTestRunResponse:
+    def __init__(self, test_case: JsonDom):
+        self.variables: InputVariables = test_case['variables']
+        self.id: str = test_case['id']
+
+
+class TestRunResponse:
+    def __init__(
+            self,
+            test_run_id: str,
+            test_cases: list[JsonDom]
+    ):
+        self.test_cases = [
+            TestCaseTestRunResponse(test_case)
+            for test_case in test_cases
+        ]
+        self.test_run_id = test_run_id
+
+
+class CallSupport:
+    def __init__(
+            self,
+            freeplay_api_key: str,
+            api_base: str,
+            record_processor: RecordProcessor,
+            **kwargs: Any
+    ) -> None:
+        self.api_base = api_base
+        self.freeplay_api_key = freeplay_api_key
+        self.client_params = LLMParameters(kwargs)
+        self.record_processor = record_processor
+
+    @staticmethod
+    def find_template_by_name(prompts: PromptTemplates, template_name: str) -> PromptTemplateWithMetadata:
+        templates = [t for t in prompts.templates if t.name == template_name]
+        if len(templates) == 0:
+            raise FreeplayConfigurationError(f'Could not find template with name "{template_name}"')
+        return templates[0]
+
+    def create_session_id(self) -> str:
+        return str(uuid4())
+
+    def check_all_values_string_or_number(self, metadata: Optional[Dict[str, Union[str, int, float]]]) -> None:
+        if metadata:
+            for key, value in metadata.items():
+                if not isinstance(value, (str, int, float)):
+                    raise FreeplayConfigurationError(f"Invalid value for key {key}: Value must be a string or number.")
+
+    def get_prompt(self, project_id: str, template_name: str, environment: str) -> PromptTemplateWithMetadata:
+        prompt_templates = self.get_prompts(project_id, environment)
+        return self.find_template_by_name(prompt_templates, template_name)
+
+    def get_prompts(self, project_id: str, tag: str) -> PromptTemplates:
+        response = api_support.get_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/projects/{project_id}/templates/all/{tag}'
+        )
+
+        if response.status_code != 200:
+            raise freeplay_response_error("Error getting prompt templates", response)
+
+        maybe_prompts = try_decode(PromptTemplates, response.content)
+        if maybe_prompts is None:
+            raise FreeplayServerError(f'Failed to parse prompt templates from server')
+
+        return maybe_prompts
+
+    def create_test_run(self, project_id: str, testlist: str) -> TestRunResponse:
+        response = api_support.post_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/projects/{project_id}/test-runs-cases',
+            payload={'testlist_name': testlist},
+        )
+
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while creating a test run.', response)
+
+        json_dom = response.json()
+
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+
+    # noinspection PyUnboundLocalVariable
+    def prepare_and_make_chat_call(
+            self,
+            session_id: str,
+            flavor: ChatFlavor,
+            provider_config: ProviderConfig,
+            tag: str,
+            target_template: PromptTemplateWithMetadata,
+            variables: InputVariables,
+            message_history: List[ChatMessage],
+            new_messages: Optional[List[ChatMessage]],
+            test_run_id: Optional[str] = None,
+            completion_parameters: Optional[LLMParameters] = None,
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
+    ) -> ChatCompletionResponse:
+        # make call
+        start = time.time()
+        params = target_template.get_params() \
+            .merge_and_override(self.client_params) \
+            .merge_and_override(completion_parameters)
+        prompt_messages = copy(message_history)
+        if new_messages is not None:
+            prompt_messages.extend(new_messages)
+        completion_response = flavor.continue_chat(messages=prompt_messages,
+                                                   provider_config=provider_config,
+                                                   llm_parameters=params)
+        end = time.time()
+
+        model = flavor.get_model_params(params).get('model')
+        formatted_prompt = json.dumps(prompt_messages)
+        # record data
+        record_call_fields = RecordCallFields(
+            completion_content=completion_response.content,
+            completion_is_complete=completion_response.is_complete,
+            end=end,
+            formatted_prompt=formatted_prompt,
+            session_id=session_id,
+            start=start,
+            target_template=target_template,
+            variables=variables,
+            record_format_type=flavor.record_format_type,
+            tag=tag,
+            test_run_id=test_run_id,
+            test_case_id=None,
+            model=model,
+            provider=flavor.provider,
+            llm_parameters=params,
+            custom_metadata=metadata,
+        )
+        self.record_processor.record_call(record_call_fields)
+
+        return completion_response
+
+    # noinspection PyUnboundLocalVariable
+    def prepare_and_make_chat_call_stream(
+            self,
+            session_id: str,
+            flavor: ChatFlavor,
+            provider_config: ProviderConfig,
+            tag: str,
+            target_template: PromptTemplateWithMetadata,
+            variables: InputVariables,
+            message_history: List[ChatMessage],
+            test_run_id: Optional[str] = None,
+            completion_parameters: Optional[LLMParameters] = None,
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
+    ) -> Generator[CompletionChunk, None, None]:
+        # make call
+        start = time.time()
+        prompt_messages = copy(message_history)
+        params = target_template.get_params() \
+            .merge_and_override(self.client_params) \
+            .merge_and_override(completion_parameters)
+        completion_response = flavor.continue_chat_stream(prompt_messages, provider_config, llm_parameters=params)
+
+        str_content = ''
+        last_is_complete = False
+        for chunk in completion_response:
+            str_content += chunk.text or ''
+            last_is_complete = chunk.is_complete
+            yield chunk
+        # End time must be logged /after/ streaming the response above, or else OpenAI latency will not be captured.
+        end = time.time()
+
+        model = flavor.get_model_params(params).get('model')
+        formatted_prompt = json.dumps(prompt_messages)
+        record_call_fields = RecordCallFields(
+            completion_content=str_content,
+            completion_is_complete=last_is_complete,
+            end=end,
+            formatted_prompt=formatted_prompt,
+            session_id=session_id,
+            start=start,
+            target_template=target_template,
+            variables=variables,
+            record_format_type=flavor.record_format_type,
+            tag=tag,
+            test_run_id=test_run_id,
+            test_case_id=None,
+            model=model,
+            provider=flavor.provider,
+            llm_parameters=params,
+            custom_metadata=metadata,
+        )
+        self.record_processor.record_call(record_call_fields)
+
+    # noinspection PyUnboundLocalVariable
+    def prepare_and_make_call(
+            self,
+            session_id: str,
+            prompts: PromptTemplates,
+            template_name: str,
+            variables: InputVariables,
+            flavor: Optional[Flavor],
+            provider_config: ProviderConfig,
+            tag: str,
+            test_run_id: Optional[str] = None,
+            completion_parameters: Optional[LLMParameters] = None,
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
+    ) -> CompletionResponse:
+        target_template = self.find_template_by_name(prompts, template_name)
+        params = target_template.get_params() \
+            .merge_and_override(self.client_params) \
+            .merge_and_override(completion_parameters)
+
+        final_flavor = pick_flavor_from_config(flavor, target_template.flavor_name)
+        formatted_prompt = final_flavor.format(target_template, variables)
+
+        # make call
+        start = time.time()
+        completion_response = final_flavor.call_service(formatted_prompt=formatted_prompt,
+                                                        provider_config=provider_config,
+                                                        llm_parameters=params)
+        end = time.time()
+
+        model = final_flavor.get_model_params(params).get('model')
+
+        # record data
+        record_call_fields = RecordCallFields(
+            completion_content=completion_response.content,
+            completion_is_complete=completion_response.is_complete,
+            end=end,
+            formatted_prompt=formatted_prompt,
+            session_id=session_id,
+            start=start,
+            target_template=target_template,
+            variables=variables,
+            record_format_type=final_flavor.record_format_type,
+            tag=tag,
+            test_run_id=test_run_id,
+            test_case_id=None,
+            model=model,
+            provider=final_flavor.provider,
+            llm_parameters=params,
+            custom_metadata=metadata,
+        )
+        self.record_processor.record_call(record_call_fields)
+
+        return completion_response
+
+    def prepare_and_make_call_stream(
+            self,
+            session_id: str,
+            prompts: PromptTemplates,
+            template_name: str,
+            variables: InputVariables,
+            flavor: Optional[Flavor],
+            provider_config: ProviderConfig,
+            tag: str,
+            test_run_id: Optional[str] = None,
+            completion_parameters: Optional[LLMParameters] = None,
+            metadata: Optional[Dict[str, Union[str, int, float]]] = None
+    ) -> Generator[CompletionChunk, None, None]:
+        target_template = self.find_template_by_name(prompts, template_name)
+        params = target_template.get_params() \
+            .merge_and_override(self.client_params) \
+            .merge_and_override(completion_parameters)
+
+        final_flavor = pick_flavor_from_config(flavor, target_template.flavor_name)
+        formatted_prompt = final_flavor.format(target_template, variables)
+
+        # make call
+        start = int(time.time())
+        completion_response = final_flavor.call_service_stream(
+            formatted_prompt=formatted_prompt, provider_config=provider_config, llm_parameters=params)
+        text_chunks = []
+        last_is_complete = False
+        for chunk in completion_response:
+            text_chunks.append(chunk.text)
+            last_is_complete = chunk.is_complete
+            yield chunk
+        # End time must be logged /after/ streaming the response above, or else OpenAI latency will not be captured.
+        end = int(time.time())
+
+        model = final_flavor.get_model_params(params).get('model')
+
+        record_call_fields = RecordCallFields(
+            completion_content=''.join(text_chunks),
+            completion_is_complete=last_is_complete,
+            end=end,
+            formatted_prompt=formatted_prompt,
+            session_id=session_id,
+            start=start,
+            target_template=target_template,
+            variables=variables,
+            record_format_type=final_flavor.record_format_type,
+            tag=tag,
+            test_run_id=test_run_id,
+            test_case_id=None,
+            model=model,
+            provider=final_flavor.provider,
+            llm_parameters=params,
+            custom_metadata=metadata,
+        )
+        self.record_processor.record_call(record_call_fields)

freeplay/utils.py

@@ -1,21 +1,51 @@
+from typing import Dict, Union, Optional
+import importlib.metadata
+import platform
+
 import pystache  # type: ignore
-from pydantic import RootModel, ValidationError
-from typing import Any, Dict, List, Union
+from pydantic import ValidationError
 
-from .errors import FreeplayError
+from .errors import FreeplayError, FreeplayConfigurationError
+from .model import PydanticInputVariables, InputVariables
 
-InputVariable = RootModel[Union[Dict[str, "InputVariable"], List["InputVariable"], str, int, bool, float]]
-InputVariable.model_rebuild()
-InputVariables = Dict[str, Union[str, int, bool, Dict[str, Any], List[Any]]]
-PydanticInputVariables = RootModel[Dict[str, InputVariable]]
 
-def format_template_variables(template: str, variables: Any) -> str:
+def bind_template_variables(template: str, variables: InputVariables) -> str:
     # Validate that the variables are of the correct type, and do not include functions or None values.
     try:
         PydanticInputVariables.model_validate(variables)
     except ValidationError as err:
-        raise FreeplayError('Variables must be a string, number, bool, or a possibly nested list or dict of strings, numbers and booleans.')
+        raise FreeplayError(
+            'Variables must be a string, number, bool, or a possibly nested'
+            ' list or dict of strings, numbers and booleans.'
+        )
 
     # When rendering mustache, do not escape HTML special characters.
     rendered: str = pystache.Renderer(escape=lambda s: s).render(template, variables)
     return rendered
+
+
+def check_all_values_string_or_number(metadata: Optional[Dict[str, Union[str, int, float]]]) -> None:
+    if metadata:
+        for key, value in metadata.items():
+            if not isinstance(value, (str, int, float)):
+                raise FreeplayConfigurationError(f"Invalid value for key {key}: Value must be a string or number.")
+
+
+def build_request_header(api_key: str) -> Dict[str, str]:
+    return {
+        'Authorization': f'Bearer {api_key}',
+        'User-Agent': get_user_agent()
+    }
+
+
+def get_user_agent() -> str:
+    sdk_name = 'Freeplay'
+    sdk_version = importlib.metadata.version('Freeplay')
+    language = 'Python'
+    language_version = platform.python_version()
+    os_name = platform.system()
+    os_version = platform.release()
+
+    # Output format
+    # Freeplay/0.2.30 (Python/3.11.4; Darwin/23.2.0)
+    return f"{sdk_name}/{sdk_version} ({language}/{language_version}; {os_name}/{os_version})"

{freeplay-0.2.30.dist-info → freeplay-0.2.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freeplay
-Version: 0.2.30
+Version: 0.2.32
 Summary: 
 License: MIT
 Author: FreePlay Engineering

freeplay-0.2.32.dist-info/RECORD

@@ -0,0 +1,20 @@
+freeplay/__init__.py,sha256=74A9S9hmLq9BNHsdx0-37yDxlSukudNl9bJ0TE60Z30,61
+freeplay/api_support.py,sha256=E4Mxa3Lx31TEw_X6o4s9eAR1TSxs0PhFYYaWax6dH2I,2027
+freeplay/completions.py,sha256=F1rMBtQaCtn0rBQqvCurkV25g8gLtwnEod5rRvf-txY,1176
+freeplay/errors.py,sha256=bPqsw32YX-xSr7O-G49M0sSFF7mq-YF1WGq928UV47s,631
+freeplay/flavors.py,sha256=XroBKT8Nf92VTsuC261Nceo9f9stkkpC5CXbi8y4NEs,17236
+freeplay/freeplay.py,sha256=BLlr4YXh3a624xcM93KvZGsiIGSMrIMFM07lLyVCIao,17089
+freeplay/freeplay_cli.py,sha256=nasbc_ckSr5-YtUKfg_w-6X1geQZ9s5u79VzRULGsbs,3868
+freeplay/freeplay_thin.py,sha256=WHoVCkS30mDM7ghjDadgfj48BNL5EvaXktUsGfeXlIY,867
+freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
+freeplay/model.py,sha256=kVARXUEJKbhOdWz7T8eR7npTPXlpaY9wWaEzJvbIvOU,534
+freeplay/provider_config.py,sha256=hruf3Khusrwb76_-hv7ouuxmvJuaRyC1UxIw7XlJx8A,1416
+freeplay/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+freeplay/record.py,sha256=_0uo0rUFZz8kwgWC0JhxFg13JyXgMfAVCkzZsuRLzBA,3354
+freeplay/support.py,sha256=OOnUZOMYnLSEG_3toycWfHb2XZ8meF77uQRllYexXeo,12616
+freeplay/utils.py,sha256=cRCCIzVqWNDKlTI-DDhXGyCkplbd-X4qzDs__aUpvww,1840
+freeplay-0.2.32.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
+freeplay-0.2.32.dist-info/METADATA,sha256=o9MiS7BrkV0M2Erp7bKomeRUBgveiqW6XQf7LLyVA4s,1633
+freeplay-0.2.32.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+freeplay-0.2.32.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
+freeplay-0.2.32.dist-info/RECORD,,

{freeplay-0.2.30.dist-info → freeplay-0.2.32.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.7.0
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any

freeplay-0.2.30.dist-info/RECORD

@@ -1,17 +0,0 @@
-freeplay/__init__.py,sha256=74A9S9hmLq9BNHsdx0-37yDxlSukudNl9bJ0TE60Z30,61
-freeplay/api_support.py,sha256=FJCm7yb1JhY0Z1Hn-JIVM7cCfBvWfCrqMQHRzzBzGLE,2012
-freeplay/completions.py,sha256=JQtwe5THpkWqywUvOLyQFNa9t8kzNhIVkJni67U3HyI,1165
-freeplay/errors.py,sha256=bPqsw32YX-xSr7O-G49M0sSFF7mq-YF1WGq928UV47s,631
-freeplay/flavors.py,sha256=AUVAjZaP8VtK5F3NtQOa7_LD9YIs84gUafbUIqt6U84,18492
-freeplay/freeplay.py,sha256=ObfW0QnSzuagfF5aMC44NPAZIokED3BUALuhqUY5-SA,28760
-freeplay/freeplay_cli.py,sha256=nasbc_ckSr5-YtUKfg_w-6X1geQZ9s5u79VzRULGsbs,3868
-freeplay/freeplay_thin.py,sha256=2YFfqFCYimUboRg_umasvHqi2-tzWTvpuf5b9-mQK3I,827
-freeplay/llm_parameters.py,sha256=bQbfuC8EICF0XMZQa5pwI3FkQqxmCUVqHO3gYHy3Tg8,898
-freeplay/provider_config.py,sha256=hruf3Khusrwb76_-hv7ouuxmvJuaRyC1UxIw7XlJx8A,1416
-freeplay/record.py,sha256=pNLu5vPw6HyyPz5u-32pDfElY3hxeujaM1h7Xzah-VE,2961
-freeplay/utils.py,sha256=SxZYKF5H6uotQa0m2B92CTk3ADr5N5SsUS_a6HQ0fnk,1026
-freeplay-0.2.30.dist-info/LICENSE,sha256=_jzIw45hB1XHGxiQ8leZ0GH_X7bR_a8qgxaqnHbCUOo,1064
-freeplay-0.2.30.dist-info/METADATA,sha256=XE8Y8IZibHed27Gi74EUrlfacpqVxWGxF7eyjE5mQZs,1633
-freeplay-0.2.30.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-freeplay-0.2.30.dist-info/entry_points.txt,sha256=32s3rf2UUCqiJT4jnClEXZhdXlvl30uwpcxz-Gsy4UU,54
-freeplay-0.2.30.dist-info/RECORD,,