freeplay 0.3.0a7__tar.gz → 0.3.0a9__tar.gz

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: freeplay
-Version: 0.3.0a7
+Version: 0.3.0a9
 Summary: 
 License: MIT
 Author: FreePlay Engineering
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: click (==8.1.7)
 Requires-Dist: dacite (>=1.8.0,<2.0.0)
 Requires-Dist: pystache (>=0.6.5,<0.7.0)

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "freeplay"
-version = "0.3.0-alpha.7"
+version = "0.3.0-alpha.9"
 description = ""
 authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
 license = "MIT"
@@ -19,6 +19,7 @@ types-requests = "^2.31"
 anthropic = { version="^0.20.0", extras = ["bedrock"] }
 openai = "^1"
 boto3 = "^1.34.97"
+google-cloud-aiplatform = "1.51.0"
 
 [tool.poetry.group.test.dependencies]
 responses = "^0.23.1"

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/src/freeplay/resources/prompts.py

@@ -27,12 +27,13 @@ class PromptInfo:
     prompt_template_id: str
     prompt_template_version_id: str
     template_name: str
-    environment: str
+    environment: Optional[str]
     model_parameters: LLMParameters
     provider_info: Optional[Dict[str, Any]]
     provider: str
     model: str
     flavor_name: str
+    project_id: str
 
 
 class FormattedPrompt:
@@ -76,7 +77,7 @@ class BoundPrompt:
             flavor_name: str,
             messages: List[Dict[str, str]]
     ) -> Union[str, List[Dict[str, str]]]:
-        if flavor_name == 'azure_openai_chat' or flavor_name == 'openai_chat':
+        if flavor_name in ['azure_openai_chat', 'openai_chat', 'baseten_mistral_chat', 'mistral_chat']:
             # We need a deepcopy here to avoid referential equality with the llm_prompt
             return copy.deepcopy(messages)
         elif flavor_name == 'anthropic_chat':
@@ -91,6 +92,24 @@ class BoundPrompt:
                 formatted += f"<|start_header_id|>{message['role']}<|end_header_id|>\n{message['content']}<|eot_id|>"
             formatted += "<|start_header_id|>assistant<|end_header_id|>"
 
+            return formatted
+        elif flavor_name == 'gemini_chat':
+            if len(messages) < 1:
+                raise ValueError("Must have at least one message to format")
+
+            def translate_role(role: str) -> str:
+                if role == "user":
+                    return "user"
+                elif role == "assistant":
+                    return "model"
+                else:
+                    raise ValueError(f"Gemini formatting found unexpected role {role}")
+
+            formatted = [  # type: ignore
+                {'role': translate_role(message['role']), 'parts': [{'text': message['content']}]}
+                for message in messages if message['role'] != 'system'
+            ]
+
             return formatted
 
         raise MissingFlavorError(flavor_name)
@@ -142,6 +161,10 @@ class TemplateResolver(ABC):
     def get_prompt(self, project_id: str, template_name: str, environment: str) -> PromptTemplate:
         pass
 
+    @abstractmethod
+    def get_prompt_version_id(self, project_id: str, template_id: str, version_id: str) -> PromptTemplate:
+        pass
+
 
 class FilesystemTemplateResolver(TemplateResolver):
     # If you think you need a change here, be sure to check the server as the translations must match. Once we have
@@ -185,6 +208,27 @@ class FilesystemTemplateResolver(TemplateResolver):
         json_dom = json.loads(expected_file.read_text())
         return self.__render_into_v2(json_dom)
 
+    def get_prompt_version_id(self, project_id: str, template_id: str, version_id: str) -> PromptTemplate:
+
+        expected_file: Path = self.prompts_directory / project_id
+
+        if not expected_file.exists():
+            raise FreeplayClientError(
+                f"Could not find project id {project_id}"
+            )
+
+        # read all files in the project directory
+        prompt_file_paths = expected_file.glob("**/*.json")
+        # find the file with the matching version id
+        for prompt_file_path in prompt_file_paths:
+            json_dom = json.loads(prompt_file_path.read_text())
+            if json_dom.get('prompt_template_version_id') == version_id:
+                return self.__render_into_v2(json_dom)
+
+        raise FreeplayClientError(
+            f"Could not find prompt with version id {version_id} for project {project_id}"
+        )
+
     @staticmethod
     def __render_into_v2(json_dom: Dict[str, Any]) -> PromptTemplate:
         format_version = json_dom.get('format_version')
@@ -206,7 +250,8 @@ class FilesystemTemplateResolver(TemplateResolver):
                     model=model,
                     params=metadata.get('params'),
                     provider_info=metadata.get('provider_info')
-                )
+                ),
+                project_id=str(json_dom.get('project_id'))
             )
         else:
             metadata = json_dom['metadata']
@@ -227,7 +272,8 @@ class FilesystemTemplateResolver(TemplateResolver):
                     model=model,
                     params=params,
                     provider_info=None
-                )
+                ),
+                project_id=str(json_dom.get('project_id'))
             )
 
     @staticmethod
@@ -291,6 +337,13 @@ class APITemplateResolver(TemplateResolver):
             environment=environment
         )
 
+    def get_prompt_version_id(self, project_id: str, template_id: str, version_id: str) -> PromptTemplate:
+        return self.call_support.get_prompt_version_id(
+            project_id=project_id,
+            template_id=template_id,
+            version_id=version_id
+        )
+
 
 class Prompts:
     def __init__(self, call_support: CallSupport, template_resolver: TemplateResolver) -> None:
@@ -327,7 +380,41 @@ class Prompts:
             provider=prompt.metadata.provider,
             model=model,
             flavor_name=prompt.metadata.flavor,
-            provider_info=prompt.metadata.provider_info
+            provider_info=prompt.metadata.provider_info,
+            project_id=prompt.project_id
+        )
+
+        return TemplatePrompt(prompt_info, prompt.content)
+
+    def get_by_version_id(self, project_id: str, template_id: str, version_id: str) -> TemplatePrompt:
+        prompt = self.template_resolver.get_prompt_version_id(project_id, template_id, version_id)
+
+        params = prompt.metadata.params
+        model = prompt.metadata.model
+
+        if not model:
+            raise FreeplayConfigurationError(
+                "Model must be configured in the Freeplay UI. Unable to fulfill request.")
+
+        if not prompt.metadata.flavor:
+            raise FreeplayConfigurationError(
+                "Flavor must be configured in the Freeplay UI. Unable to fulfill request.")
+
+        if not prompt.metadata.provider:
+            raise FreeplayConfigurationError(
+                "Provider must be configured in the Freeplay UI. Unable to fulfill request.")
+
+        prompt_info = PromptInfo(
+            prompt_template_id=prompt.prompt_template_id,
+            prompt_template_version_id=prompt.prompt_template_version_id,
+            template_name=prompt.prompt_template_name,
+            environment=prompt.environment if prompt.environment else '',
+            model_parameters=cast(LLMParameters, params) or LLMParameters({}),
+            provider=prompt.metadata.provider,
+            model=model,
+            flavor_name=prompt.metadata.flavor,
+            provider_info=prompt.metadata.provider_info,
+            project_id=prompt.project_id
         )
 
         return TemplatePrompt(prompt_info, prompt.content)
@@ -347,3 +434,19 @@ class Prompts:
         ).bind(variables=variables)
 
         return bound_prompt.format(flavor_name)
+
+    def get_formatted_by_version_id(
+            self,
+            project_id: str,
+            template_id: str,
+            version_id: str,
+            variables: InputVariables,
+            flavor_name: Optional[str] = None,
+    ) -> FormattedPrompt:
+        bound_prompt = self.get_by_version_id(
+            project_id=project_id,
+            template_id=template_id,
+            version_id=version_id
+        ).bind(variables=variables)
+
+        return bound_prompt.format(flavor_name)

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/src/freeplay/resources/recordings.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from requests import HTTPError
 
@@ -39,7 +39,7 @@ class CallInfo:
 
 @dataclass
 class ResponseInfo:
-    is_complete: bool
+    is_complete: Optional[bool] = None
     function_call_response: Optional[OpenAIFunctionCall] = None
     prompt_tokens: Optional[int] = None
     response_tokens: Optional[int] = None
@@ -59,8 +59,9 @@ class RecordPayload:
     session_info: SessionInfo
     prompt_info: PromptInfo
     call_info: CallInfo
-    response_info: ResponseInfo
+    response_info: Optional[ResponseInfo] = None
     test_run_info: Optional[TestRunInfo] = None
+    eval_results: Optional[Dict[str, Union[bool, float]]] = None
 
 
 @dataclass
@@ -77,44 +78,49 @@ class Recordings:
             raise FreeplayClientError("Messages list must have at least one message. "
                                       "The last message should be the current response.")
 
-        completion = record_payload.all_messages[-1]
-        history_as_string = json.dumps(record_payload.all_messages[0:-1])
-
         record_api_payload = {
-            "session_id": record_payload.session_info.session_id,
-            "prompt_template_id": record_payload.prompt_info.prompt_template_id,
-            "project_version_id": record_payload.prompt_info.prompt_template_version_id,
-            "start_time": record_payload.call_info.start_time,
-            "end_time": record_payload.call_info.end_time,
-            "tag": record_payload.prompt_info.environment,
+            "messages": record_payload.all_messages,
             "inputs": record_payload.inputs,
-            "prompt_content": history_as_string,
-            # Content may not be set for function calls, but it is required in the record API payload.
-            "return_content": completion.get('content', ''),
-            "format_type": None,
-            "is_complete": record_payload.response_info.is_complete,
-            "model": record_payload.call_info.model,
-            "provider": record_payload.call_info.provider,
-            "llm_parameters": record_payload.call_info.model_parameters,
-            "provider_info": record_payload.call_info.provider_info,
+            "session_info": {"custom_metadata": record_payload.session_info.custom_metadata},
+            "prompt_info": {
+                "environment": record_payload.prompt_info.environment,
+                "prompt_template_version_id": record_payload.prompt_info.prompt_template_version_id,
+            },
+            "call_info": {
+                "start_time": record_payload.call_info.start_time,
+                "end_time": record_payload.call_info.end_time,
+                "model": record_payload.call_info.model,
+                "provider": record_payload.call_info.provider,
+                "provider_info": record_payload.call_info.provider_info,
+                "llm_parameters": record_payload.call_info.model_parameters,
+            }
         }
 
         if record_payload.session_info.custom_metadata is not None:
             record_api_payload['custom_metadata'] = record_payload.session_info.custom_metadata
 
-        if record_payload.response_info.function_call_response is not None:
-            record_api_payload['function_call_response'] = record_payload.response_info.function_call_response
+        if record_payload.response_info is not None:
+            if record_payload.response_info.function_call_response is not None:
+                record_api_payload['response_info'] = {
+                    "function_call_response": {
+                        "name": record_payload.response_info.function_call_response["name"],
+                        "arguments": record_payload.response_info.function_call_response["arguments"],
+                    }
+                }
 
         if record_payload.test_run_info is not None:
-            record_api_payload['test_run_id'] = record_payload.test_run_info.test_run_id
+            record_api_payload['test_run_info'] = {
+                "test_run_id": record_payload.test_run_info.test_run_id,
+                "test_case_id": record_payload.test_run_info.test_case_id
+            }
 
-        if record_payload.test_run_info is not None:
-            record_api_payload['test_case_id'] = record_payload.test_run_info.test_case_id
+        if record_payload.eval_results is not None:
+            record_api_payload['eval_results'] = record_payload.eval_results
 
         try:
             recorded_response = api_support.post_raw(
                 api_key=self.call_support.freeplay_api_key,
-                url=f'{self.call_support.api_base}/v1/record',
+                url=f'{self.call_support.api_base}/v2/projects/{record_payload.prompt_info.project_id}/sessions/{record_payload.session_info.session_id}/completions',
                 payload=record_api_payload
             )
             recorded_response.raise_for_status()

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/src/freeplay/resources/test_runs.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 
 from freeplay.model import InputVariables
 from freeplay.resources.recordings import TestRunInfo
-from freeplay.support import CallSupport
+from freeplay.support import CallSupport, SummaryStatistics
 
 
 @dataclass
@@ -35,16 +35,46 @@ class TestRun:
     def get_test_run_info(self, test_case_id: str) -> TestRunInfo:
         return TestRunInfo(self.test_run_id, test_case_id)
 
+@dataclass
+class TestRunResults:
+    def __init__(
+            self,
+            name: str,
+            description: str,
+            test_run_id: str,
+            summary_statistics: SummaryStatistics,
+    ):
+        self.name = name
+        self.description = description
+        self.test_run_id = test_run_id
+        self.summary_statistics = summary_statistics
+
 
 class TestRuns:
     def __init__(self, call_support: CallSupport) -> None:
         self.call_support = call_support
 
-    def create(self, project_id: str, testlist: str, include_outputs: bool = False) -> TestRun:
-        test_run = self.call_support.create_test_run(project_id, testlist, include_outputs)
+    def create(
+            self,
+            project_id: str,
+            testlist: str,
+            include_outputs: bool = False,
+            name: Optional[str] = None,
+            description: Optional[str] = None
+    ) -> TestRun:
+        test_run = self.call_support.create_test_run(project_id, testlist, include_outputs, name, description)
         test_cases = [
             TestCase(test_case_id=test_case.id, variables=test_case.variables, output=test_case.output)
             for test_case in test_run.test_cases
         ]
 
         return TestRun(test_run.test_run_id, test_cases)
+
+    def get(self, project_id: str, test_run_id: str) -> TestRunResults:
+        test_run_results = self.call_support.get_test_run_results(project_id, test_run_id)
+        return TestRunResults(
+            test_run_results.name,
+            test_run_results.description,
+            test_run_results.test_run_id,
+            test_run_results.summary_statistics
+        )

{freeplay-0.3.0a7 → freeplay-0.3.0a9}/src/freeplay/support.py

@@ -24,13 +24,20 @@ class PromptTemplate:
     prompt_template_name: str
     content: List[Dict[str, str]]
     metadata: PromptTemplateMetadata
+    project_id: str
     format_version: int
+    environment: Optional[str] = None
 
 
 @dataclass
 class PromptTemplates:
     prompt_templates: List[PromptTemplate]
 
+@dataclass
+class SummaryStatistics:
+    auto_evaluation: Dict[str, Any]
+    human_evaluation: Dict[str, Any]
+
 
 class PromptTemplateEncoder(JSONEncoder):
     def default(self, prompt_template: PromptTemplate) -> Dict[str, Any]:
@@ -40,7 +47,7 @@ class PromptTemplateEncoder(JSONEncoder):
 class TestCaseTestRunResponse:
     def __init__(self, test_case: Dict[str, Any]):
         self.variables: InputVariables = test_case['variables']
-        self.id: str = test_case['id']
+        self.id: str = test_case['test_case_id']
         self.output: Optional[str] = test_case.get('output')
 
 
@@ -57,6 +64,23 @@ class TestRunResponse:
         self.test_run_id = test_run_id
 
 
+class TestRunRetrievalResponse:
+    def __init__(
+            self,
+            name: str,
+            description: str,
+            test_run_id: str,
+            summary_statistics: Dict[str, Any],
+    ):
+        self.name = name
+        self.description = description
+        self.test_run_id = test_run_id
+        self.summary_statistics = SummaryStatistics(
+            auto_evaluation=summary_statistics['auto_evaluation'],
+            human_evaluation=summary_statistics['human_evaluation']
+        )
+
+
 class CallSupport:
     def __init__(
             self,
@@ -106,6 +130,26 @@ class CallSupport:
 
         return maybe_prompt
 
+    def get_prompt_version_id(self, project_id: str, template_id: str, version_id: str) -> PromptTemplate:
+        response = api_support.get_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/v2/projects/{project_id}/prompt-templates/id/{template_id}/versions/{version_id}'
+        )
+
+        if response.status_code != 200:
+            raise freeplay_response_error(
+                f"Error getting version id {version_id} for template {template_id} in project {project_id}",
+                response
+            )
+
+        maybe_prompt = try_decode(PromptTemplate, response.content)
+        if maybe_prompt is None:
+            raise FreeplayServerError(
+                f"Error handling version id {version_id} for template {template_id} in project {project_id}"
+            )
+
+        return maybe_prompt
+
     def update_customer_feedback(
             self,
             completion_id: str,
@@ -123,14 +167,18 @@ class CallSupport:
             self,
             project_id: str,
             testlist: str,
-            include_test_case_outputs: bool = False
+            include_test_case_outputs: bool = False,
+            name: Optional[str] = None,
+            description: Optional[str] = None
     ) -> TestRunResponse:
         response = api_support.post_raw(
             api_key=self.freeplay_api_key,
-            url=f'{self.api_base}/projects/{project_id}/test-runs-cases',
+            url=f'{self.api_base}/v2/projects/{project_id}/test-runs',
             payload={
-                'testlist_name': testlist,
+                'dataset_name': testlist,
                 'include_test_case_outputs': include_test_case_outputs,
+                'name': name,
+                'description': description
             },
         )
 
@@ -139,4 +187,26 @@ class CallSupport:
 
         json_dom = response.json()
 
-        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+        return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
+
+    def get_test_run_results(
+            self,
+            project_id: str,
+            test_run_id: str,
+    ) -> TestRunRetrievalResponse:
+        response = api_support.get_raw(
+            api_key=self.freeplay_api_key,
+            url=f'{self.api_base}/v2/projects/{project_id}/test-runs/id/{test_run_id}'
+        )
+        if response.status_code != 201:
+            raise freeplay_response_error('Error while retrieving test run results.', response)
+
+        json_dom = response.json()
+
+        return TestRunRetrievalResponse(
+            name=json_dom['name'],
+            description=json_dom['description'],
+            test_run_id=json_dom['id'],
+            summary_statistics=json_dom['summary_statistics']
+        )
+