freeplay 0.2.42__tar.gz → 0.3.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {freeplay-0.2.42 → freeplay-0.3.0a2}/PKG-INFO +1 -3
- {freeplay-0.2.42 → freeplay-0.3.0a2}/pyproject.toml +2 -3
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/__init__.py +2 -2
- freeplay-0.2.42/src/freeplay/thin/freeplay_thin.py → freeplay-0.3.0a2/src/freeplay/freeplay.py +6 -8
- {freeplay-0.2.42 → freeplay-0.3.0a2}/src/freeplay/freeplay_cli.py +14 -28
- freeplay-0.3.0a2/src/freeplay/model.py +17 -0
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/prompts.py +97 -41
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/recordings.py +5 -15
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/test_runs.py +1 -1
- freeplay-0.3.0a2/src/freeplay/support.py +142 -0
- {freeplay-0.2.42 → freeplay-0.3.0a2}/src/freeplay/utils.py +15 -7
- freeplay-0.2.42/src/freeplay/__init__.py +0 -5
- freeplay-0.2.42/src/freeplay/completions.py +0 -56
- freeplay-0.2.42/src/freeplay/flavors.py +0 -459
- freeplay-0.2.42/src/freeplay/freeplay.py +0 -426
- freeplay-0.2.42/src/freeplay/model.py +0 -20
- freeplay-0.2.42/src/freeplay/provider_config.py +0 -49
- freeplay-0.2.42/src/freeplay/py.typed +0 -0
- freeplay-0.2.42/src/freeplay/record.py +0 -113
- freeplay-0.2.42/src/freeplay/support.py +0 -381
- {freeplay-0.2.42 → freeplay-0.3.0a2}/LICENSE +0 -0
- {freeplay-0.2.42 → freeplay-0.3.0a2}/README.md +0 -0
- {freeplay-0.2.42 → freeplay-0.3.0a2}/src/freeplay/api_support.py +0 -0
- {freeplay-0.2.42 → freeplay-0.3.0a2}/src/freeplay/errors.py +0 -0
- {freeplay-0.2.42 → freeplay-0.3.0a2}/src/freeplay/llm_parameters.py +0 -0
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/__init__.py +0 -0
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/customer_feedback.py +0 -0
- {freeplay-0.2.42/src/freeplay/thin → freeplay-0.3.0a2/src/freeplay}/resources/sessions.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: freeplay
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.3.0a2
|
4
4
|
Summary:
|
5
5
|
License: MIT
|
6
6
|
Author: FreePlay Engineering
|
@@ -13,10 +13,8 @@ Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
15
15
|
Classifier: Programming Language :: Python :: 3.12
|
16
|
-
Requires-Dist: anthropic (>=0.20.0,<0.21.0)
|
17
16
|
Requires-Dist: click (==8.1.7)
|
18
17
|
Requires-Dist: dacite (>=1.8.0,<2.0.0)
|
19
|
-
Requires-Dist: openai (>=1,<2)
|
20
18
|
Requires-Dist: pystache (>=0.6.5,<0.7.0)
|
21
19
|
Requires-Dist: requests (>=2.20.0,<3.0.0dev)
|
22
20
|
Description-Content-Type: text/markdown
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "freeplay"
|
3
|
-
version = "0.2
|
3
|
+
version = "0.3.0-alpha.2"
|
4
4
|
description = ""
|
5
5
|
authors = ["FreePlay Engineering <engineering@freeplay.ai>"]
|
6
6
|
license = "MIT"
|
@@ -10,8 +10,6 @@ readme = "README.md"
|
|
10
10
|
python = ">=3.8, <4"
|
11
11
|
requests = ">=2.20.0,<3.0.0dev"
|
12
12
|
dacite = "^1.8.0"
|
13
|
-
anthropic = "^0.20.0"
|
14
|
-
openai = "^1"
|
15
13
|
click = "8.1.7"
|
16
14
|
pystache = "^0.6.5"
|
17
15
|
|
@@ -19,6 +17,7 @@ pystache = "^0.6.5"
|
|
19
17
|
mypy = "^1"
|
20
18
|
types-requests = "^2.31"
|
21
19
|
anthropic = { version="^0.20.0", extras = ["bedrock"] }
|
20
|
+
openai = "^1"
|
22
21
|
|
23
22
|
[tool.poetry.group.test.dependencies]
|
24
23
|
responses = "^0.23.1"
|
@@ -1,11 +1,11 @@
|
|
1
|
-
from .
|
1
|
+
from .freeplay import Freeplay
|
2
2
|
from .resources.prompts import PromptInfo
|
3
3
|
from .resources.recordings import CallInfo, ResponseInfo, RecordPayload, TestRunInfo
|
4
4
|
from .resources.sessions import SessionInfo
|
5
5
|
|
6
6
|
__all__ = [
|
7
|
-
'Freeplay',
|
8
7
|
'CallInfo',
|
8
|
+
'Freeplay',
|
9
9
|
'PromptInfo',
|
10
10
|
'RecordPayload',
|
11
11
|
'ResponseInfo',
|
freeplay-0.2.42/src/freeplay/thin/freeplay_thin.py → freeplay-0.3.0a2/src/freeplay/freeplay.py
RENAMED
@@ -1,13 +1,12 @@
|
|
1
1
|
from typing import Optional
|
2
2
|
|
3
3
|
from freeplay.errors import FreeplayConfigurationError
|
4
|
-
from freeplay.
|
4
|
+
from freeplay.resources.customer_feedback import CustomerFeedback
|
5
|
+
from freeplay.resources.prompts import Prompts, APITemplateResolver, TemplateResolver
|
6
|
+
from freeplay.resources.recordings import Recordings
|
7
|
+
from freeplay.resources.sessions import Sessions
|
8
|
+
from freeplay.resources.test_runs import TestRuns
|
5
9
|
from freeplay.support import CallSupport
|
6
|
-
from freeplay.thin.resources.customer_feedback import CustomerFeedback
|
7
|
-
from freeplay.thin.resources.prompts import Prompts, APITemplateResolver, TemplateResolver
|
8
|
-
from freeplay.thin.resources.recordings import Recordings
|
9
|
-
from freeplay.thin.resources.sessions import Sessions
|
10
|
-
from freeplay.thin.resources.test_runs import TestRuns
|
11
10
|
|
12
11
|
|
13
12
|
class Freeplay:
|
@@ -22,8 +21,7 @@ class Freeplay:
|
|
22
21
|
|
23
22
|
self.call_support = CallSupport(
|
24
23
|
freeplay_api_key,
|
25
|
-
api_base
|
26
|
-
DefaultRecordProcessor(freeplay_api_key, api_base)
|
24
|
+
api_base
|
27
25
|
)
|
28
26
|
self.freeplay_api_key = freeplay_api_key
|
29
27
|
self.api_base = api_base
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import dataclasses
|
2
1
|
import json
|
3
2
|
import os
|
4
3
|
import sys
|
@@ -7,9 +6,9 @@ from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWUSR
|
|
7
6
|
|
8
7
|
import click
|
9
8
|
|
10
|
-
from .
|
11
|
-
from
|
12
|
-
from .
|
9
|
+
from freeplay.errors import FreeplayClientError, FreeplayServerError
|
10
|
+
from freeplay import Freeplay
|
11
|
+
from freeplay.support import PromptTemplates, PromptTemplate, PromptTemplateEncoder
|
13
12
|
|
14
13
|
|
15
14
|
@click.group()
|
@@ -37,8 +36,9 @@ def download(project_id: str, environment: str, output_dir: str) -> None:
|
|
37
36
|
freeplay_api_url = f'{os.environ["FREEPLAY_API_URL"]}/api'
|
38
37
|
click.echo("Using URL override for Freeplay specified in the FREEPLAY_API_URL environment variable")
|
39
38
|
|
40
|
-
click.echo(
|
41
|
-
|
39
|
+
click.echo(
|
40
|
+
f"Downloading prompts for project {project_id}, environment {environment}, "
|
41
|
+
f"to directory {output_dir} from {freeplay_api_url}")
|
42
42
|
|
43
43
|
fp_client = Freeplay(
|
44
44
|
freeplay_api_key=FREEPLAY_API_KEY,
|
@@ -47,19 +47,19 @@ def download(project_id: str, environment: str, output_dir: str) -> None:
|
|
47
47
|
|
48
48
|
try:
|
49
49
|
prompts: PromptTemplates = fp_client.prompts.get_all(project_id, environment=environment)
|
50
|
-
click.echo("Found
|
50
|
+
click.echo(f"Found {len(prompts.prompt_templates)} prompt templates")
|
51
51
|
|
52
|
-
for prompt in prompts.
|
52
|
+
for prompt in prompts.prompt_templates:
|
53
53
|
__write_single_file(environment, output_dir, project_id, prompt)
|
54
54
|
except FreeplayClientError as e:
|
55
|
-
print("Error downloading templates:
|
55
|
+
print(f"Error downloading templates: {e}.\nIs your project ID correct?", file=sys.stderr)
|
56
56
|
exit(1)
|
57
57
|
except FreeplayServerError as e:
|
58
|
-
print("Error on Freeplay's servers downloading templates:
|
58
|
+
print(f"Error on Freeplay's servers downloading templates: {e}.\nTry again after a short wait.",
|
59
59
|
file=sys.stderr)
|
60
60
|
exit(2)
|
61
61
|
except Exception as e:
|
62
|
-
print("Error downloading templates:
|
62
|
+
print(f"Error downloading templates: {e}", file=sys.stderr)
|
63
63
|
exit(3)
|
64
64
|
|
65
65
|
|
@@ -67,33 +67,19 @@ def __write_single_file(
|
|
67
67
|
environment: str,
|
68
68
|
output_dir: str,
|
69
69
|
project_id: str,
|
70
|
-
prompt:
|
70
|
+
prompt: PromptTemplate
|
71
71
|
) -> None:
|
72
72
|
directory = __root_dir(environment, output_dir, project_id)
|
73
|
-
basename = f'{prompt.
|
73
|
+
basename = f'{prompt.prompt_template_name}'
|
74
74
|
prompt_path = directory / f'{basename}.json'
|
75
75
|
click.echo("Writing prompt file: %s" % prompt_path)
|
76
76
|
|
77
|
-
full_dict = dataclasses.asdict(prompt)
|
78
|
-
del full_dict['prompt_template_id']
|
79
|
-
del full_dict['prompt_template_version_id']
|
80
|
-
del full_dict['name']
|
81
|
-
del full_dict['content']
|
82
|
-
|
83
|
-
output_dict = {
|
84
|
-
'prompt_template_id': prompt.prompt_template_id,
|
85
|
-
'prompt_template_version_id': prompt.prompt_template_version_id,
|
86
|
-
'name': prompt.name,
|
87
|
-
'content': prompt.content,
|
88
|
-
'metadata': full_dict
|
89
|
-
}
|
90
|
-
|
91
77
|
# Make sure it's owner writable if it already exists
|
92
78
|
if prompt_path.is_file():
|
93
79
|
os.chmod(prompt_path, S_IWUSR | S_IREAD)
|
94
80
|
|
95
81
|
with prompt_path.open(mode='w') as f:
|
96
|
-
f.write(json.dumps(
|
82
|
+
f.write(json.dumps(prompt, sort_keys=True, indent=4, cls=PromptTemplateEncoder))
|
97
83
|
f.write('\n')
|
98
84
|
|
99
85
|
# Make the file read-only to discourage local changes
|
@@ -0,0 +1,17 @@
|
|
1
|
+
from dataclasses import dataclass
|
2
|
+
from typing import List, Union, Any, Dict, Mapping, TypedDict
|
3
|
+
|
4
|
+
InputValue = Union[str, int, bool, Dict[str, Any], List[Any]]
|
5
|
+
InputVariables = Mapping[str, InputValue]
|
6
|
+
TestRunInput = Mapping[str, InputValue]
|
7
|
+
|
8
|
+
|
9
|
+
@dataclass
|
10
|
+
class TestRun:
|
11
|
+
id: str
|
12
|
+
inputs: List[TestRunInput]
|
13
|
+
|
14
|
+
|
15
|
+
class OpenAIFunctionCall(TypedDict):
|
16
|
+
name: str
|
17
|
+
arguments: str
|
@@ -4,15 +4,22 @@ from dataclasses import dataclass
|
|
4
4
|
from pathlib import Path
|
5
5
|
from typing import Dict, Optional, List, Union, cast, Any
|
6
6
|
|
7
|
-
from freeplay.completions import PromptTemplates, ChatMessage, PromptTemplateWithMetadata
|
8
7
|
from freeplay.errors import FreeplayConfigurationError, FreeplayClientError
|
9
|
-
from freeplay.flavors import Flavor
|
10
8
|
from freeplay.llm_parameters import LLMParameters
|
11
9
|
from freeplay.model import InputVariables
|
12
|
-
from freeplay.support import
|
10
|
+
from freeplay.support import PromptTemplate, PromptTemplates, PromptTemplateMetadata
|
11
|
+
from freeplay.support import CallSupport
|
13
12
|
from freeplay.utils import bind_template_variables
|
14
13
|
|
15
14
|
|
15
|
+
class MissingFlavorError(FreeplayConfigurationError):
|
16
|
+
def __init__(self, flavor_name: str):
|
17
|
+
super().__init__(
|
18
|
+
f'Configured flavor ({flavor_name}) not found in SDK. Please update your SDK version or configure '
|
19
|
+
'a different model in the Freeplay UI.'
|
20
|
+
)
|
21
|
+
|
22
|
+
|
16
23
|
# SDK-Exposed Classes
|
17
24
|
@dataclass
|
18
25
|
class PromptInfo:
|
@@ -54,18 +61,40 @@ class BoundPrompt:
|
|
54
61
|
self.prompt_info = prompt_info
|
55
62
|
self.messages = messages
|
56
63
|
|
64
|
+
@staticmethod
|
65
|
+
def __to_anthropic_role(role: str) -> str:
|
66
|
+
if role == 'assistant' or role == 'Assistant':
|
67
|
+
return 'Assistant'
|
68
|
+
else:
|
69
|
+
# Anthropic does not support system role for now.
|
70
|
+
return 'Human'
|
71
|
+
|
72
|
+
@staticmethod
|
73
|
+
def __format_messages_for_flavor(flavor_name: str, messages: List[Dict[str, str]]) -> Union[
|
74
|
+
str, List[Dict[str, str]]]:
|
75
|
+
if flavor_name == 'azure_openai_chat' or flavor_name == 'openai_chat':
|
76
|
+
return messages
|
77
|
+
elif flavor_name == 'anthropic_chat':
|
78
|
+
formatted_messages = []
|
79
|
+
for message in messages:
|
80
|
+
role = BoundPrompt.__to_anthropic_role(message['role'])
|
81
|
+
formatted_messages.append(f"{role}: {message['content']}")
|
82
|
+
formatted_messages.append('Assistant:')
|
83
|
+
|
84
|
+
return "\n\n" + "\n\n".join(formatted_messages)
|
85
|
+
raise MissingFlavorError(flavor_name)
|
86
|
+
|
57
87
|
def format(
|
58
88
|
self,
|
59
89
|
flavor_name: Optional[str] = None
|
60
90
|
) -> FormattedPrompt:
|
61
91
|
final_flavor = flavor_name or self.prompt_info.flavor_name
|
62
|
-
|
63
|
-
llm_format = flavor.to_llm_syntax(cast(List[ChatMessage], self.messages))
|
92
|
+
llm_format = BoundPrompt.__format_messages_for_flavor(final_flavor, self.messages)
|
64
93
|
|
65
94
|
return FormattedPrompt(
|
66
95
|
self.prompt_info,
|
67
96
|
self.messages,
|
68
|
-
|
97
|
+
llm_format,
|
69
98
|
)
|
70
99
|
|
71
100
|
|
@@ -120,15 +149,7 @@ class FilesystemTemplateResolver(TemplateResolver):
|
|
120
149
|
prompt_list = []
|
121
150
|
for prompt_file_path in prompt_file_paths:
|
122
151
|
json_dom = json.loads(prompt_file_path.read_text())
|
123
|
-
|
124
|
-
prompt_list.append(PromptTemplateWithMetadata(
|
125
|
-
prompt_template_id=json_dom.get('prompt_template_id'),
|
126
|
-
prompt_template_version_id=json_dom.get('prompt_template_version_id'),
|
127
|
-
name=json_dom.get('name'),
|
128
|
-
content=json_dom.get('content'),
|
129
|
-
flavor_name=json_dom.get('metadata').get('flavor_name'),
|
130
|
-
params=json_dom.get('metadata').get('params')
|
131
|
-
))
|
152
|
+
prompt_list.append(self.__render_into_v2(json_dom))
|
132
153
|
|
133
154
|
return PromptTemplates(prompt_list)
|
134
155
|
|
@@ -144,38 +165,59 @@ class FilesystemTemplateResolver(TemplateResolver):
|
|
144
165
|
)
|
145
166
|
|
146
167
|
json_dom = json.loads(expected_file.read_text())
|
168
|
+
return self.__render_into_v2(json_dom)
|
147
169
|
|
170
|
+
@staticmethod
|
171
|
+
def __render_into_v2(json_dom: Dict[str, Any]) -> PromptTemplate:
|
148
172
|
format_version = json_dom.get('format_version')
|
149
173
|
|
150
174
|
if format_version == 2:
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
175
|
+
metadata = json_dom['metadata']
|
176
|
+
flavor_name = metadata.get('flavor')
|
177
|
+
model = metadata.get('model')
|
178
|
+
|
179
|
+
return PromptTemplate(
|
180
|
+
format_version=2,
|
181
|
+
prompt_template_id=json_dom.get('prompt_template_id'), # type: ignore
|
182
|
+
prompt_template_version_id=json_dom.get('prompt_template_version_id'), # type: ignore
|
183
|
+
prompt_template_name=json_dom.get('prompt_template_name'), # type: ignore
|
184
|
+
content=FilesystemTemplateResolver.__normalize_roles(json_dom['content']),
|
185
|
+
metadata=PromptTemplateMetadata(
|
186
|
+
provider=FilesystemTemplateResolver.__flavor_to_provider(flavor_name),
|
187
|
+
flavor=flavor_name,
|
188
|
+
model=model,
|
189
|
+
params=metadata.get('params'),
|
190
|
+
provider_info=metadata.get('provider_info')
|
191
|
+
)
|
192
|
+
)
|
193
|
+
else:
|
194
|
+
metadata = json_dom['metadata']
|
195
|
+
|
196
|
+
flavor_name = metadata.get('flavor_name')
|
197
|
+
params = metadata.get('params')
|
198
|
+
model = params.pop('model') if 'model' in params else None
|
199
|
+
|
200
|
+
return PromptTemplate(
|
201
|
+
format_version=2,
|
202
|
+
prompt_template_id=json_dom.get('prompt_template_id'), # type: ignore
|
203
|
+
prompt_template_version_id=json_dom.get('prompt_template_version_id'), # type: ignore
|
204
|
+
prompt_template_name=json_dom.get('name'), # type: ignore
|
205
|
+
content=FilesystemTemplateResolver.__normalize_roles(json.loads(str(json_dom['content']))),
|
206
|
+
metadata=PromptTemplateMetadata(
|
207
|
+
provider=FilesystemTemplateResolver.__flavor_to_provider(flavor_name),
|
208
|
+
flavor=flavor_name,
|
209
|
+
model=model,
|
210
|
+
params=params,
|
211
|
+
provider_info=None
|
212
|
+
)
|
170
213
|
)
|
171
|
-
)
|
172
214
|
|
173
215
|
@staticmethod
|
174
|
-
def __normalize_roles(messages: List[
|
216
|
+
def __normalize_roles(messages: List[Dict[str, str]]) -> List[Dict[str, str]]:
|
175
217
|
normalized = []
|
176
218
|
for message in messages:
|
177
219
|
role = FilesystemTemplateResolver.__role_translations.get(message['role']) or message['role']
|
178
|
-
normalized.append(
|
220
|
+
normalized.append({'role': role, 'content': message['content']})
|
179
221
|
return normalized
|
180
222
|
|
181
223
|
@staticmethod
|
@@ -200,6 +242,18 @@ class FilesystemTemplateResolver(TemplateResolver):
|
|
200
242
|
(project_id, environment)
|
201
243
|
)
|
202
244
|
|
245
|
+
@staticmethod
|
246
|
+
def __flavor_to_provider(flavor: str) -> str:
|
247
|
+
flavor_provider = {
|
248
|
+
'azure_openai_chat': 'azure',
|
249
|
+
'anthropic_chat': 'anthropic',
|
250
|
+
'openai_chat': 'openai',
|
251
|
+
}
|
252
|
+
provider = flavor_provider.get(flavor)
|
253
|
+
if not provider:
|
254
|
+
raise MissingFlavorError(flavor)
|
255
|
+
return provider
|
256
|
+
|
203
257
|
|
204
258
|
class APITemplateResolver(TemplateResolver):
|
205
259
|
|
@@ -209,7 +263,7 @@ class APITemplateResolver(TemplateResolver):
|
|
209
263
|
def get_prompts(self, project_id: str, environment: str) -> PromptTemplates:
|
210
264
|
return self.call_support.get_prompts(
|
211
265
|
project_id=project_id,
|
212
|
-
|
266
|
+
environment=environment
|
213
267
|
)
|
214
268
|
|
215
269
|
def get_prompt(self, project_id: str, template_name: str, environment: str) -> PromptTemplate:
|
@@ -226,7 +280,7 @@ class Prompts:
|
|
226
280
|
self.template_resolver = template_resolver
|
227
281
|
|
228
282
|
def get_all(self, project_id: str, environment: str) -> PromptTemplates:
|
229
|
-
return self.call_support.get_prompts(project_id=project_id,
|
283
|
+
return self.call_support.get_prompts(project_id=project_id, environment=environment)
|
230
284
|
|
231
285
|
def get(self, project_id: str, template_name: str, environment: str) -> TemplatePrompt:
|
232
286
|
prompt = self.template_resolver.get_prompt(project_id, template_name, environment)
|
@@ -242,7 +296,9 @@ class Prompts:
|
|
242
296
|
raise FreeplayConfigurationError(
|
243
297
|
"Flavor must be configured in the Freeplay UI. Unable to fulfill request.")
|
244
298
|
|
245
|
-
|
299
|
+
if not prompt.metadata.provider:
|
300
|
+
raise FreeplayConfigurationError(
|
301
|
+
"Provider must be configured in the Freeplay UI. Unable to fulfill request.")
|
246
302
|
|
247
303
|
prompt_info = PromptInfo(
|
248
304
|
prompt_template_id=prompt.prompt_template_id,
|
@@ -250,7 +306,7 @@ class Prompts:
|
|
250
306
|
template_name=prompt.prompt_template_name,
|
251
307
|
environment=environment,
|
252
308
|
model_parameters=cast(LLMParameters, params) or LLMParameters({}),
|
253
|
-
provider=
|
309
|
+
provider=prompt.metadata.provider,
|
254
310
|
model=model,
|
255
311
|
flavor_name=prompt.metadata.flavor,
|
256
312
|
provider_info=prompt.metadata.provider_info
|
@@ -6,13 +6,12 @@ from typing import Dict, Optional, List
|
|
6
6
|
from requests import HTTPError
|
7
7
|
|
8
8
|
from freeplay import api_support
|
9
|
-
from freeplay.completions import PromptTemplateWithMetadata, OpenAIFunctionCall
|
10
9
|
from freeplay.errors import FreeplayClientError, FreeplayError
|
11
10
|
from freeplay.llm_parameters import LLMParameters
|
12
|
-
from freeplay.model import InputVariables
|
11
|
+
from freeplay.model import InputVariables, OpenAIFunctionCall
|
12
|
+
from freeplay.resources.prompts import PromptInfo
|
13
|
+
from freeplay.resources.sessions import SessionInfo
|
13
14
|
from freeplay.support import CallSupport
|
14
|
-
from freeplay.thin.resources.prompts import PromptInfo
|
15
|
-
from freeplay.thin.resources.sessions import SessionInfo
|
16
15
|
|
17
16
|
logger = logging.getLogger(__name__)
|
18
17
|
|
@@ -79,19 +78,10 @@ class Recordings:
|
|
79
78
|
completion = record_payload.all_messages[-1]
|
80
79
|
history_as_string = json.dumps(record_payload.all_messages[0:-1])
|
81
80
|
|
82
|
-
template = PromptTemplateWithMetadata(
|
83
|
-
prompt_template_id=record_payload.prompt_info.prompt_template_id,
|
84
|
-
prompt_template_version_id=record_payload.prompt_info.prompt_template_version_id,
|
85
|
-
name=record_payload.prompt_info.template_name,
|
86
|
-
content=history_as_string,
|
87
|
-
flavor_name=record_payload.prompt_info.flavor_name,
|
88
|
-
params=record_payload.prompt_info.model_parameters
|
89
|
-
)
|
90
|
-
|
91
81
|
record_api_payload = {
|
92
82
|
"session_id": record_payload.session_info.session_id,
|
93
|
-
"
|
94
|
-
"
|
83
|
+
"prompt_template_id": record_payload.prompt_info.prompt_template_id,
|
84
|
+
"project_version_id": record_payload.prompt_info.prompt_template_version_id,
|
95
85
|
"start_time": record_payload.call_info.start_time,
|
96
86
|
"end_time": record_payload.call_info.end_time,
|
97
87
|
"tag": record_payload.prompt_info.environment,
|
@@ -2,8 +2,8 @@ from dataclasses import dataclass
|
|
2
2
|
from typing import List, Optional
|
3
3
|
|
4
4
|
from freeplay.model import InputVariables
|
5
|
+
from freeplay.resources.recordings import TestRunInfo
|
5
6
|
from freeplay.support import CallSupport
|
6
|
-
from freeplay.thin.resources.recordings import TestRunInfo
|
7
7
|
|
8
8
|
|
9
9
|
@dataclass
|
@@ -0,0 +1,142 @@
|
|
1
|
+
from dataclasses import dataclass
|
2
|
+
from json import JSONEncoder
|
3
|
+
from typing import Optional, Dict, Any, List, Union
|
4
|
+
|
5
|
+
from freeplay import api_support
|
6
|
+
from freeplay.api_support import try_decode
|
7
|
+
from freeplay.errors import freeplay_response_error, FreeplayServerError
|
8
|
+
from freeplay.model import InputVariables
|
9
|
+
|
10
|
+
|
11
|
+
@dataclass
|
12
|
+
class PromptTemplateMetadata:
|
13
|
+
provider: Optional[str]
|
14
|
+
flavor: Optional[str]
|
15
|
+
model: Optional[str]
|
16
|
+
params: Optional[Dict[str, Any]] = None
|
17
|
+
provider_info: Optional[Dict[str, Any]] = None
|
18
|
+
|
19
|
+
|
20
|
+
@dataclass
|
21
|
+
class PromptTemplate:
|
22
|
+
prompt_template_id: str
|
23
|
+
prompt_template_version_id: str
|
24
|
+
prompt_template_name: str
|
25
|
+
content: List[Dict[str, str]]
|
26
|
+
metadata: PromptTemplateMetadata
|
27
|
+
format_version: int
|
28
|
+
|
29
|
+
|
30
|
+
@dataclass
|
31
|
+
class PromptTemplates:
|
32
|
+
prompt_templates: List[PromptTemplate]
|
33
|
+
|
34
|
+
|
35
|
+
class PromptTemplateEncoder(JSONEncoder):
|
36
|
+
def default(self, prompt_template: PromptTemplate) -> Dict[str, Any]:
|
37
|
+
return prompt_template.__dict__
|
38
|
+
|
39
|
+
|
40
|
+
class TestCaseTestRunResponse:
|
41
|
+
def __init__(self, test_case: Dict[str, Any]):
|
42
|
+
self.variables: InputVariables = test_case['variables']
|
43
|
+
self.id: str = test_case['id']
|
44
|
+
self.output: Optional[str] = test_case.get('output')
|
45
|
+
|
46
|
+
|
47
|
+
class TestRunResponse:
|
48
|
+
def __init__(
|
49
|
+
self,
|
50
|
+
test_run_id: str,
|
51
|
+
test_cases: List[Dict[str, Any]]
|
52
|
+
):
|
53
|
+
self.test_cases = [
|
54
|
+
TestCaseTestRunResponse(test_case)
|
55
|
+
for test_case in test_cases
|
56
|
+
]
|
57
|
+
self.test_run_id = test_run_id
|
58
|
+
|
59
|
+
|
60
|
+
class CallSupport:
|
61
|
+
def __init__(
|
62
|
+
self,
|
63
|
+
freeplay_api_key: str,
|
64
|
+
api_base: str
|
65
|
+
) -> None:
|
66
|
+
self.api_base = api_base
|
67
|
+
self.freeplay_api_key = freeplay_api_key
|
68
|
+
|
69
|
+
def get_prompts(self, project_id: str, environment: str) -> PromptTemplates:
|
70
|
+
response = api_support.get_raw(
|
71
|
+
api_key=self.freeplay_api_key,
|
72
|
+
url=f'{self.api_base}/v2/projects/{project_id}/prompt-templates/all/{environment}'
|
73
|
+
)
|
74
|
+
|
75
|
+
if response.status_code != 200:
|
76
|
+
raise freeplay_response_error("Error getting prompt templates", response)
|
77
|
+
|
78
|
+
maybe_prompts = try_decode(PromptTemplates, response.content)
|
79
|
+
if maybe_prompts is None:
|
80
|
+
raise FreeplayServerError('Failed to parse prompt templates from server')
|
81
|
+
|
82
|
+
return maybe_prompts
|
83
|
+
|
84
|
+
def get_prompt(self, project_id: str, template_name: str, environment: str) -> PromptTemplate:
|
85
|
+
response = api_support.get_raw(
|
86
|
+
api_key=self.freeplay_api_key,
|
87
|
+
url=f'{self.api_base}/v2/projects/{project_id}/prompt-templates/name/{template_name}',
|
88
|
+
params={
|
89
|
+
'environment': environment
|
90
|
+
}
|
91
|
+
)
|
92
|
+
|
93
|
+
if response.status_code != 200:
|
94
|
+
raise freeplay_response_error(
|
95
|
+
f"Error getting prompt template {template_name} in project {project_id} "
|
96
|
+
f"and environment {environment}",
|
97
|
+
response
|
98
|
+
)
|
99
|
+
|
100
|
+
maybe_prompt = try_decode(PromptTemplate, response.content)
|
101
|
+
if maybe_prompt is None:
|
102
|
+
raise FreeplayServerError(
|
103
|
+
f"Error handling prompt {template_name} in project {project_id} "
|
104
|
+
f"and environment {environment}"
|
105
|
+
)
|
106
|
+
|
107
|
+
return maybe_prompt
|
108
|
+
|
109
|
+
def update_customer_feedback(
|
110
|
+
self,
|
111
|
+
completion_id: str,
|
112
|
+
feedback: Dict[str, Union[bool, str, int, float]]
|
113
|
+
) -> None:
|
114
|
+
response = api_support.put_raw(
|
115
|
+
self.freeplay_api_key,
|
116
|
+
f'{self.api_base}/v1/completion_feedback/{completion_id}',
|
117
|
+
feedback
|
118
|
+
)
|
119
|
+
if response.status_code != 201:
|
120
|
+
raise freeplay_response_error("Error updating customer feedback", response)
|
121
|
+
|
122
|
+
def create_test_run(
|
123
|
+
self,
|
124
|
+
project_id: str,
|
125
|
+
testlist: str,
|
126
|
+
include_test_case_outputs: bool = False
|
127
|
+
) -> TestRunResponse:
|
128
|
+
response = api_support.post_raw(
|
129
|
+
api_key=self.freeplay_api_key,
|
130
|
+
url=f'{self.api_base}/projects/{project_id}/test-runs-cases',
|
131
|
+
payload={
|
132
|
+
'testlist_name': testlist,
|
133
|
+
'include_test_case_outputs': include_test_case_outputs,
|
134
|
+
},
|
135
|
+
)
|
136
|
+
|
137
|
+
if response.status_code != 201:
|
138
|
+
raise freeplay_response_error('Error while creating a test run.', response)
|
139
|
+
|
140
|
+
json_dom = response.json()
|
141
|
+
|
142
|
+
return TestRunResponse(json_dom['test_run_id'], json_dom['test_cases'])
|
@@ -1,19 +1,27 @@
|
|
1
|
-
from typing import Dict, Union, Optional
|
1
|
+
from typing import Dict, Union, Optional, Any
|
2
2
|
import importlib.metadata
|
3
3
|
import platform
|
4
4
|
|
5
5
|
import pystache # type: ignore
|
6
|
-
from pydantic import ValidationError
|
7
6
|
|
8
7
|
from .errors import FreeplayError, FreeplayConfigurationError
|
9
|
-
from .model import
|
8
|
+
from .model import InputVariables
|
9
|
+
|
10
|
+
|
11
|
+
# Validate that the variables are of the correct type, and do not include functions, dates, classes or None values.
|
12
|
+
def all_valid(obj: Any) -> bool:
|
13
|
+
if isinstance(obj, (int, str, bool)):
|
14
|
+
return True
|
15
|
+
elif isinstance(obj, list):
|
16
|
+
return all(all_valid(item) for item in obj)
|
17
|
+
elif isinstance(obj, dict):
|
18
|
+
return all(isinstance(key, str) and all_valid(value) for key, value in obj.items())
|
19
|
+
else:
|
20
|
+
return False
|
10
21
|
|
11
22
|
|
12
23
|
def bind_template_variables(template: str, variables: InputVariables) -> str:
|
13
|
-
|
14
|
-
try:
|
15
|
-
PydanticInputVariables.model_validate(variables)
|
16
|
-
except ValidationError as err:
|
24
|
+
if not all_valid(variables):
|
17
25
|
raise FreeplayError(
|
18
26
|
'Variables must be a string, number, bool, or a possibly nested'
|
19
27
|
' list or dict of strings, numbers and booleans.'
|