sapiopycommons 2025.10.1a772__py3-none-any.whl → 2025.10.9a776__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sapiopycommons might be problematic. Click here for more details.
- sapiopycommons/ai/agent_service_base.py +1114 -0
- sapiopycommons/ai/converter_service_base.py +163 -0
- sapiopycommons/ai/protoapi/externalcredentials/external_credentials_pb2.py +41 -0
- sapiopycommons/ai/protoapi/externalcredentials/external_credentials_pb2.pyi +35 -0
- sapiopycommons/ai/protoapi/externalcredentials/external_credentials_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/fielddefinitions/fields_pb2.py +43 -0
- sapiopycommons/ai/protoapi/fielddefinitions/fields_pb2.pyi +31 -0
- sapiopycommons/ai/protoapi/fielddefinitions/fields_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/fielddefinitions/velox_field_def_pb2.py +123 -0
- sapiopycommons/ai/protoapi/fielddefinitions/velox_field_def_pb2.pyi +598 -0
- sapiopycommons/ai/protoapi/fielddefinitions/velox_field_def_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/plan/converter/converter_pb2.py +51 -0
- sapiopycommons/ai/protoapi/plan/converter/converter_pb2.pyi +63 -0
- sapiopycommons/ai/protoapi/plan/converter/converter_pb2_grpc.py +149 -0
- sapiopycommons/ai/protoapi/plan/item/item_container_pb2.py +55 -0
- sapiopycommons/ai/protoapi/plan/item/item_container_pb2.pyi +90 -0
- sapiopycommons/ai/protoapi/plan/item/item_container_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/plan/script/script_pb2.py +61 -0
- sapiopycommons/ai/protoapi/plan/script/script_pb2.pyi +108 -0
- sapiopycommons/ai/protoapi/plan/script/script_pb2_grpc.py +153 -0
- sapiopycommons/ai/protoapi/plan/step_output_pb2.py +45 -0
- sapiopycommons/ai/protoapi/plan/step_output_pb2.pyi +42 -0
- sapiopycommons/ai/protoapi/plan/step_output_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/plan/step_pb2.py +43 -0
- sapiopycommons/ai/protoapi/plan/step_pb2.pyi +43 -0
- sapiopycommons/ai/protoapi/plan/step_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/plan/tool/entry_pb2.py +41 -0
- sapiopycommons/ai/protoapi/plan/tool/entry_pb2.pyi +35 -0
- sapiopycommons/ai/protoapi/plan/tool/entry_pb2_grpc.py +24 -0
- sapiopycommons/ai/protoapi/plan/tool/tool_pb2.py +79 -0
- sapiopycommons/ai/protoapi/plan/tool/tool_pb2.pyi +261 -0
- sapiopycommons/ai/protoapi/plan/tool/tool_pb2_grpc.py +154 -0
- sapiopycommons/ai/protoapi/session/sapio_conn_info_pb2.py +39 -0
- sapiopycommons/ai/protoapi/session/sapio_conn_info_pb2.pyi +32 -0
- sapiopycommons/ai/protoapi/session/sapio_conn_info_pb2_grpc.py +24 -0
- sapiopycommons/ai/protobuf_utils.py +504 -0
- sapiopycommons/ai/request_validation.py +470 -0
- sapiopycommons/ai/server.py +152 -0
- sapiopycommons/ai/test_client.py +370 -0
- sapiopycommons/files/file_util.py +128 -1
- sapiopycommons/files/temp_files.py +82 -0
- sapiopycommons/webhook/webservice_handlers.py +1 -1
- {sapiopycommons-2025.10.1a772.dist-info → sapiopycommons-2025.10.9a776.dist-info}/METADATA +1 -1
- {sapiopycommons-2025.10.1a772.dist-info → sapiopycommons-2025.10.9a776.dist-info}/RECORD +46 -7
- sapiopycommons/ai/tool_of_tools.py +0 -917
- {sapiopycommons-2025.10.1a772.dist-info → sapiopycommons-2025.10.9a776.dist-info}/WHEEL +0 -0
- {sapiopycommons-2025.10.1a772.dist-info → sapiopycommons-2025.10.9a776.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import json
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import grpc
|
|
7
|
+
from sapiopylib.rest.User import SapioUser
|
|
8
|
+
|
|
9
|
+
from sapiopycommons.ai.protoapi.fielddefinitions.fields_pb2 import FieldValuePbo
|
|
10
|
+
from sapiopycommons.ai.protoapi.plan.converter.converter_pb2 import ConverterDetailsRequestPbo, \
|
|
11
|
+
ConverterDetailsResponsePbo, ConvertResponsePbo, ConvertRequestPbo
|
|
12
|
+
from sapiopycommons.ai.protoapi.plan.converter.converter_pb2_grpc import ConverterServiceStub
|
|
13
|
+
from sapiopycommons.ai.protoapi.plan.item.item_container_pb2 import ContentTypePbo
|
|
14
|
+
from sapiopycommons.ai.protoapi.plan.tool.entry_pb2 import StepBinaryContainerPbo, StepCsvRowPbo, \
|
|
15
|
+
StepCsvHeaderRowPbo, StepCsvContainerPbo, StepJsonContainerPbo, StepTextContainerPbo, \
|
|
16
|
+
StepItemContainerPbo, StepInputBatchPbo
|
|
17
|
+
from sapiopycommons.ai.protoapi.plan.tool.tool_pb2 import ProcessStepResponsePbo, ProcessStepRequestPbo, \
|
|
18
|
+
ToolDetailsRequestPbo, ToolDetailsResponsePbo, ProcessStepResponseStatusPbo
|
|
19
|
+
from sapiopycommons.ai.protoapi.plan.tool.tool_pb2_grpc import ToolServiceStub
|
|
20
|
+
from sapiopycommons.ai.protoapi.session.sapio_conn_info_pb2 import SapioConnectionInfoPbo, SapioUserSecretTypePbo
|
|
21
|
+
from sapiopycommons.ai.protobuf_utils import ProtobufUtils
|
|
22
|
+
from sapiopycommons.general.aliases import FieldValue
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ContainerType(Enum):
|
|
26
|
+
"""
|
|
27
|
+
An enum of the different container contents of a StepItemContainerPbo.
|
|
28
|
+
"""
|
|
29
|
+
BINARY = "binary"
|
|
30
|
+
CSV = "csv"
|
|
31
|
+
JSON = "json"
|
|
32
|
+
TEXT = "text"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# FR-47422: Created class.
|
|
36
|
+
class AgentOutput:
|
|
37
|
+
"""
|
|
38
|
+
A class for holding the output of a TestClient that calls an AgentService. AgentOutput objects an be
|
|
39
|
+
printed to show the output of the agent in a human-readable format.
|
|
40
|
+
"""
|
|
41
|
+
agent_name: str
|
|
42
|
+
|
|
43
|
+
status: str
|
|
44
|
+
message: str
|
|
45
|
+
|
|
46
|
+
binary_output: list[list[bytes]]
|
|
47
|
+
csv_output: list[list[dict[str, Any]]]
|
|
48
|
+
json_output: list[list[dict[str, Any]]]
|
|
49
|
+
text_output: list[list[str]]
|
|
50
|
+
|
|
51
|
+
new_records: list[dict[str, FieldValue]]
|
|
52
|
+
|
|
53
|
+
logs: list[str]
|
|
54
|
+
|
|
55
|
+
def __init__(self, agent_name: str):
|
|
56
|
+
self.agent_name = agent_name
|
|
57
|
+
self.binary_output = []
|
|
58
|
+
self.csv_output = []
|
|
59
|
+
self.json_output = []
|
|
60
|
+
self.text_output = []
|
|
61
|
+
self.new_records = []
|
|
62
|
+
self.logs = []
|
|
63
|
+
|
|
64
|
+
def __bool__(self):
|
|
65
|
+
return self.status == "Success"
|
|
66
|
+
|
|
67
|
+
def __str__(self):
|
|
68
|
+
ret_val: str = f"{self.agent_name} Output:\n"
|
|
69
|
+
ret_val += f"\tStatus: {self.status}\n"
|
|
70
|
+
ret_val += f"\tMessage: {self.message}\n"
|
|
71
|
+
ret_val += "-" * 25 + "\n"
|
|
72
|
+
|
|
73
|
+
if self.status == "Success":
|
|
74
|
+
ret_val += f"Binary Output: {sum(len(x) for x in self.binary_output)} item(s) across {len(self.binary_output)} output(s)\n"
|
|
75
|
+
for i, output in enumerate(self.binary_output, start=1):
|
|
76
|
+
output: list[bytes]
|
|
77
|
+
ret_val += f"\tBinary Output {i}:\n"
|
|
78
|
+
for binary in output:
|
|
79
|
+
ret_val += f"\t\t{len(binary)} byte(s): {binary[:50]}...\n"
|
|
80
|
+
|
|
81
|
+
ret_val += f"CSV Output: {sum(len(x) for x in self.csv_output)} item(s) across {len(self.csv_output)} output(s)\n"
|
|
82
|
+
for i, output in enumerate(self.csv_output, start=1):
|
|
83
|
+
output: list[dict[str, Any]]
|
|
84
|
+
ret_val += f"\tCSV Output {i}:\n"
|
|
85
|
+
ret_val += f"\t\tHeaders: {', '.join(output[0].keys())}\n"
|
|
86
|
+
for j, csv_row in enumerate(output):
|
|
87
|
+
ret_val += f"\t\t{j}: {', '.join(f'{v}' for k, v in csv_row.items())}\n"
|
|
88
|
+
|
|
89
|
+
ret_val += f"JSON Output: {sum(len(x) for x in self.json_output)} item(s) across {len(self.json_output)} output(s)\n"
|
|
90
|
+
for i, output in enumerate(self.json_output, start=1):
|
|
91
|
+
output: list[Any]
|
|
92
|
+
ret_val += f"\tJSON Output {i}:\n"
|
|
93
|
+
for json_obj in output:
|
|
94
|
+
ret_val += f"\t\t"
|
|
95
|
+
ret_val += json.dumps(json_obj, indent=2).replace("\n", "\n\t\t") + "\n"
|
|
96
|
+
|
|
97
|
+
ret_val += f"Text Output: {sum(len(x) for x in self.text_output)} item(s) across {len(self.text_output)} output(s)\n"
|
|
98
|
+
for i, output in enumerate(self.text_output, start=1):
|
|
99
|
+
output: list[str]
|
|
100
|
+
ret_val += f"\tText Output {i}:\n"
|
|
101
|
+
for text in output:
|
|
102
|
+
ret_val += f"\t\t{text}\n"
|
|
103
|
+
|
|
104
|
+
ret_val += f"New Records: {len(self.new_records)} item(s)\n"
|
|
105
|
+
for record in self.new_records:
|
|
106
|
+
ret_val += f"{json.dumps(record, indent=2)}\n"
|
|
107
|
+
|
|
108
|
+
ret_val += f"Logs: {len(self.logs)} item(s)\n"
|
|
109
|
+
for log in self.logs:
|
|
110
|
+
ret_val += f"\t{log}\n"
|
|
111
|
+
return ret_val
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class TestClient:
|
|
115
|
+
"""
|
|
116
|
+
A client for testing an AgentService.
|
|
117
|
+
"""
|
|
118
|
+
grpc_server_url: str
|
|
119
|
+
options: list[tuple[str, Any]] | None
|
|
120
|
+
connection: SapioConnectionInfoPbo
|
|
121
|
+
_request_inputs: list[StepItemContainerPbo]
|
|
122
|
+
_config_fields: dict[str, FieldValuePbo]
|
|
123
|
+
|
|
124
|
+
def __init__(self, grpc_server_url: str, message_mb_size: int = 1024, user: SapioUser | None = None,
|
|
125
|
+
options: list[tuple[str, Any]] | None = None):
|
|
126
|
+
"""
|
|
127
|
+
:param grpc_server_url: The URL of the gRPC server to connect to.
|
|
128
|
+
:param message_mb_size: The maximum size of a sent or received message in megabytes.
|
|
129
|
+
:param user: Optional SapioUser object to use for the connection. If not provided, a default connection
|
|
130
|
+
will be created with test credentials.
|
|
131
|
+
:param options: Optional list of gRPC channel options.
|
|
132
|
+
"""
|
|
133
|
+
self.grpc_server_url = grpc_server_url
|
|
134
|
+
self.options = [
|
|
135
|
+
('grpc.max_send_message_length', message_mb_size * 1024 * 1024),
|
|
136
|
+
('grpc.max_receive_message_length', message_mb_size * 1024 * 1024)
|
|
137
|
+
]
|
|
138
|
+
if options:
|
|
139
|
+
self.options.extend(options)
|
|
140
|
+
self._create_connection(user)
|
|
141
|
+
self._request_inputs = []
|
|
142
|
+
self._config_fields = {}
|
|
143
|
+
|
|
144
|
+
def _create_connection(self, user: SapioUser | None = None):
|
|
145
|
+
"""
|
|
146
|
+
Create a SapioConnectionInfoPbo object with test credentials. This method can be overridden to
|
|
147
|
+
create a user with specific credentials for testing.
|
|
148
|
+
"""
|
|
149
|
+
self.connection = SapioConnectionInfoPbo()
|
|
150
|
+
self.connection.username = user.username if user else "Testing"
|
|
151
|
+
self.connection.webservice_url = user.url if user else "https://localhost:8080/webservice/api"
|
|
152
|
+
self.connection.app_guid = user.guid if user else "1234567890"
|
|
153
|
+
self.connection.rmi_host.append("Testing")
|
|
154
|
+
self.connection.rmi_port = 9001
|
|
155
|
+
if user and user.password:
|
|
156
|
+
self.connection.secret_type = SapioUserSecretTypePbo.PASSWORD
|
|
157
|
+
self.connection.secret = "Basic " + base64.b64encode(f'{user.username}:{user.password}'.encode()).decode()
|
|
158
|
+
else:
|
|
159
|
+
self.connection.secret_type = SapioUserSecretTypePbo.SESSION_TOKEN
|
|
160
|
+
self.connection.secret = user.api_token if user and user.api_token else "test_api_token"
|
|
161
|
+
|
|
162
|
+
def add_binary_input(self, input_data: list[bytes]) -> None:
|
|
163
|
+
"""
|
|
164
|
+
Add a binary input to the the next request.
|
|
165
|
+
"""
|
|
166
|
+
self._add_input(ContainerType.BINARY, StepBinaryContainerPbo(items=input_data))
|
|
167
|
+
|
|
168
|
+
def add_csv_input(self, input_data: list[dict[str, Any]]) -> None:
|
|
169
|
+
"""
|
|
170
|
+
Add a CSV input to the next request.
|
|
171
|
+
"""
|
|
172
|
+
csv_items = []
|
|
173
|
+
for row in input_data:
|
|
174
|
+
csv_items.append(StepCsvRowPbo(cells=[str(value) for value in row.values()]))
|
|
175
|
+
header = StepCsvHeaderRowPbo(cells=list(input_data[0].keys()))
|
|
176
|
+
self._add_input(ContainerType.CSV, StepCsvContainerPbo(header=header, items=csv_items))
|
|
177
|
+
|
|
178
|
+
def add_json_input(self, input_data: list[dict[str, Any]]) -> None:
|
|
179
|
+
"""
|
|
180
|
+
Add a JSON input to the next request.
|
|
181
|
+
"""
|
|
182
|
+
self._add_input(ContainerType.JSON, StepJsonContainerPbo(items=[json.dumps(x) for x in input_data]))
|
|
183
|
+
|
|
184
|
+
def add_text_input(self, input_data: list[str]) -> None:
|
|
185
|
+
"""
|
|
186
|
+
Add a text input to the next request.
|
|
187
|
+
"""
|
|
188
|
+
self._add_input(ContainerType.TEXT, StepTextContainerPbo(items=input_data))
|
|
189
|
+
|
|
190
|
+
def clear_inputs(self) -> None:
|
|
191
|
+
"""
|
|
192
|
+
Clear all inputs that have been added to the next request.
|
|
193
|
+
This is useful if you want to start a new request without the previous inputs.
|
|
194
|
+
"""
|
|
195
|
+
self._request_inputs.clear()
|
|
196
|
+
|
|
197
|
+
def add_config_field(self, field_name: str, value: FieldValue | list[str]) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Add a configuration field value to the next request.
|
|
200
|
+
|
|
201
|
+
:param field_name: The name of the configuration field.
|
|
202
|
+
:param value: The value to set for the configuration field. If a list is provided, it will be
|
|
203
|
+
converted to a comma-separated string.
|
|
204
|
+
"""
|
|
205
|
+
if isinstance(value, list):
|
|
206
|
+
value = ",".join(str(x) for x in value)
|
|
207
|
+
if not isinstance(value, FieldValuePbo):
|
|
208
|
+
value = ProtobufUtils.value_to_field_pbo(value)
|
|
209
|
+
self._config_fields[field_name] = value
|
|
210
|
+
|
|
211
|
+
def add_config_fields(self, config_fields: dict[str, FieldValue | list[str]]) -> None:
|
|
212
|
+
"""
|
|
213
|
+
Add multiple configuration field values to the next request.
|
|
214
|
+
|
|
215
|
+
:param config_fields: A dictionary of configuration field names and their corresponding values.
|
|
216
|
+
"""
|
|
217
|
+
for x, y in config_fields.items():
|
|
218
|
+
self.add_config_field(x, y)
|
|
219
|
+
|
|
220
|
+
def clear_configs(self) -> None:
|
|
221
|
+
"""
|
|
222
|
+
Clear all configuration field values that have been added to the next request.
|
|
223
|
+
This is useful if you want to start a new request without the previous configurations.
|
|
224
|
+
"""
|
|
225
|
+
self._config_fields.clear()
|
|
226
|
+
|
|
227
|
+
def clear_request(self) -> None:
|
|
228
|
+
"""
|
|
229
|
+
Clear all inputs and configuration fields that have been added to the next request.
|
|
230
|
+
This is useful if you want to start a new request without the previous inputs and configurations.
|
|
231
|
+
"""
|
|
232
|
+
self.clear_inputs()
|
|
233
|
+
self.clear_configs()
|
|
234
|
+
|
|
235
|
+
def _add_input(self, container_type: ContainerType, items: Any) -> None:
|
|
236
|
+
"""
|
|
237
|
+
Helper method for adding inputs to the next request.
|
|
238
|
+
"""
|
|
239
|
+
container: StepItemContainerPbo | None = None
|
|
240
|
+
match container_type:
|
|
241
|
+
# The content type doesn't matter when we're just testing.
|
|
242
|
+
case ContainerType.BINARY:
|
|
243
|
+
container = StepItemContainerPbo(content_type=ContentTypePbo(), binary_container=items)
|
|
244
|
+
case ContainerType.CSV:
|
|
245
|
+
container = StepItemContainerPbo(content_type=ContentTypePbo(), csv_container=items)
|
|
246
|
+
case ContainerType.JSON:
|
|
247
|
+
container = StepItemContainerPbo(content_type=ContentTypePbo(), json_container=items)
|
|
248
|
+
case ContainerType.TEXT:
|
|
249
|
+
container = StepItemContainerPbo(content_type=ContentTypePbo(), text_container=items)
|
|
250
|
+
case _:
|
|
251
|
+
raise ValueError(f"Unsupported container type: {container_type}")
|
|
252
|
+
self._request_inputs.append(container)
|
|
253
|
+
|
|
254
|
+
def get_service_details(self) -> ToolDetailsResponsePbo:
|
|
255
|
+
"""
|
|
256
|
+
Get the details of the agents from the server.
|
|
257
|
+
|
|
258
|
+
:return: A ToolDetailsResponsePbo object containing the details of the agent service.
|
|
259
|
+
"""
|
|
260
|
+
with grpc.insecure_channel(self.grpc_server_url, options=self.options) as channel:
|
|
261
|
+
stub = ToolServiceStub(channel)
|
|
262
|
+
return stub.GetToolDetails(ToolDetailsRequestPbo(sapio_conn_info=self.connection))
|
|
263
|
+
|
|
264
|
+
def call_agent(self, agent_name: str, is_verbose: bool = True, is_dry_run: bool = False) -> AgentOutput:
|
|
265
|
+
"""
|
|
266
|
+
Send the request to the agent service for a particular agent name. This will send all the inputs that have been
|
|
267
|
+
added using the add_X_input functions.
|
|
268
|
+
|
|
269
|
+
:param agent_name: The name of the agent to call on the server.
|
|
270
|
+
:param is_verbose: If True, the agent will log verbosely.
|
|
271
|
+
:param is_dry_run: If True, the agent will not be executed, but the request will be validated.
|
|
272
|
+
:return: An AgentOutput object containing the results of the agent service call.
|
|
273
|
+
"""
|
|
274
|
+
with grpc.insecure_channel(self.grpc_server_url, options=self.options) as channel:
|
|
275
|
+
stub = ToolServiceStub(channel)
|
|
276
|
+
|
|
277
|
+
response: ProcessStepResponsePbo = stub.ProcessData(
|
|
278
|
+
ProcessStepRequestPbo(
|
|
279
|
+
sapio_user=self.connection,
|
|
280
|
+
tool_name=agent_name,
|
|
281
|
+
config_field_values=self._config_fields,
|
|
282
|
+
dry_run=is_dry_run,
|
|
283
|
+
verbose_logging=is_verbose,
|
|
284
|
+
input=[
|
|
285
|
+
StepInputBatchPbo(is_partial=False, item_container=item)
|
|
286
|
+
for item in self._request_inputs
|
|
287
|
+
]
|
|
288
|
+
)
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
results = AgentOutput(agent_name)
|
|
292
|
+
|
|
293
|
+
match response.status:
|
|
294
|
+
case ProcessStepResponseStatusPbo.SUCCESS:
|
|
295
|
+
results.status = "Success"
|
|
296
|
+
case ProcessStepResponseStatusPbo.FAILURE:
|
|
297
|
+
results.status = "Failure"
|
|
298
|
+
case _:
|
|
299
|
+
results.status = "Unknown"
|
|
300
|
+
results.message = response.status_message
|
|
301
|
+
|
|
302
|
+
for item in response.output:
|
|
303
|
+
container = item.item_container
|
|
304
|
+
|
|
305
|
+
if container.HasField("binary_container"):
|
|
306
|
+
results.binary_output.append(list(container.binary_container.items))
|
|
307
|
+
elif container.HasField("csv_container"):
|
|
308
|
+
csv_output: list[dict[str, Any]] = []
|
|
309
|
+
for header in container.csv_container.header.cells:
|
|
310
|
+
output_row: dict[str, Any] = {}
|
|
311
|
+
for i, row in enumerate(container.csv_container.items):
|
|
312
|
+
output_row[header] = row.cells[i]
|
|
313
|
+
csv_output.append(output_row)
|
|
314
|
+
results.csv_output.append(csv_output)
|
|
315
|
+
elif container.HasField("json_container"):
|
|
316
|
+
results.json_output.append([json.loads(x) for x in container.json_container.items])
|
|
317
|
+
elif container.HasField("text_container"):
|
|
318
|
+
results.text_output.append(list(container.text_container.items))
|
|
319
|
+
|
|
320
|
+
for record in response.new_records:
|
|
321
|
+
field_map: dict[str, Any] = {x: ProtobufUtils.field_pbo_to_value(y) for x, y in record.fields.items()}
|
|
322
|
+
results.new_records.append(field_map)
|
|
323
|
+
|
|
324
|
+
results.logs.extend(response.log)
|
|
325
|
+
|
|
326
|
+
return results
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class TestConverterClient:
|
|
330
|
+
"""
|
|
331
|
+
A client for testing a ConverterService.
|
|
332
|
+
"""
|
|
333
|
+
grpc_server_url: str
|
|
334
|
+
options: list[tuple[str, Any]] | None
|
|
335
|
+
|
|
336
|
+
def __init__(self, grpc_server_url: str, options: list[tuple[str, Any]] | None = None):
|
|
337
|
+
"""
|
|
338
|
+
:param grpc_server_url: The URL of the gRPC server to connect to.
|
|
339
|
+
:param options: Optional list of gRPC channel options.
|
|
340
|
+
"""
|
|
341
|
+
self.grpc_server_url = grpc_server_url
|
|
342
|
+
self.options = options
|
|
343
|
+
|
|
344
|
+
def get_converter_details(self) -> ConverterDetailsResponsePbo:
|
|
345
|
+
"""
|
|
346
|
+
Get the details of the converters from the server.
|
|
347
|
+
|
|
348
|
+
:return: A ToolDetailsResponsePbo object containing the details of the converter service.
|
|
349
|
+
"""
|
|
350
|
+
with grpc.insecure_channel(self.grpc_server_url, options=self.options) as channel:
|
|
351
|
+
stub = ConverterServiceStub(channel)
|
|
352
|
+
return stub.GetConverterDetails(ConverterDetailsRequestPbo())
|
|
353
|
+
|
|
354
|
+
def convert_content(self, input_container: StepItemContainerPbo, target_type: ContentTypePbo) \
|
|
355
|
+
-> StepItemContainerPbo:
|
|
356
|
+
"""
|
|
357
|
+
Convert the content of the input container to the target content type.
|
|
358
|
+
|
|
359
|
+
:param input_container: The input container to convert. This container must have a ContentTypePbo set that
|
|
360
|
+
matches one of the input types that the converter service supports.
|
|
361
|
+
:param target_type: The target content type to convert to. This must match one of the target types that the
|
|
362
|
+
converter service supports.
|
|
363
|
+
:return: A StepItemContainerPbo object containing the converted content.
|
|
364
|
+
"""
|
|
365
|
+
with grpc.insecure_channel(self.grpc_server_url, options=self.options) as channel:
|
|
366
|
+
stub = ConverterServiceStub(channel)
|
|
367
|
+
response: ConvertResponsePbo = stub.ConvertContent(
|
|
368
|
+
ConvertRequestPbo(item_container=input_container, target_content_type=target_type)
|
|
369
|
+
)
|
|
370
|
+
return response.item_container
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import gzip
|
|
1
2
|
import io
|
|
3
|
+
import tarfile
|
|
4
|
+
import time
|
|
2
5
|
import warnings
|
|
3
6
|
import zipfile
|
|
4
7
|
|
|
@@ -322,7 +325,7 @@ class FileUtil:
|
|
|
322
325
|
@staticmethod
|
|
323
326
|
def zip_files(files: dict[str, str | bytes]) -> bytes:
|
|
324
327
|
"""
|
|
325
|
-
Create a zip file for a collection of files.
|
|
328
|
+
Create a .zip file for a collection of files.
|
|
326
329
|
|
|
327
330
|
:param files: A dictionary of file name to file data as a string or bytes.
|
|
328
331
|
:return: The bytes for a zip file containing the input files.
|
|
@@ -335,6 +338,130 @@ class FileUtil:
|
|
|
335
338
|
# throws an I/O exception.
|
|
336
339
|
return zip_buffer.getvalue()
|
|
337
340
|
|
|
341
|
+
# FR-47422: Add a function for unzipping files that may have been zipped by the above function.
|
|
342
|
+
@staticmethod
|
|
343
|
+
def unzip_files(zip_file: bytes) -> dict[str, bytes]:
|
|
344
|
+
"""
|
|
345
|
+
Decompress a .zip file from an in-memory bytes object and extracts all files into a dictionary.
|
|
346
|
+
|
|
347
|
+
:param zip_file: The bytes of the zip file to be decompressed.
|
|
348
|
+
:return: A dictionary of file name to file bytes for each file in the zip.
|
|
349
|
+
"""
|
|
350
|
+
extracted_files: dict[str, bytes] = {}
|
|
351
|
+
with io.BytesIO(zip_file) as zip_buffer:
|
|
352
|
+
with zipfile.ZipFile(zip_buffer, "r") as zip_file:
|
|
353
|
+
for file_name in zip_file.namelist():
|
|
354
|
+
with zip_file.open(file_name) as file:
|
|
355
|
+
extracted_files[file_name] = file.read()
|
|
356
|
+
return extracted_files
|
|
357
|
+
|
|
358
|
+
# FR-47422: Add functions for compressing and decompressing .gz, .tar, and .tar.gz files.
|
|
359
|
+
@staticmethod
|
|
360
|
+
def gzip_file(file_data: bytes | str) -> bytes:
|
|
361
|
+
"""
|
|
362
|
+
Create a .gz file for a single file.
|
|
363
|
+
|
|
364
|
+
:param file_data: The file data to be compressed as bytes or a string.
|
|
365
|
+
:return: The bytes of the gzip-compressed file.
|
|
366
|
+
"""
|
|
367
|
+
return gzip.compress(file_data.encode() if isinstance(file_data, str) else file_data)
|
|
368
|
+
|
|
369
|
+
@staticmethod
|
|
370
|
+
def ungzip_file(gzip_file: bytes) -> bytes:
|
|
371
|
+
"""
|
|
372
|
+
Decompress a .gz file.
|
|
373
|
+
|
|
374
|
+
:param gzip_file: The bytes of the gzip-compressed file.
|
|
375
|
+
:return: The decompressed file data as bytes.
|
|
376
|
+
"""
|
|
377
|
+
return gzip.decompress(gzip_file)
|
|
378
|
+
|
|
379
|
+
@staticmethod
|
|
380
|
+
def tar_files(files: dict[str, str | bytes]) -> bytes:
|
|
381
|
+
"""
|
|
382
|
+
Create a .tar file for a collection of files.
|
|
383
|
+
|
|
384
|
+
:param files: A dictionary of file name to file data as a string or bytes.
|
|
385
|
+
:return: The bytes for a tar file containing the input files.
|
|
386
|
+
"""
|
|
387
|
+
with io.BytesIO() as tar_buffer:
|
|
388
|
+
with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
|
|
389
|
+
for name, data in files.items():
|
|
390
|
+
if isinstance(data, str):
|
|
391
|
+
data: bytes = data.encode('utf-8')
|
|
392
|
+
|
|
393
|
+
tarinfo = tarfile.TarInfo(name=name)
|
|
394
|
+
tarinfo.size = len(data)
|
|
395
|
+
tarinfo.mtime = int(time.time())
|
|
396
|
+
|
|
397
|
+
with io.BytesIO(data) as file:
|
|
398
|
+
tar.addfile(tarinfo=tarinfo, fileobj=file)
|
|
399
|
+
|
|
400
|
+
tar_buffer.seek(0)
|
|
401
|
+
return tar_buffer.getvalue()
|
|
402
|
+
|
|
403
|
+
@staticmethod
|
|
404
|
+
def untar_files(tar_file: bytes) -> dict[str, bytes]:
|
|
405
|
+
"""
|
|
406
|
+
Decompress a .tar file from an in-memory bytes object and extracts all files into a dictionary.
|
|
407
|
+
|
|
408
|
+
:param tar_file: The bytes of the tar file to be decompressed.
|
|
409
|
+
:return: A dictionary of file name to file bytes for each file in the tar.
|
|
410
|
+
"""
|
|
411
|
+
extracted_files: dict[str, bytes] = {}
|
|
412
|
+
with io.BytesIO(tar_file) as tar_buffer:
|
|
413
|
+
with tarfile.open(fileobj=tar_buffer, mode="r") as tar:
|
|
414
|
+
for member in tar.getmembers():
|
|
415
|
+
if member.isfile():
|
|
416
|
+
file_obj = tar.extractfile(member)
|
|
417
|
+
if file_obj:
|
|
418
|
+
with file_obj:
|
|
419
|
+
extracted_files[member.name] = file_obj.read()
|
|
420
|
+
return extracted_files
|
|
421
|
+
|
|
422
|
+
@staticmethod
|
|
423
|
+
def tar_gzip_files(files: dict[str, str | bytes]) -> bytes:
|
|
424
|
+
"""
|
|
425
|
+
Create a .tar.gz file for a collection of files.
|
|
426
|
+
|
|
427
|
+
:param files: A dictionary of file name to file data as a string or bytes.
|
|
428
|
+
:return: The bytes for a tar.gz file containing the input files.
|
|
429
|
+
"""
|
|
430
|
+
with io.BytesIO() as tar_buffer:
|
|
431
|
+
with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar:
|
|
432
|
+
for name, data in files.items():
|
|
433
|
+
if isinstance(data, str):
|
|
434
|
+
data: bytes = data.encode('utf-8')
|
|
435
|
+
|
|
436
|
+
tarinfo = tarfile.TarInfo(name=name)
|
|
437
|
+
tarinfo.size = len(data)
|
|
438
|
+
tarinfo.mtime = int(time.time())
|
|
439
|
+
|
|
440
|
+
with io.BytesIO(data) as file:
|
|
441
|
+
tar.addfile(tarinfo=tarinfo, fileobj=file)
|
|
442
|
+
|
|
443
|
+
tar_buffer.seek(0)
|
|
444
|
+
return tar_buffer.getvalue()
|
|
445
|
+
|
|
446
|
+
@staticmethod
|
|
447
|
+
def untar_gzip_files(tar_gzip_file: bytes) -> dict[str, bytes]:
|
|
448
|
+
"""
|
|
449
|
+
Decompress a .tar.gz file from an in-memory bytes object and extracts all files into a dictionary.
|
|
450
|
+
|
|
451
|
+
:param tar_gzip_file: The bytes of the tar.gz file to be decompressed.
|
|
452
|
+
:return: A dictionary of file name to file bytes for each file in the tar.gz
|
|
453
|
+
"""
|
|
454
|
+
extracted_files: dict[str, bytes] = {}
|
|
455
|
+
with io.BytesIO(tar_gzip_file) as tar_buffer:
|
|
456
|
+
with tarfile.open(fileobj=tar_buffer, mode="r:gz") as tar:
|
|
457
|
+
for member in tar.getmembers():
|
|
458
|
+
if member.isfile():
|
|
459
|
+
file_obj = tar.extractfile(member)
|
|
460
|
+
if file_obj:
|
|
461
|
+
with file_obj:
|
|
462
|
+
extracted_files[member.name] = file_obj.read()
|
|
463
|
+
return extracted_files
|
|
464
|
+
|
|
338
465
|
# Deprecated functions:
|
|
339
466
|
|
|
340
467
|
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import tempfile
|
|
4
|
+
from typing import Callable, Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
# FR-47422: Created class.
|
|
8
|
+
class TempFileHandler:
|
|
9
|
+
"""
|
|
10
|
+
A utility class to manage temporary files and directories.
|
|
11
|
+
"""
|
|
12
|
+
directories: list[str]
|
|
13
|
+
files: list[str]
|
|
14
|
+
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self.directories = []
|
|
17
|
+
self.files = []
|
|
18
|
+
|
|
19
|
+
def create_temp_directory(self) -> str:
|
|
20
|
+
"""
|
|
21
|
+
Create a temporary directory.
|
|
22
|
+
|
|
23
|
+
:return: The path to a newly created temporary directory.
|
|
24
|
+
"""
|
|
25
|
+
directory: str = tempfile.mkdtemp()
|
|
26
|
+
self.directories.append(directory)
|
|
27
|
+
return directory
|
|
28
|
+
|
|
29
|
+
def create_temp_file(self, data: str | bytes, suffix: str = "") -> str:
|
|
30
|
+
"""
|
|
31
|
+
Create a temporary file with the specified data and optional suffix.
|
|
32
|
+
|
|
33
|
+
:param data: The data to write to the temporary file.
|
|
34
|
+
:param suffix: An optional suffix for the temporary file.
|
|
35
|
+
:return: The path to a newly created temporary file containing the provided data.
|
|
36
|
+
"""
|
|
37
|
+
mode: str = 'w' if isinstance(data, str) else 'wb'
|
|
38
|
+
with tempfile.NamedTemporaryFile(mode=mode, suffix=suffix, delete=False) as tmp_file:
|
|
39
|
+
tmp_file.write(data)
|
|
40
|
+
file_path: str = tmp_file.name
|
|
41
|
+
self.files.append(file_path)
|
|
42
|
+
return file_path
|
|
43
|
+
|
|
44
|
+
def create_temp_file_from_func(self, func: Callable, params: dict[str, Any], suffix: str = "",
|
|
45
|
+
is_binary: bool = True) -> str:
|
|
46
|
+
"""
|
|
47
|
+
Create a temporary file and populate it using the provided function. The function should accept parameters as
|
|
48
|
+
specified in the `params` dictionary.
|
|
49
|
+
|
|
50
|
+
:param func: The function to call with the temporary file path that will populate the file.
|
|
51
|
+
:param params: Keyword arguments to pass to the function. If "<NEW_FILE>" is used as a value, it will be
|
|
52
|
+
replaced with the temporary file object. If "<NEW_FILE_PATH>" is used as a value, it will be replaced with
|
|
53
|
+
the temporary file path.
|
|
54
|
+
:param suffix: An optional suffix for the temporary file.
|
|
55
|
+
:param is_binary: Whether to open the temporary file in binary mode.
|
|
56
|
+
:return: The path to the newly created temporary file.
|
|
57
|
+
"""
|
|
58
|
+
mode: str = 'wb' if is_binary else 'w'
|
|
59
|
+
with tempfile.NamedTemporaryFile(mode, suffix=suffix, delete=False) as tmp_file:
|
|
60
|
+
for key, value in params.items():
|
|
61
|
+
if value == "<NEW_FILE>":
|
|
62
|
+
params[key] = tmp_file
|
|
63
|
+
elif value == "<NEW_FILE_PATH>":
|
|
64
|
+
params[key] = tmp_file.name
|
|
65
|
+
func(**params)
|
|
66
|
+
file_path: str = tmp_file.name
|
|
67
|
+
self.files.append(file_path)
|
|
68
|
+
return file_path
|
|
69
|
+
|
|
70
|
+
def cleanup(self) -> None:
|
|
71
|
+
"""
|
|
72
|
+
Delete all temporary files and directories created by this handler.
|
|
73
|
+
"""
|
|
74
|
+
for directory in self.directories:
|
|
75
|
+
if os.path.exists(directory):
|
|
76
|
+
shutil.rmtree(directory)
|
|
77
|
+
self.directories.clear()
|
|
78
|
+
|
|
79
|
+
for file_path in self.files:
|
|
80
|
+
if os.path.exists(file_path):
|
|
81
|
+
os.remove(file_path)
|
|
82
|
+
self.files.clear()
|
|
@@ -140,7 +140,7 @@ class AbstractWebserviceHandler(AbstractWebhookHandler):
|
|
|
140
140
|
# Get the login credentials from the headers.
|
|
141
141
|
auth: str = headers.get("Authorization")
|
|
142
142
|
if auth and auth.startswith("Basic "):
|
|
143
|
-
credentials: list[str] = b64decode(auth.split("Basic ")[1]).decode().split(":")
|
|
143
|
+
credentials: list[str] = b64decode(auth.split("Basic ")[1]).decode().split(":", 1)
|
|
144
144
|
user = self.basic_auth(url, credentials[0], credentials[1])
|
|
145
145
|
elif auth and auth.startswith("Bearer "):
|
|
146
146
|
user = self.bearer_token_auth(url, auth.split("Bearer ")[1])
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: sapiopycommons
|
|
3
|
-
Version: 2025.10.
|
|
3
|
+
Version: 2025.10.9a776
|
|
4
4
|
Summary: Official Sapio Python API Utilities Package
|
|
5
5
|
Project-URL: Homepage, https://github.com/sapiosciences
|
|
6
6
|
Author-email: Jonathan Steck <jsteck@sapiosciences.com>, Yechen Qiao <yqiao@sapiosciences.com>
|