sapiopycommons 2024.3.19a157__py3-none-any.whl → 2025.1.17a402__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sapiopycommons might be problematic. Click here for more details.
- sapiopycommons/callbacks/__init__.py +0 -0
- sapiopycommons/callbacks/callback_util.py +2041 -0
- sapiopycommons/callbacks/field_builder.py +545 -0
- sapiopycommons/chem/IndigoMolecules.py +46 -1
- sapiopycommons/chem/Molecules.py +100 -21
- sapiopycommons/customreport/__init__.py +0 -0
- sapiopycommons/customreport/column_builder.py +60 -0
- sapiopycommons/customreport/custom_report_builder.py +137 -0
- sapiopycommons/customreport/term_builder.py +315 -0
- sapiopycommons/datatype/attachment_util.py +14 -15
- sapiopycommons/datatype/data_fields.py +61 -0
- sapiopycommons/datatype/pseudo_data_types.py +440 -0
- sapiopycommons/eln/experiment_handler.py +355 -91
- sapiopycommons/eln/experiment_report_util.py +649 -0
- sapiopycommons/eln/plate_designer.py +152 -0
- sapiopycommons/files/complex_data_loader.py +31 -0
- sapiopycommons/files/file_bridge.py +149 -25
- sapiopycommons/files/file_bridge_handler.py +555 -0
- sapiopycommons/files/file_data_handler.py +633 -0
- sapiopycommons/files/file_util.py +263 -163
- sapiopycommons/files/file_validator.py +569 -0
- sapiopycommons/files/file_writer.py +377 -0
- sapiopycommons/flowcyto/flow_cyto.py +77 -0
- sapiopycommons/flowcyto/flowcyto_data.py +75 -0
- sapiopycommons/general/accession_service.py +375 -0
- sapiopycommons/general/aliases.py +250 -15
- sapiopycommons/general/audit_log.py +185 -0
- sapiopycommons/general/custom_report_util.py +251 -31
- sapiopycommons/general/directive_util.py +86 -0
- sapiopycommons/general/exceptions.py +69 -7
- sapiopycommons/general/popup_util.py +59 -7
- sapiopycommons/general/sapio_links.py +50 -0
- sapiopycommons/general/storage_util.py +148 -0
- sapiopycommons/general/time_util.py +91 -7
- sapiopycommons/multimodal/multimodal.py +146 -0
- sapiopycommons/multimodal/multimodal_data.py +490 -0
- sapiopycommons/processtracking/__init__.py +0 -0
- sapiopycommons/processtracking/custom_workflow_handler.py +406 -0
- sapiopycommons/processtracking/endpoints.py +192 -0
- sapiopycommons/recordmodel/record_handler.py +621 -148
- sapiopycommons/rules/eln_rule_handler.py +87 -8
- sapiopycommons/rules/on_save_rule_handler.py +87 -12
- sapiopycommons/sftpconnect/__init__.py +0 -0
- sapiopycommons/sftpconnect/sftp_builder.py +70 -0
- sapiopycommons/webhook/webhook_context.py +39 -0
- sapiopycommons/webhook/webhook_handlers.py +614 -71
- sapiopycommons/webhook/webservice_handlers.py +317 -0
- {sapiopycommons-2024.3.19a157.dist-info → sapiopycommons-2025.1.17a402.dist-info}/METADATA +5 -4
- sapiopycommons-2025.1.17a402.dist-info/RECORD +60 -0
- {sapiopycommons-2024.3.19a157.dist-info → sapiopycommons-2025.1.17a402.dist-info}/WHEEL +1 -1
- sapiopycommons-2024.3.19a157.dist-info/RECORD +0 -28
- {sapiopycommons-2024.3.19a157.dist-info → sapiopycommons-2025.1.17a402.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import io
|
|
2
|
+
import warnings
|
|
3
|
+
import zipfile
|
|
2
4
|
|
|
3
5
|
import pandas
|
|
4
6
|
from numpy import dtype
|
|
@@ -13,165 +15,16 @@ from sapiopycommons.general.exceptions import SapioUserErrorException
|
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
# FR-46064 - Initial port of PyWebhookUtils to sapiopycommons.
|
|
18
|
+
# FR-46716 - Add comments noting that some functions are deprecated in favor of CallbackUtil.
|
|
16
19
|
class FileUtil:
|
|
17
20
|
"""
|
|
18
21
|
Utilities for the handling of files, including the requesting of files from the user and the parsing of files into
|
|
19
22
|
tokenized lists. Makes use of Pandas DataFrames for any file parsing purposes.
|
|
20
23
|
"""
|
|
21
|
-
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
22
|
-
@staticmethod
|
|
23
|
-
def write_file(file_name: str, file_bytes: bytes, *, request_context: str | None = None) -> SapioWebhookResult:
|
|
24
|
-
"""
|
|
25
|
-
Send a file to the client.
|
|
26
|
-
|
|
27
|
-
The calling webhook must catch the WriteFileResult that the client will send back.
|
|
28
|
-
|
|
29
|
-
:param file_name: The name of the file.
|
|
30
|
-
:param file_bytes: The bytes of the file.
|
|
31
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
32
|
-
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
33
|
-
"""
|
|
34
|
-
return SapioWebhookResult(True, client_callback_request=WriteFileRequest(file_bytes, file_name,
|
|
35
|
-
request_context))
|
|
36
|
-
|
|
37
|
-
@staticmethod
|
|
38
|
-
def write_files(files: dict[str, bytes], *, request_context: str | None = None) -> SapioWebhookResult:
|
|
39
|
-
"""
|
|
40
|
-
Send a collection of files to the client.
|
|
41
|
-
|
|
42
|
-
The calling webhook must catch the MultiFileResult that the client will send back.
|
|
43
|
-
|
|
44
|
-
:param files: A dictionary of files names to file bytes.
|
|
45
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
46
|
-
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
47
|
-
"""
|
|
48
|
-
return SapioWebhookResult(True, client_callback_request=MultiFileRequest(files, request_context))
|
|
49
|
-
|
|
50
|
-
@staticmethod
|
|
51
|
-
def request_file(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
52
|
-
*, request_context: str | None = None) \
|
|
53
|
-
-> tuple[SapioWebhookResult | None, str | None, bytes | None]:
|
|
54
|
-
"""
|
|
55
|
-
Request a single file from the user. This function handles the entire client callback interaction for the
|
|
56
|
-
requesting of the file, including if the user cancels the file upload prompt.
|
|
57
|
-
|
|
58
|
-
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
59
|
-
result containing the client callback to request the file. The second time it is called, it will return the
|
|
60
|
-
file name and bytes from the callback result.
|
|
61
|
-
|
|
62
|
-
:param context: The current webhook context.
|
|
63
|
-
:param title: The title of the file prompt dialog.
|
|
64
|
-
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
65
|
-
exception if an incorrect file extension is provided.
|
|
66
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
67
|
-
:return: A tuple with the following elements.
|
|
68
|
-
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
69
|
-
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
70
|
-
1 - The file name of the requested file if the user provided one.
|
|
71
|
-
2 - The file bytes of the requested file if the user provided one.
|
|
72
|
-
"""
|
|
73
|
-
client_callback = context.client_callback_result
|
|
74
|
-
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
75
|
-
# If the user cancels, terminate the interaction.
|
|
76
|
-
if client_callback is not None and client_callback.user_cancelled:
|
|
77
|
-
return SapioWebhookResult(True), None, None
|
|
78
|
-
# If no extensions were provided, use an empty list for the extensions instead.
|
|
79
|
-
if exts is None:
|
|
80
|
-
exts = []
|
|
81
|
-
|
|
82
|
-
# If the client callback isn't a FilePromptResult, then it's either None or some other callback result, meaning
|
|
83
|
-
# we need to send a new request. We may also send a new request if the client callback result is a
|
|
84
|
-
# FilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
85
|
-
# result from a different call to request_file.
|
|
86
|
-
is_file_result = isinstance(client_callback, FilePromptResult)
|
|
87
|
-
if not is_file_result or (is_file_result and result_context != request_context):
|
|
88
|
-
prompt = FilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
89
|
-
callback_context_data=request_context)
|
|
90
|
-
return SapioWebhookResult(True, client_callback_request=prompt), None, None
|
|
91
|
-
|
|
92
|
-
# Get the file from the result. Enforce that the provided data isn't empty, and that the file path ends in
|
|
93
|
-
# one of the allowed extensions.
|
|
94
|
-
# noinspection PyTypeChecker
|
|
95
|
-
result: FilePromptResult = client_callback
|
|
96
|
-
file_path: str | None = result.file_path
|
|
97
|
-
file_bytes: bytes | None = result.file_bytes
|
|
98
|
-
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
99
|
-
return None, file_path, file_bytes
|
|
100
|
-
|
|
101
24
|
@staticmethod
|
|
102
|
-
def
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
"""
|
|
106
|
-
Request multiple files from the user. This function handles the entire client callback interaction for the
|
|
107
|
-
requesting of the files, including if the user cancels the file upload prompt.
|
|
108
|
-
|
|
109
|
-
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
110
|
-
result containing the client callback to request the files. The second time it is called, it will return each
|
|
111
|
-
file name and bytes from the callback result.
|
|
112
|
-
|
|
113
|
-
:param context: The current webhook context.
|
|
114
|
-
:param title: The title of the file prompt dialog.
|
|
115
|
-
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
116
|
-
exception if an incorrect file extension is provided.
|
|
117
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
118
|
-
:return: A tuple with the following elements.
|
|
119
|
-
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
120
|
-
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
121
|
-
1 - A dictionary that maps the file names to the file bytes for each provided file.
|
|
122
|
-
"""
|
|
123
|
-
client_callback = context.client_callback_result
|
|
124
|
-
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
125
|
-
# If the user cancels, terminate the interaction.
|
|
126
|
-
if client_callback is not None and client_callback.user_cancelled:
|
|
127
|
-
return SapioWebhookResult(True), None
|
|
128
|
-
# If no extensions were provided, use an empty list for the extensions instead.
|
|
129
|
-
if exts is None:
|
|
130
|
-
exts = []
|
|
131
|
-
|
|
132
|
-
# If the client callback isn't a MultiFilePromptResult, then it's either None or some other callback result,
|
|
133
|
-
# meaning we need to send a new request. We may also send a new request if the client callback result is a
|
|
134
|
-
# MultiFilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
135
|
-
# result from a different call to request_file.
|
|
136
|
-
is_file_result = isinstance(client_callback, MultiFilePromptResult)
|
|
137
|
-
if not is_file_result or (is_file_result and result_context != request_context):
|
|
138
|
-
prompt = MultiFilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
139
|
-
callback_context_data=request_context)
|
|
140
|
-
return SapioWebhookResult(True, client_callback_request=prompt), None
|
|
141
|
-
|
|
142
|
-
# Get the files from the result. Enforce that the provided data isn't empty, and that the file paths end in
|
|
143
|
-
# one of the allowed extensions.
|
|
144
|
-
# noinspection PyTypeChecker
|
|
145
|
-
result: MultiFilePromptResult = client_callback
|
|
146
|
-
for file_path, file_bytes in result.files.items():
|
|
147
|
-
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
148
|
-
return None, result.files
|
|
149
|
-
|
|
150
|
-
@staticmethod
|
|
151
|
-
def __verify_file(file_path: str, file_bytes: bytes, allowed_extensions: list[str]):
|
|
152
|
-
"""
|
|
153
|
-
Verify that the provided file was read (i.e. the file path and file bytes aren't None or empty) and that it
|
|
154
|
-
has the correct file extension. Raises a user error exception if something about the file is incorrect.
|
|
155
|
-
|
|
156
|
-
:param file_path: The name of the file to verify.
|
|
157
|
-
:param file_bytes: The bytes of the file to verify.
|
|
158
|
-
:param allowed_extensions: The file extensions that the file path is allowed to have.
|
|
159
|
-
"""
|
|
160
|
-
if file_path is None or len(file_path) == 0 or file_bytes is None or len(file_bytes) == 0:
|
|
161
|
-
raise SapioUserErrorException("Empty file provided or file unable to be read.")
|
|
162
|
-
if len(allowed_extensions) != 0:
|
|
163
|
-
matches: bool = False
|
|
164
|
-
for ext in allowed_extensions:
|
|
165
|
-
if file_path.endswith("." + ext):
|
|
166
|
-
matches = True
|
|
167
|
-
break
|
|
168
|
-
if matches is False:
|
|
169
|
-
raise SapioUserErrorException("Unsupported file type. Expecting the following extension(s): "
|
|
170
|
-
+ (",".join(allowed_extensions)))
|
|
171
|
-
|
|
172
|
-
@staticmethod
|
|
173
|
-
def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0) \
|
|
174
|
-
-> tuple[list[dict[str, str]], list[list[str]]]:
|
|
25
|
+
def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
26
|
+
seperator: str = ",", *, encoding: str | None = None, encoding_error: str | None = "strict",
|
|
27
|
+
exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
|
|
175
28
|
"""
|
|
176
29
|
Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
|
|
177
30
|
must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
|
|
@@ -183,22 +36,35 @@ class FileUtil:
|
|
|
183
36
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
184
37
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
185
38
|
is assumed to be the header row.
|
|
39
|
+
:param seperator: The character that separates cells in the table.
|
|
40
|
+
:param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
|
|
41
|
+
contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
|
|
42
|
+
ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
|
|
43
|
+
:param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
|
|
44
|
+
is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
|
|
45
|
+
characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
|
|
46
|
+
https://docs.python.org/3/library/codecs.html#error-handlers
|
|
47
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
48
|
+
the first element of the returned tuple.
|
|
186
49
|
:return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
187
50
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
188
51
|
If the header row index is 0 or None, this list will be empty.
|
|
189
52
|
"""
|
|
190
53
|
# Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
|
|
191
54
|
# while the second is the body of the file below the header row.
|
|
192
|
-
file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index
|
|
55
|
+
file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
|
|
56
|
+
encoding=encoding, encoding_error=encoding_error)
|
|
193
57
|
# Parse the metadata from above the header row index into a list of lists.
|
|
194
58
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
195
59
|
# Parse the data from the file body into a list of dicts.
|
|
196
60
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
61
|
+
if exception_on_empty and not rows:
|
|
62
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
197
63
|
return rows, metadata
|
|
198
64
|
|
|
199
65
|
@staticmethod
|
|
200
|
-
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0
|
|
201
|
-
|
|
66
|
+
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
67
|
+
*, exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
|
|
202
68
|
"""
|
|
203
69
|
Tokenize an XLSX file row by row.
|
|
204
70
|
|
|
@@ -209,6 +75,8 @@ class FileUtil:
|
|
|
209
75
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
210
76
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
211
77
|
is assumed to be the header row.
|
|
78
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
79
|
+
the first element of the returned tuple.
|
|
212
80
|
:return: The XLSX parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
213
81
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
214
82
|
If the header row index is 0 or None, this list will be empty.
|
|
@@ -220,10 +88,13 @@ class FileUtil:
|
|
|
220
88
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
221
89
|
# Parse the data from the file body into a list of dicts.
|
|
222
90
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
91
|
+
if exception_on_empty and not rows:
|
|
92
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
223
93
|
return rows, metadata
|
|
224
94
|
|
|
225
95
|
@staticmethod
|
|
226
|
-
def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0
|
|
96
|
+
def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
|
|
97
|
+
*, encoding: str | None = None, encoding_error: str | None = "strict") \
|
|
227
98
|
-> tuple[DataFrame, DataFrame | None]:
|
|
228
99
|
"""
|
|
229
100
|
Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
|
|
@@ -234,6 +105,14 @@ class FileUtil:
|
|
|
234
105
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
235
106
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
236
107
|
is assumed to be the header row.
|
|
108
|
+
:param seperator: The character that separates cells in the table.
|
|
109
|
+
:param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
|
|
110
|
+
contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
|
|
111
|
+
ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
|
|
112
|
+
:param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
|
|
113
|
+
is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
|
|
114
|
+
characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
|
|
115
|
+
https://docs.python.org/3/library/codecs.html#error-handlers
|
|
237
116
|
:return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
|
|
238
117
|
metadata from above the header row, or None if there is no metadata.
|
|
239
118
|
"""
|
|
@@ -245,13 +124,14 @@ class FileUtil:
|
|
|
245
124
|
# can throw off the header row index.
|
|
246
125
|
file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
|
|
247
126
|
skiprows=lambda x: x >= header_row_index,
|
|
248
|
-
skip_blank_lines=False
|
|
127
|
+
skip_blank_lines=False, sep=seperator, encoding=encoding,
|
|
128
|
+
encoding_errors=encoding_error)
|
|
249
129
|
with io.BytesIO(file_bytes) as file_io:
|
|
250
130
|
# The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
|
|
251
131
|
# because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
|
|
252
132
|
# string.
|
|
253
133
|
file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
|
|
254
|
-
skip_blank_lines=False)
|
|
134
|
+
skip_blank_lines=False, sep=seperator, encoding=encoding)
|
|
255
135
|
|
|
256
136
|
return file_body, file_metadata
|
|
257
137
|
|
|
@@ -350,11 +230,15 @@ class FileUtil:
|
|
|
350
230
|
|
|
351
231
|
# Warn about improper headers. Confirm that the header contains each of the header column names that we want.
|
|
352
232
|
if required_headers is not None:
|
|
233
|
+
# FR-46702: Report all missing headers instead of only the first missing header.
|
|
234
|
+
missing_headers: list[str] = []
|
|
353
235
|
for required in required_headers:
|
|
354
236
|
if required not in witnessed_headers:
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
237
|
+
missing_headers.append("\"" + required + "\"")
|
|
238
|
+
if missing_headers:
|
|
239
|
+
at_row = " at row " + str(header_row_index + 1) if header_row_index is not None else ""
|
|
240
|
+
raise SapioUserErrorException(f"Incorrect file headers or incorrectly formatted table. Header(s) "
|
|
241
|
+
f"{', '.join(missing_headers)} not found{at_row}.")
|
|
358
242
|
|
|
359
243
|
return rows
|
|
360
244
|
|
|
@@ -366,7 +250,7 @@ class FileUtil:
|
|
|
366
250
|
:param file_data: The CSV file to be converted.
|
|
367
251
|
:return: The bytes of the CSV file converted to an XLSX file.
|
|
368
252
|
"""
|
|
369
|
-
with (io.BytesIO(file_data) if isinstance(file_data,
|
|
253
|
+
with (io.BytesIO(file_data.encode() if isinstance(file_data, str) else file_data)) as csv:
|
|
370
254
|
# Setting header to false makes pandas read the CSV as-is.
|
|
371
255
|
data_frame = pandas.read_csv(csv, sep=",", header=None)
|
|
372
256
|
|
|
@@ -376,3 +260,219 @@ class FileUtil:
|
|
|
376
260
|
data_frame.to_excel(writer, sheet_name='Sheet1', header=False, index=False)
|
|
377
261
|
xlsx_data = output.getvalue()
|
|
378
262
|
return xlsx_data
|
|
263
|
+
|
|
264
|
+
@staticmethod
|
|
265
|
+
def csv_to_xls(file_data: bytes | str, delimiter: str = ",", newline: str = "\r\n") -> bytes:
|
|
266
|
+
"""
|
|
267
|
+
Convert the bytes or string of a .csv file to .xls bytes.
|
|
268
|
+
|
|
269
|
+
:param file_data: The .csv bytes or string to convert.
|
|
270
|
+
:param delimiter: The delimiter character separating columns, with "," being the default.
|
|
271
|
+
:param newline: The newline character(s) separating rows, with "\r\n" being the default.
|
|
272
|
+
:return: The bytes of the new .xls file.
|
|
273
|
+
"""
|
|
274
|
+
# Import the libraries we'll need locally since we won't be using them anywhere else in the class.
|
|
275
|
+
from xlwt import Workbook, Worksheet
|
|
276
|
+
|
|
277
|
+
# Create an Excel workbook along with a worksheet, which is where the data will be written.
|
|
278
|
+
workbook: Workbook = Workbook()
|
|
279
|
+
sheet: Worksheet = workbook.add_sheet("Sheet1")
|
|
280
|
+
|
|
281
|
+
# Make sure the file data is in a string format so that we can work with it.
|
|
282
|
+
formatted_data: str = bytes.decode(file_data, "utf-8") if isinstance(file_data, bytes) else file_data
|
|
283
|
+
|
|
284
|
+
# Write each row of the file to the .xls sheet.
|
|
285
|
+
rows: list[str] = formatted_data.split(newline)
|
|
286
|
+
for i, row in enumerate(rows):
|
|
287
|
+
values: list[str] = row.split(delimiter)
|
|
288
|
+
for j, value in enumerate(values):
|
|
289
|
+
sheet.write(i, j, value)
|
|
290
|
+
|
|
291
|
+
# Save the worksheet data to the byte buffer and return the bytes.
|
|
292
|
+
with io.BytesIO() as buffer:
|
|
293
|
+
workbook.save(buffer)
|
|
294
|
+
file_bytes: bytes = buffer.getvalue()
|
|
295
|
+
return file_bytes
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def zip_files(files: dict[str, str | bytes]) -> bytes:
|
|
299
|
+
"""
|
|
300
|
+
Create a zip file for a collection of files.
|
|
301
|
+
|
|
302
|
+
:param files: A dictionary of file name to file data as a string or bytes.
|
|
303
|
+
:return: The bytes for a zip file containing the input files.
|
|
304
|
+
"""
|
|
305
|
+
zip_buffer: io.BytesIO = io.BytesIO()
|
|
306
|
+
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
|
|
307
|
+
for file_name, file_data in files.items():
|
|
308
|
+
zip_file.writestr(file_name, file_data)
|
|
309
|
+
return zip_buffer.getvalue()
|
|
310
|
+
|
|
311
|
+
# Deprecated functions:
|
|
312
|
+
|
|
313
|
+
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
314
|
+
@staticmethod
|
|
315
|
+
def write_file(file_name: str, file_bytes: bytes, *, request_context: str | None = None) -> SapioWebhookResult:
|
|
316
|
+
"""
|
|
317
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
318
|
+
|
|
319
|
+
Send a file to the client.
|
|
320
|
+
|
|
321
|
+
The calling webhook must catch the WriteFileResult that the client will send back.
|
|
322
|
+
|
|
323
|
+
:param file_name: The name of the file.
|
|
324
|
+
:param file_bytes: The bytes of the file.
|
|
325
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
326
|
+
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
327
|
+
"""
|
|
328
|
+
warnings.warn("FileUtil.write_file is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
329
|
+
DeprecationWarning)
|
|
330
|
+
return SapioWebhookResult(True, client_callback_request=WriteFileRequest(file_bytes, file_name,
|
|
331
|
+
request_context))
|
|
332
|
+
|
|
333
|
+
@staticmethod
|
|
334
|
+
def write_files(files: dict[str, bytes], *, request_context: str | None = None) -> SapioWebhookResult:
|
|
335
|
+
"""
|
|
336
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
337
|
+
|
|
338
|
+
Send a collection of files to the client.
|
|
339
|
+
|
|
340
|
+
The calling webhook must catch the MultiFileResult that the client will send back.
|
|
341
|
+
|
|
342
|
+
:param files: A dictionary of files names to file bytes.
|
|
343
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
344
|
+
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
345
|
+
"""
|
|
346
|
+
warnings.warn("FileUtil.write_files is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
347
|
+
DeprecationWarning)
|
|
348
|
+
return SapioWebhookResult(True, client_callback_request=MultiFileRequest(files, request_context))
|
|
349
|
+
|
|
350
|
+
@staticmethod
|
|
351
|
+
def request_file(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
352
|
+
*, request_context: str | None = None) \
|
|
353
|
+
-> tuple[SapioWebhookResult | None, str | None, bytes | None]:
|
|
354
|
+
"""
|
|
355
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
356
|
+
|
|
357
|
+
Request a single file from the user. This function handles the entire client callback interaction for the
|
|
358
|
+
requesting of the file, including if the user cancels the file upload prompt.
|
|
359
|
+
|
|
360
|
+
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
361
|
+
result containing the client callback to request the file. The second time it is called, it will return the
|
|
362
|
+
file name and bytes from the callback result.
|
|
363
|
+
|
|
364
|
+
:param context: The current webhook context.
|
|
365
|
+
:param title: The title of the file prompt dialog.
|
|
366
|
+
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
367
|
+
exception if an incorrect file extension is provided.
|
|
368
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
369
|
+
:return: A tuple with the following elements.
|
|
370
|
+
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
371
|
+
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
372
|
+
1 - The file name of the requested file if the user provided one.
|
|
373
|
+
2 - The file bytes of the requested file if the user provided one.
|
|
374
|
+
"""
|
|
375
|
+
warnings.warn("FileUtil.request_file is deprecated as of 24.5+. Use CallbackUtil.request_file instead.",
|
|
376
|
+
DeprecationWarning)
|
|
377
|
+
client_callback = context.client_callback_result
|
|
378
|
+
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
379
|
+
# If the user cancels, terminate the interaction.
|
|
380
|
+
if client_callback is not None and client_callback.user_cancelled:
|
|
381
|
+
return SapioWebhookResult(True), None, None
|
|
382
|
+
# If no extensions were provided, use an empty list for the extensions instead.
|
|
383
|
+
if exts is None:
|
|
384
|
+
exts = []
|
|
385
|
+
|
|
386
|
+
# If the client callback isn't a FilePromptResult, then it's either None or some other callback result, meaning
|
|
387
|
+
# we need to send a new request. We may also send a new request if the client callback result is a
|
|
388
|
+
# FilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
389
|
+
# result from a different call to request_file.
|
|
390
|
+
is_file_result = isinstance(client_callback, FilePromptResult)
|
|
391
|
+
if not is_file_result or (is_file_result and result_context != request_context):
|
|
392
|
+
prompt = FilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
393
|
+
callback_context_data=request_context)
|
|
394
|
+
return SapioWebhookResult(True, client_callback_request=prompt), None, None
|
|
395
|
+
|
|
396
|
+
# Get the file from the result. Enforce that the provided data isn't empty, and that the file path ends in
|
|
397
|
+
# one of the allowed extensions.
|
|
398
|
+
# noinspection PyTypeChecker
|
|
399
|
+
result: FilePromptResult = client_callback
|
|
400
|
+
file_path: str | None = result.file_path
|
|
401
|
+
file_bytes: bytes | None = result.file_bytes
|
|
402
|
+
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
403
|
+
return None, file_path, file_bytes
|
|
404
|
+
|
|
405
|
+
@staticmethod
|
|
406
|
+
def request_files(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
407
|
+
*, request_context: str | None = None) \
|
|
408
|
+
-> tuple[SapioWebhookResult | None, dict[str, bytes] | None]:
|
|
409
|
+
"""
|
|
410
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
411
|
+
|
|
412
|
+
Request multiple files from the user. This function handles the entire client callback interaction for the
|
|
413
|
+
requesting of the files, including if the user cancels the file upload prompt.
|
|
414
|
+
|
|
415
|
+
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
416
|
+
result containing the client callback to request the files. The second time it is called, it will return each
|
|
417
|
+
file name and bytes from the callback result.
|
|
418
|
+
|
|
419
|
+
:param context: The current webhook context.
|
|
420
|
+
:param title: The title of the file prompt dialog.
|
|
421
|
+
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
422
|
+
exception if an incorrect file extension is provided.
|
|
423
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
424
|
+
:return: A tuple with the following elements.
|
|
425
|
+
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
426
|
+
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
427
|
+
1 - A dictionary that maps the file names to the file bytes for each provided file.
|
|
428
|
+
"""
|
|
429
|
+
warnings.warn("FileUtil.request_files is deprecated as of 24.5+. Use CallbackUtil.request_files instead.",
|
|
430
|
+
DeprecationWarning)
|
|
431
|
+
client_callback = context.client_callback_result
|
|
432
|
+
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
433
|
+
# If the user cancels, terminate the interaction.
|
|
434
|
+
if client_callback is not None and client_callback.user_cancelled:
|
|
435
|
+
return SapioWebhookResult(True), None
|
|
436
|
+
# If no extensions were provided, use an empty list for the extensions instead.
|
|
437
|
+
if exts is None:
|
|
438
|
+
exts = []
|
|
439
|
+
|
|
440
|
+
# If the client callback isn't a MultiFilePromptResult, then it's either None or some other callback result,
|
|
441
|
+
# meaning we need to send a new request. We may also send a new request if the client callback result is a
|
|
442
|
+
# MultiFilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
443
|
+
# result from a different call to request_file.
|
|
444
|
+
is_file_result = isinstance(client_callback, MultiFilePromptResult)
|
|
445
|
+
if not is_file_result or (is_file_result and result_context != request_context):
|
|
446
|
+
prompt = MultiFilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
447
|
+
callback_context_data=request_context)
|
|
448
|
+
return SapioWebhookResult(True, client_callback_request=prompt), None
|
|
449
|
+
|
|
450
|
+
# Get the files from the result. Enforce that the provided data isn't empty, and that the file paths end in
|
|
451
|
+
# one of the allowed extensions.
|
|
452
|
+
# noinspection PyTypeChecker
|
|
453
|
+
result: MultiFilePromptResult = client_callback
|
|
454
|
+
for file_path, file_bytes in result.files.items():
|
|
455
|
+
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
456
|
+
return None, result.files
|
|
457
|
+
|
|
458
|
+
@staticmethod
|
|
459
|
+
def __verify_file(file_path: str, file_bytes: bytes, allowed_extensions: list[str]):
|
|
460
|
+
"""
|
|
461
|
+
Verify that the provided file was read (i.e. the file path and file bytes aren't None or empty) and that it
|
|
462
|
+
has the correct file extension. Raises a user error exception if something about the file is incorrect.
|
|
463
|
+
|
|
464
|
+
:param file_path: The name of the file to verify.
|
|
465
|
+
:param file_bytes: The bytes of the file to verify.
|
|
466
|
+
:param allowed_extensions: The file extensions that the file path is allowed to have.
|
|
467
|
+
"""
|
|
468
|
+
if file_path is None or len(file_path) == 0 or file_bytes is None or len(file_bytes) == 0:
|
|
469
|
+
raise SapioUserErrorException("Empty file provided or file unable to be read.")
|
|
470
|
+
if len(allowed_extensions) != 0:
|
|
471
|
+
matches: bool = False
|
|
472
|
+
for ext in allowed_extensions:
|
|
473
|
+
if file_path.endswith("." + ext.lstrip(".")):
|
|
474
|
+
matches = True
|
|
475
|
+
break
|
|
476
|
+
if matches is False:
|
|
477
|
+
raise SapioUserErrorException("Unsupported file type. Expecting the following extension(s): "
|
|
478
|
+
+ (",".join(allowed_extensions)))
|