sapiopycommons 2024.3.18a156__py3-none-any.whl → 2025.1.17a402__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sapiopycommons might be problematic. Click here for more details.
- sapiopycommons/callbacks/__init__.py +0 -0
- sapiopycommons/callbacks/callback_util.py +2041 -0
- sapiopycommons/callbacks/field_builder.py +545 -0
- sapiopycommons/chem/IndigoMolecules.py +52 -5
- sapiopycommons/chem/Molecules.py +114 -30
- sapiopycommons/customreport/__init__.py +0 -0
- sapiopycommons/customreport/column_builder.py +60 -0
- sapiopycommons/customreport/custom_report_builder.py +137 -0
- sapiopycommons/customreport/term_builder.py +315 -0
- sapiopycommons/datatype/attachment_util.py +17 -15
- sapiopycommons/datatype/data_fields.py +61 -0
- sapiopycommons/datatype/pseudo_data_types.py +440 -0
- sapiopycommons/eln/experiment_handler.py +390 -90
- sapiopycommons/eln/experiment_report_util.py +649 -0
- sapiopycommons/eln/plate_designer.py +152 -0
- sapiopycommons/files/complex_data_loader.py +31 -0
- sapiopycommons/files/file_bridge.py +153 -25
- sapiopycommons/files/file_bridge_handler.py +555 -0
- sapiopycommons/files/file_data_handler.py +633 -0
- sapiopycommons/files/file_util.py +270 -158
- sapiopycommons/files/file_validator.py +569 -0
- sapiopycommons/files/file_writer.py +377 -0
- sapiopycommons/flowcyto/flow_cyto.py +77 -0
- sapiopycommons/flowcyto/flowcyto_data.py +75 -0
- sapiopycommons/general/accession_service.py +375 -0
- sapiopycommons/general/aliases.py +259 -18
- sapiopycommons/general/audit_log.py +185 -0
- sapiopycommons/general/custom_report_util.py +252 -31
- sapiopycommons/general/directive_util.py +86 -0
- sapiopycommons/general/exceptions.py +69 -7
- sapiopycommons/general/popup_util.py +85 -18
- sapiopycommons/general/sapio_links.py +50 -0
- sapiopycommons/general/storage_util.py +148 -0
- sapiopycommons/general/time_util.py +97 -7
- sapiopycommons/multimodal/multimodal.py +146 -0
- sapiopycommons/multimodal/multimodal_data.py +490 -0
- sapiopycommons/processtracking/__init__.py +0 -0
- sapiopycommons/processtracking/custom_workflow_handler.py +406 -0
- sapiopycommons/processtracking/endpoints.py +192 -0
- sapiopycommons/recordmodel/record_handler.py +653 -149
- sapiopycommons/rules/eln_rule_handler.py +89 -8
- sapiopycommons/rules/on_save_rule_handler.py +89 -12
- sapiopycommons/sftpconnect/__init__.py +0 -0
- sapiopycommons/sftpconnect/sftp_builder.py +70 -0
- sapiopycommons/webhook/webhook_context.py +39 -0
- sapiopycommons/webhook/webhook_handlers.py +617 -69
- sapiopycommons/webhook/webservice_handlers.py +317 -0
- {sapiopycommons-2024.3.18a156.dist-info → sapiopycommons-2025.1.17a402.dist-info}/METADATA +5 -4
- sapiopycommons-2025.1.17a402.dist-info/RECORD +60 -0
- {sapiopycommons-2024.3.18a156.dist-info → sapiopycommons-2025.1.17a402.dist-info}/WHEEL +1 -1
- sapiopycommons-2024.3.18a156.dist-info/RECORD +0 -28
- {sapiopycommons-2024.3.18a156.dist-info → sapiopycommons-2025.1.17a402.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import io
|
|
2
|
+
import warnings
|
|
3
|
+
import zipfile
|
|
2
4
|
|
|
3
5
|
import pandas
|
|
4
6
|
from numpy import dtype
|
|
@@ -13,163 +15,20 @@ from sapiopycommons.general.exceptions import SapioUserErrorException
|
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
# FR-46064 - Initial port of PyWebhookUtils to sapiopycommons.
|
|
18
|
+
# FR-46716 - Add comments noting that some functions are deprecated in favor of CallbackUtil.
|
|
16
19
|
class FileUtil:
|
|
17
20
|
"""
|
|
18
21
|
Utilities for the handling of files, including the requesting of files from the user and the parsing of files into
|
|
19
22
|
tokenized lists. Makes use of Pandas DataFrames for any file parsing purposes.
|
|
20
23
|
"""
|
|
21
|
-
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
22
|
-
@staticmethod
|
|
23
|
-
def write_file(file_name: str, file_bytes: bytes, *, request_context: str | None = None) -> SapioWebhookResult:
|
|
24
|
-
"""
|
|
25
|
-
Send a file to the client.
|
|
26
|
-
|
|
27
|
-
The calling webhook must catch the WriteFileResult that the client will send back.
|
|
28
|
-
:param file_name: The name of the file.
|
|
29
|
-
:param file_bytes: The bytes of the file.
|
|
30
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
31
|
-
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
32
|
-
"""
|
|
33
|
-
return SapioWebhookResult(True, client_callback_request=WriteFileRequest(file_bytes, file_name,
|
|
34
|
-
request_context))
|
|
35
|
-
|
|
36
|
-
@staticmethod
|
|
37
|
-
def write_files(files: dict[str, bytes], *, request_context: str | None = None) -> SapioWebhookResult:
|
|
38
|
-
"""
|
|
39
|
-
Send a collection of files to the client.
|
|
40
|
-
|
|
41
|
-
The calling webhook must catch the MultiFileResult that the client will send back.
|
|
42
|
-
:param files: A dictionary of files names to file bytes.
|
|
43
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
44
|
-
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
45
|
-
"""
|
|
46
|
-
return SapioWebhookResult(True, client_callback_request=MultiFileRequest(files, request_context))
|
|
47
|
-
|
|
48
|
-
@staticmethod
|
|
49
|
-
def request_file(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
50
|
-
*, request_context: str | None = None) \
|
|
51
|
-
-> tuple[SapioWebhookResult | None, str | None, bytes | None]:
|
|
52
|
-
"""
|
|
53
|
-
Request a single file from the user. This function handles the entire client callback interaction for the
|
|
54
|
-
requesting of the file, including if the user cancels the file upload prompt.
|
|
55
|
-
|
|
56
|
-
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
57
|
-
result containing the client callback to request the file. The second time it is called, it will return the
|
|
58
|
-
file name and bytes from the callback result.
|
|
59
|
-
:param context: The current webhook context.
|
|
60
|
-
:param title: The title of the file prompt dialog.
|
|
61
|
-
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
62
|
-
exception if an incorrect file extension is provided.
|
|
63
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
64
|
-
:return: A tuple with the following elements.
|
|
65
|
-
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
66
|
-
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
67
|
-
1 - The file name of the requested file if the user provided one.
|
|
68
|
-
2 - The file bytes of the requested file if the user provided one.
|
|
69
|
-
"""
|
|
70
|
-
client_callback = context.client_callback_result
|
|
71
|
-
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
72
|
-
# If the user cancels, terminate the interaction.
|
|
73
|
-
if client_callback is not None and client_callback.user_cancelled:
|
|
74
|
-
return SapioWebhookResult(True), None, None
|
|
75
|
-
# If no extensions were provided, use an empty list for the extensions instead.
|
|
76
|
-
if exts is None:
|
|
77
|
-
exts = []
|
|
78
|
-
|
|
79
|
-
# If the client callback isn't a FilePromptResult, then it's either None or some other callback result, meaning
|
|
80
|
-
# we need to send a new request. We may also send a new request if the client callback result is a
|
|
81
|
-
# FilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
82
|
-
# result from a different call to request_file.
|
|
83
|
-
is_file_result = isinstance(client_callback, FilePromptResult)
|
|
84
|
-
if not is_file_result or (is_file_result and result_context != request_context):
|
|
85
|
-
prompt = FilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
86
|
-
callback_context_data=request_context)
|
|
87
|
-
return SapioWebhookResult(True, client_callback_request=prompt), None, None
|
|
88
|
-
|
|
89
|
-
# Get the file from the result. Enforce that the provided data isn't empty, and that the file path ends in
|
|
90
|
-
# one of the allowed extensions.
|
|
91
|
-
# noinspection PyTypeChecker
|
|
92
|
-
result: FilePromptResult = client_callback
|
|
93
|
-
file_path: str | None = result.file_path
|
|
94
|
-
file_bytes: bytes | None = result.file_bytes
|
|
95
|
-
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
96
|
-
return None, file_path, file_bytes
|
|
97
|
-
|
|
98
24
|
@staticmethod
|
|
99
|
-
def
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
"""
|
|
103
|
-
Request multiple files from the user. This function handles the entire client callback interaction for the
|
|
104
|
-
requesting of the files, including if the user cancels the file upload prompt.
|
|
105
|
-
|
|
106
|
-
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
107
|
-
result containing the client callback to request the files. The second time it is called, it will return each
|
|
108
|
-
file name and bytes from the callback result.
|
|
109
|
-
:param context: The current webhook context.
|
|
110
|
-
:param title: The title of the file prompt dialog.
|
|
111
|
-
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
112
|
-
exception if an incorrect file extension is provided.
|
|
113
|
-
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
114
|
-
:return: A tuple with the following elements.
|
|
115
|
-
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
116
|
-
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
117
|
-
1 - A dictionary that maps the file names to the file bytes for each provided file.
|
|
118
|
-
"""
|
|
119
|
-
client_callback = context.client_callback_result
|
|
120
|
-
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
121
|
-
# If the user cancels, terminate the interaction.
|
|
122
|
-
if client_callback is not None and client_callback.user_cancelled:
|
|
123
|
-
return SapioWebhookResult(True), None
|
|
124
|
-
# If no extensions were provided, use an empty list for the extensions instead.
|
|
125
|
-
if exts is None:
|
|
126
|
-
exts = []
|
|
127
|
-
|
|
128
|
-
# If the client callback isn't a MultiFilePromptResult, then it's either None or some other callback result,
|
|
129
|
-
# meaning we need to send a new request. We may also send a new request if the client callback result is a
|
|
130
|
-
# MultiFilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
131
|
-
# result from a different call to request_file.
|
|
132
|
-
is_file_result = isinstance(client_callback, MultiFilePromptResult)
|
|
133
|
-
if not is_file_result or (is_file_result and result_context != request_context):
|
|
134
|
-
prompt = MultiFilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
135
|
-
callback_context_data=request_context)
|
|
136
|
-
return SapioWebhookResult(True, client_callback_request=prompt), None
|
|
137
|
-
|
|
138
|
-
# Get the files from the result. Enforce that the provided data isn't empty, and that the file paths end in
|
|
139
|
-
# one of the allowed extensions.
|
|
140
|
-
# noinspection PyTypeChecker
|
|
141
|
-
result: MultiFilePromptResult = client_callback
|
|
142
|
-
for file_path, file_bytes in result.files.items():
|
|
143
|
-
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
144
|
-
return None, result.files
|
|
145
|
-
|
|
146
|
-
@staticmethod
|
|
147
|
-
def __verify_file(file_path: str, file_bytes: bytes, allowed_extensions: list[str]):
|
|
148
|
-
"""
|
|
149
|
-
Verify that the provided file was read (i.e. the file path and file bytes aren't None or empty) and that it
|
|
150
|
-
has the correct file extension. Raises a user error exception if something about the file is incorrect.
|
|
151
|
-
:param file_path: The name of the file to verify.
|
|
152
|
-
:param file_bytes: The bytes of the file to verify.
|
|
153
|
-
:param allowed_extensions: The file extensions that the file path is allowed to have.
|
|
154
|
-
"""
|
|
155
|
-
if file_path is None or len(file_path) == 0 or file_bytes is None or len(file_bytes) == 0:
|
|
156
|
-
raise SapioUserErrorException("Empty file provided or file unable to be read.")
|
|
157
|
-
if len(allowed_extensions) != 0:
|
|
158
|
-
matches: bool = False
|
|
159
|
-
for ext in allowed_extensions:
|
|
160
|
-
if file_path.endswith("." + ext):
|
|
161
|
-
matches = True
|
|
162
|
-
break
|
|
163
|
-
if matches is False:
|
|
164
|
-
raise SapioUserErrorException("Unsupported file type. Expecting the following extension(s): "
|
|
165
|
-
+ (",".join(allowed_extensions)))
|
|
166
|
-
|
|
167
|
-
@staticmethod
|
|
168
|
-
def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0) \
|
|
169
|
-
-> tuple[list[dict[str, str]], list[list[str]]]:
|
|
25
|
+
def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
26
|
+
seperator: str = ",", *, encoding: str | None = None, encoding_error: str | None = "strict",
|
|
27
|
+
exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
|
|
170
28
|
"""
|
|
171
29
|
Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
|
|
172
30
|
must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
|
|
31
|
+
|
|
173
32
|
:param file_bytes: The bytes of the CSV to be parsed.
|
|
174
33
|
:param required_headers: The headers that must be present in the file. If a provided header is missing, raises
|
|
175
34
|
a user error exception.
|
|
@@ -177,24 +36,38 @@ class FileUtil:
|
|
|
177
36
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
178
37
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
179
38
|
is assumed to be the header row.
|
|
39
|
+
:param seperator: The character that separates cells in the table.
|
|
40
|
+
:param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
|
|
41
|
+
contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
|
|
42
|
+
ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
|
|
43
|
+
:param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
|
|
44
|
+
is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
|
|
45
|
+
characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
|
|
46
|
+
https://docs.python.org/3/library/codecs.html#error-handlers
|
|
47
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
48
|
+
the first element of the returned tuple.
|
|
180
49
|
:return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
181
50
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
182
51
|
If the header row index is 0 or None, this list will be empty.
|
|
183
52
|
"""
|
|
184
53
|
# Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
|
|
185
54
|
# while the second is the body of the file below the header row.
|
|
186
|
-
file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index
|
|
55
|
+
file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
|
|
56
|
+
encoding=encoding, encoding_error=encoding_error)
|
|
187
57
|
# Parse the metadata from above the header row index into a list of lists.
|
|
188
58
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
189
59
|
# Parse the data from the file body into a list of dicts.
|
|
190
60
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
61
|
+
if exception_on_empty and not rows:
|
|
62
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
191
63
|
return rows, metadata
|
|
192
64
|
|
|
193
65
|
@staticmethod
|
|
194
|
-
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0
|
|
195
|
-
|
|
66
|
+
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
67
|
+
*, exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
|
|
196
68
|
"""
|
|
197
69
|
Tokenize an XLSX file row by row.
|
|
70
|
+
|
|
198
71
|
:param file_bytes: The bytes of the XLSX to be parsed.
|
|
199
72
|
:param required_headers: The headers that must be present in the file. If a provided header is missing, raises
|
|
200
73
|
a user error exception.
|
|
@@ -202,6 +75,8 @@ class FileUtil:
|
|
|
202
75
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
203
76
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
204
77
|
is assumed to be the header row.
|
|
78
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
79
|
+
the first element of the returned tuple.
|
|
205
80
|
:return: The XLSX parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
206
81
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
207
82
|
If the header row index is 0 or None, this list will be empty.
|
|
@@ -213,19 +88,31 @@ class FileUtil:
|
|
|
213
88
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
214
89
|
# Parse the data from the file body into a list of dicts.
|
|
215
90
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
91
|
+
if exception_on_empty and not rows:
|
|
92
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
216
93
|
return rows, metadata
|
|
217
94
|
|
|
218
95
|
@staticmethod
|
|
219
|
-
def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0
|
|
96
|
+
def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
|
|
97
|
+
*, encoding: str | None = None, encoding_error: str | None = "strict") \
|
|
220
98
|
-> tuple[DataFrame, DataFrame | None]:
|
|
221
99
|
"""
|
|
222
100
|
Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
|
|
223
101
|
cells, all the rows in the file must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
|
|
102
|
+
|
|
224
103
|
:param file_bytes: The bytes of the CSV to be parsed.
|
|
225
104
|
:param header_row_index: The row index in the file that the headers are located at. Everything above the header
|
|
226
105
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
227
106
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
228
107
|
is assumed to be the header row.
|
|
108
|
+
:param seperator: The character that separates cells in the table.
|
|
109
|
+
:param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
|
|
110
|
+
contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
|
|
111
|
+
ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
|
|
112
|
+
:param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
|
|
113
|
+
is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
|
|
114
|
+
characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
|
|
115
|
+
https://docs.python.org/3/library/codecs.html#error-handlers
|
|
229
116
|
:return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
|
|
230
117
|
metadata from above the header row, or None if there is no metadata.
|
|
231
118
|
"""
|
|
@@ -237,13 +124,14 @@ class FileUtil:
|
|
|
237
124
|
# can throw off the header row index.
|
|
238
125
|
file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
|
|
239
126
|
skiprows=lambda x: x >= header_row_index,
|
|
240
|
-
skip_blank_lines=False
|
|
127
|
+
skip_blank_lines=False, sep=seperator, encoding=encoding,
|
|
128
|
+
encoding_errors=encoding_error)
|
|
241
129
|
with io.BytesIO(file_bytes) as file_io:
|
|
242
130
|
# The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
|
|
243
131
|
# because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
|
|
244
132
|
# string.
|
|
245
133
|
file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
|
|
246
|
-
skip_blank_lines=False)
|
|
134
|
+
skip_blank_lines=False, sep=seperator, encoding=encoding)
|
|
247
135
|
|
|
248
136
|
return file_body, file_metadata
|
|
249
137
|
|
|
@@ -252,6 +140,7 @@ class FileUtil:
|
|
|
252
140
|
-> tuple[DataFrame, DataFrame | None]:
|
|
253
141
|
"""
|
|
254
142
|
Parse the file bytes for an XLSX into DataFrames.
|
|
143
|
+
|
|
255
144
|
:param file_bytes: The bytes of the XLSX to be parsed.
|
|
256
145
|
:param header_row_index: The row index in the file that the headers are located at. Everything above the header
|
|
257
146
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
@@ -280,6 +169,7 @@ class FileUtil:
|
|
|
280
169
|
"""
|
|
281
170
|
Parse a Pandas DataFrame to a list of lists. Each outer list is a row, while each inner list is the cells
|
|
282
171
|
for that row.
|
|
172
|
+
|
|
283
173
|
:param data_frame: The DataFrame to be parsed.
|
|
284
174
|
:return: The input DataFrame parsed into a row-wise list of lists.
|
|
285
175
|
"""
|
|
@@ -310,6 +200,7 @@ class FileUtil:
|
|
|
310
200
|
keyed by the column header that the cell is under. Capable of requiring that certain headers are present.
|
|
311
201
|
|
|
312
202
|
The names of the data_frame.columns values are expected to be the header names.
|
|
203
|
+
|
|
313
204
|
:param data_frame: The DataFrame to be parsed.
|
|
314
205
|
:param required_headers: The headers that must be present in the DataFrame. If a header is missing, raises an
|
|
315
206
|
exception. If no headers are provided, doesn't do any enforcement.
|
|
@@ -339,11 +230,15 @@ class FileUtil:
|
|
|
339
230
|
|
|
340
231
|
# Warn about improper headers. Confirm that the header contains each of the header column names that we want.
|
|
341
232
|
if required_headers is not None:
|
|
233
|
+
# FR-46702: Report all missing headers instead of only the first missing header.
|
|
234
|
+
missing_headers: list[str] = []
|
|
342
235
|
for required in required_headers:
|
|
343
236
|
if required not in witnessed_headers:
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
237
|
+
missing_headers.append("\"" + required + "\"")
|
|
238
|
+
if missing_headers:
|
|
239
|
+
at_row = " at row " + str(header_row_index + 1) if header_row_index is not None else ""
|
|
240
|
+
raise SapioUserErrorException(f"Incorrect file headers or incorrectly formatted table. Header(s) "
|
|
241
|
+
f"{', '.join(missing_headers)} not found{at_row}.")
|
|
347
242
|
|
|
348
243
|
return rows
|
|
349
244
|
|
|
@@ -351,10 +246,11 @@ class FileUtil:
|
|
|
351
246
|
def csv_to_xlsx(file_data: bytes | str) -> bytes:
|
|
352
247
|
"""
|
|
353
248
|
Convert a CSV file into an XLSX file.
|
|
249
|
+
|
|
354
250
|
:param file_data: The CSV file to be converted.
|
|
355
251
|
:return: The bytes of the CSV file converted to an XLSX file.
|
|
356
252
|
"""
|
|
357
|
-
with (io.BytesIO(file_data) if isinstance(file_data,
|
|
253
|
+
with (io.BytesIO(file_data.encode() if isinstance(file_data, str) else file_data)) as csv:
|
|
358
254
|
# Setting header to false makes pandas read the CSV as-is.
|
|
359
255
|
data_frame = pandas.read_csv(csv, sep=",", header=None)
|
|
360
256
|
|
|
@@ -364,3 +260,219 @@ class FileUtil:
|
|
|
364
260
|
data_frame.to_excel(writer, sheet_name='Sheet1', header=False, index=False)
|
|
365
261
|
xlsx_data = output.getvalue()
|
|
366
262
|
return xlsx_data
|
|
263
|
+
|
|
264
|
+
@staticmethod
|
|
265
|
+
def csv_to_xls(file_data: bytes | str, delimiter: str = ",", newline: str = "\r\n") -> bytes:
|
|
266
|
+
"""
|
|
267
|
+
Convert the bytes or string of a .csv file to .xls bytes.
|
|
268
|
+
|
|
269
|
+
:param file_data: The .csv bytes or string to convert.
|
|
270
|
+
:param delimiter: The delimiter character separating columns, with "," being the default.
|
|
271
|
+
:param newline: The newline character(s) separating rows, with "\r\n" being the default.
|
|
272
|
+
:return: The bytes of the new .xls file.
|
|
273
|
+
"""
|
|
274
|
+
# Import the libraries we'll need locally since we won't be using them anywhere else in the class.
|
|
275
|
+
from xlwt import Workbook, Worksheet
|
|
276
|
+
|
|
277
|
+
# Create an Excel workbook along with a worksheet, which is where the data will be written.
|
|
278
|
+
workbook: Workbook = Workbook()
|
|
279
|
+
sheet: Worksheet = workbook.add_sheet("Sheet1")
|
|
280
|
+
|
|
281
|
+
# Make sure the file data is in a string format so that we can work with it.
|
|
282
|
+
formatted_data: str = bytes.decode(file_data, "utf-8") if isinstance(file_data, bytes) else file_data
|
|
283
|
+
|
|
284
|
+
# Write each row of the file to the .xls sheet.
|
|
285
|
+
rows: list[str] = formatted_data.split(newline)
|
|
286
|
+
for i, row in enumerate(rows):
|
|
287
|
+
values: list[str] = row.split(delimiter)
|
|
288
|
+
for j, value in enumerate(values):
|
|
289
|
+
sheet.write(i, j, value)
|
|
290
|
+
|
|
291
|
+
# Save the worksheet data to the byte buffer and return the bytes.
|
|
292
|
+
with io.BytesIO() as buffer:
|
|
293
|
+
workbook.save(buffer)
|
|
294
|
+
file_bytes: bytes = buffer.getvalue()
|
|
295
|
+
return file_bytes
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def zip_files(files: dict[str, str | bytes]) -> bytes:
|
|
299
|
+
"""
|
|
300
|
+
Create a zip file for a collection of files.
|
|
301
|
+
|
|
302
|
+
:param files: A dictionary of file name to file data as a string or bytes.
|
|
303
|
+
:return: The bytes for a zip file containing the input files.
|
|
304
|
+
"""
|
|
305
|
+
zip_buffer: io.BytesIO = io.BytesIO()
|
|
306
|
+
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
|
|
307
|
+
for file_name, file_data in files.items():
|
|
308
|
+
zip_file.writestr(file_name, file_data)
|
|
309
|
+
return zip_buffer.getvalue()
|
|
310
|
+
|
|
311
|
+
# Deprecated functions:
|
|
312
|
+
|
|
313
|
+
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
314
|
+
@staticmethod
|
|
315
|
+
def write_file(file_name: str, file_bytes: bytes, *, request_context: str | None = None) -> SapioWebhookResult:
|
|
316
|
+
"""
|
|
317
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
318
|
+
|
|
319
|
+
Send a file to the client.
|
|
320
|
+
|
|
321
|
+
The calling webhook must catch the WriteFileResult that the client will send back.
|
|
322
|
+
|
|
323
|
+
:param file_name: The name of the file.
|
|
324
|
+
:param file_bytes: The bytes of the file.
|
|
325
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
326
|
+
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
327
|
+
"""
|
|
328
|
+
warnings.warn("FileUtil.write_file is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
329
|
+
DeprecationWarning)
|
|
330
|
+
return SapioWebhookResult(True, client_callback_request=WriteFileRequest(file_bytes, file_name,
|
|
331
|
+
request_context))
|
|
332
|
+
|
|
333
|
+
@staticmethod
|
|
334
|
+
def write_files(files: dict[str, bytes], *, request_context: str | None = None) -> SapioWebhookResult:
|
|
335
|
+
"""
|
|
336
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
337
|
+
|
|
338
|
+
Send a collection of files to the client.
|
|
339
|
+
|
|
340
|
+
The calling webhook must catch the MultiFileResult that the client will send back.
|
|
341
|
+
|
|
342
|
+
:param files: A dictionary of files names to file bytes.
|
|
343
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
344
|
+
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
345
|
+
"""
|
|
346
|
+
warnings.warn("FileUtil.write_files is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
347
|
+
DeprecationWarning)
|
|
348
|
+
return SapioWebhookResult(True, client_callback_request=MultiFileRequest(files, request_context))
|
|
349
|
+
|
|
350
|
+
@staticmethod
|
|
351
|
+
def request_file(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
352
|
+
*, request_context: str | None = None) \
|
|
353
|
+
-> tuple[SapioWebhookResult | None, str | None, bytes | None]:
|
|
354
|
+
"""
|
|
355
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
356
|
+
|
|
357
|
+
Request a single file from the user. This function handles the entire client callback interaction for the
|
|
358
|
+
requesting of the file, including if the user cancels the file upload prompt.
|
|
359
|
+
|
|
360
|
+
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
361
|
+
result containing the client callback to request the file. The second time it is called, it will return the
|
|
362
|
+
file name and bytes from the callback result.
|
|
363
|
+
|
|
364
|
+
:param context: The current webhook context.
|
|
365
|
+
:param title: The title of the file prompt dialog.
|
|
366
|
+
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
367
|
+
exception if an incorrect file extension is provided.
|
|
368
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
369
|
+
:return: A tuple with the following elements.
|
|
370
|
+
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
371
|
+
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
372
|
+
1 - The file name of the requested file if the user provided one.
|
|
373
|
+
2 - The file bytes of the requested file if the user provided one.
|
|
374
|
+
"""
|
|
375
|
+
warnings.warn("FileUtil.request_file is deprecated as of 24.5+. Use CallbackUtil.request_file instead.",
|
|
376
|
+
DeprecationWarning)
|
|
377
|
+
client_callback = context.client_callback_result
|
|
378
|
+
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
379
|
+
# If the user cancels, terminate the interaction.
|
|
380
|
+
if client_callback is not None and client_callback.user_cancelled:
|
|
381
|
+
return SapioWebhookResult(True), None, None
|
|
382
|
+
# If no extensions were provided, use an empty list for the extensions instead.
|
|
383
|
+
if exts is None:
|
|
384
|
+
exts = []
|
|
385
|
+
|
|
386
|
+
# If the client callback isn't a FilePromptResult, then it's either None or some other callback result, meaning
|
|
387
|
+
# we need to send a new request. We may also send a new request if the client callback result is a
|
|
388
|
+
# FilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
389
|
+
# result from a different call to request_file.
|
|
390
|
+
is_file_result = isinstance(client_callback, FilePromptResult)
|
|
391
|
+
if not is_file_result or (is_file_result and result_context != request_context):
|
|
392
|
+
prompt = FilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
393
|
+
callback_context_data=request_context)
|
|
394
|
+
return SapioWebhookResult(True, client_callback_request=prompt), None, None
|
|
395
|
+
|
|
396
|
+
# Get the file from the result. Enforce that the provided data isn't empty, and that the file path ends in
|
|
397
|
+
# one of the allowed extensions.
|
|
398
|
+
# noinspection PyTypeChecker
|
|
399
|
+
result: FilePromptResult = client_callback
|
|
400
|
+
file_path: str | None = result.file_path
|
|
401
|
+
file_bytes: bytes | None = result.file_bytes
|
|
402
|
+
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
403
|
+
return None, file_path, file_bytes
|
|
404
|
+
|
|
405
|
+
@staticmethod
|
|
406
|
+
def request_files(context: SapioWebhookContext, title: str, exts: list[str] = None,
|
|
407
|
+
*, request_context: str | None = None) \
|
|
408
|
+
-> tuple[SapioWebhookResult | None, dict[str, bytes] | None]:
|
|
409
|
+
"""
|
|
410
|
+
DEPRECATED: Make use of CallbackUtil as of 24.5.
|
|
411
|
+
|
|
412
|
+
Request multiple files from the user. This function handles the entire client callback interaction for the
|
|
413
|
+
requesting of the files, including if the user cancels the file upload prompt.
|
|
414
|
+
|
|
415
|
+
The first time this method is called in the course of an interaction with the client, it will return a webhook
|
|
416
|
+
result containing the client callback to request the files. The second time it is called, it will return each
|
|
417
|
+
file name and bytes from the callback result.
|
|
418
|
+
|
|
419
|
+
:param context: The current webhook context.
|
|
420
|
+
:param title: The title of the file prompt dialog.
|
|
421
|
+
:param exts: The allowable file extensions of the uploaded file. If blank, any file can be uploaded. Throws an
|
|
422
|
+
exception if an incorrect file extension is provided.
|
|
423
|
+
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
424
|
+
:return: A tuple with the following elements.
|
|
425
|
+
0 - A webhook result that contains a file prompt if this is the first interaction with this request.
|
|
426
|
+
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
427
|
+
1 - A dictionary that maps the file names to the file bytes for each provided file.
|
|
428
|
+
"""
|
|
429
|
+
warnings.warn("FileUtil.request_files is deprecated as of 24.5+. Use CallbackUtil.request_files instead.",
|
|
430
|
+
DeprecationWarning)
|
|
431
|
+
client_callback = context.client_callback_result
|
|
432
|
+
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
433
|
+
# If the user cancels, terminate the interaction.
|
|
434
|
+
if client_callback is not None and client_callback.user_cancelled:
|
|
435
|
+
return SapioWebhookResult(True), None
|
|
436
|
+
# If no extensions were provided, use an empty list for the extensions instead.
|
|
437
|
+
if exts is None:
|
|
438
|
+
exts = []
|
|
439
|
+
|
|
440
|
+
# If the client callback isn't a MultiFilePromptResult, then it's either None or some other callback result,
|
|
441
|
+
# meaning we need to send a new request. We may also send a new request if the client callback result is a
|
|
442
|
+
# MultiFilePromptResult, but its callback context doesn't match the provided callback context, meaning it's a
|
|
443
|
+
# result from a different call to request_file.
|
|
444
|
+
is_file_result = isinstance(client_callback, MultiFilePromptResult)
|
|
445
|
+
if not is_file_result or (is_file_result and result_context != request_context):
|
|
446
|
+
prompt = MultiFilePromptRequest(dialog_title=title, file_extension=",".join(exts),
|
|
447
|
+
callback_context_data=request_context)
|
|
448
|
+
return SapioWebhookResult(True, client_callback_request=prompt), None
|
|
449
|
+
|
|
450
|
+
# Get the files from the result. Enforce that the provided data isn't empty, and that the file paths end in
|
|
451
|
+
# one of the allowed extensions.
|
|
452
|
+
# noinspection PyTypeChecker
|
|
453
|
+
result: MultiFilePromptResult = client_callback
|
|
454
|
+
for file_path, file_bytes in result.files.items():
|
|
455
|
+
FileUtil.__verify_file(file_path, file_bytes, exts)
|
|
456
|
+
return None, result.files
|
|
457
|
+
|
|
458
|
+
@staticmethod
|
|
459
|
+
def __verify_file(file_path: str, file_bytes: bytes, allowed_extensions: list[str]):
|
|
460
|
+
"""
|
|
461
|
+
Verify that the provided file was read (i.e. the file path and file bytes aren't None or empty) and that it
|
|
462
|
+
has the correct file extension. Raises a user error exception if something about the file is incorrect.
|
|
463
|
+
|
|
464
|
+
:param file_path: The name of the file to verify.
|
|
465
|
+
:param file_bytes: The bytes of the file to verify.
|
|
466
|
+
:param allowed_extensions: The file extensions that the file path is allowed to have.
|
|
467
|
+
"""
|
|
468
|
+
if file_path is None or len(file_path) == 0 or file_bytes is None or len(file_bytes) == 0:
|
|
469
|
+
raise SapioUserErrorException("Empty file provided or file unable to be read.")
|
|
470
|
+
if len(allowed_extensions) != 0:
|
|
471
|
+
matches: bool = False
|
|
472
|
+
for ext in allowed_extensions:
|
|
473
|
+
if file_path.endswith("." + ext.lstrip(".")):
|
|
474
|
+
matches = True
|
|
475
|
+
break
|
|
476
|
+
if matches is False:
|
|
477
|
+
raise SapioUserErrorException("Unsupported file type. Expecting the following extension(s): "
|
|
478
|
+
+ (",".join(allowed_extensions)))
|