sapiopycommons 2024.8.15a304__py3-none-any.whl → 2024.8.19a305__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sapiopycommons might be problematic. Click here for more details.
- sapiopycommons/callbacks/callback_util.py +122 -25
- sapiopycommons/customreport/__init__.py +0 -0
- sapiopycommons/customreport/column_builder.py +60 -0
- sapiopycommons/customreport/custom_report_builder.py +125 -0
- sapiopycommons/customreport/term_builder.py +296 -0
- sapiopycommons/datatype/attachment_util.py +15 -6
- sapiopycommons/eln/experiment_handler.py +193 -39
- sapiopycommons/files/complex_data_loader.py +1 -1
- sapiopycommons/files/file_bridge.py +1 -1
- sapiopycommons/files/file_bridge_handler.py +21 -0
- sapiopycommons/files/file_util.py +38 -5
- sapiopycommons/files/file_validator.py +21 -6
- sapiopycommons/files/file_writer.py +44 -15
- sapiopycommons/general/aliases.py +93 -2
- sapiopycommons/general/audit_log.py +200 -0
- sapiopycommons/general/popup_util.py +17 -0
- sapiopycommons/general/sapio_links.py +48 -0
- sapiopycommons/general/time_util.py +40 -0
- sapiopycommons/recordmodel/record_handler.py +114 -17
- sapiopycommons/rules/eln_rule_handler.py +29 -22
- sapiopycommons/rules/on_save_rule_handler.py +29 -28
- sapiopycommons/webhook/webhook_handlers.py +90 -26
- sapiopycommons/webhook/webservice_handlers.py +67 -0
- {sapiopycommons-2024.8.15a304.dist-info → sapiopycommons-2024.8.19a305.dist-info}/METADATA +1 -1
- sapiopycommons-2024.8.19a305.dist-info/RECORD +50 -0
- sapiopycommons-2024.8.15a304.dist-info/RECORD +0 -43
- {sapiopycommons-2024.8.15a304.dist-info → sapiopycommons-2024.8.19a305.dist-info}/WHEEL +0 -0
- {sapiopycommons-2024.8.15a304.dist-info → sapiopycommons-2024.8.19a305.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import time
|
|
2
4
|
from collections.abc import Mapping, Iterable
|
|
5
|
+
from weakref import WeakValueDictionary
|
|
3
6
|
|
|
7
|
+
from sapiopylib.rest.DataMgmtService import DataMgmtServer
|
|
4
8
|
from sapiopylib.rest.ELNService import ElnManager
|
|
9
|
+
from sapiopylib.rest.User import SapioUser
|
|
5
10
|
from sapiopylib.rest.pojo.DataRecord import DataRecord
|
|
6
11
|
from sapiopylib.rest.pojo.eln.ElnExperiment import ElnExperiment, TemplateExperimentQueryPojo, ElnTemplate, \
|
|
7
12
|
InitializeNotebookExperimentPojo, ElnExperimentUpdateCriteria
|
|
@@ -16,8 +21,9 @@ from sapiopylib.rest.utils.Protocols import ElnEntryStep, ElnExperimentProtocol
|
|
|
16
21
|
from sapiopylib.rest.utils.recordmodel.PyRecordModel import PyRecordModel
|
|
17
22
|
from sapiopylib.rest.utils.recordmodel.RecordModelManager import RecordModelInstanceManager, RecordModelManager
|
|
18
23
|
from sapiopylib.rest.utils.recordmodel.RecordModelWrapper import WrappedType
|
|
24
|
+
from sapiopylib.rest.utils.recordmodel.properties import Child
|
|
19
25
|
|
|
20
|
-
from sapiopycommons.general.aliases import AliasUtil, SapioRecord
|
|
26
|
+
from sapiopycommons.general.aliases import AliasUtil, SapioRecord, ExperimentIdentifier, RecordModel
|
|
21
27
|
from sapiopycommons.general.exceptions import SapioException
|
|
22
28
|
|
|
23
29
|
Step = str | ElnEntryStep
|
|
@@ -27,7 +33,8 @@ itself."""
|
|
|
27
33
|
|
|
28
34
|
# FR-46064 - Initial port of PyWebhookUtils to sapiopycommons.
|
|
29
35
|
class ExperimentHandler:
|
|
30
|
-
|
|
36
|
+
user: SapioUser
|
|
37
|
+
context: SapioWebhookContext | None
|
|
31
38
|
"""The context that this handler is working from."""
|
|
32
39
|
|
|
33
40
|
# Basic experiment info from the context.
|
|
@@ -77,45 +84,107 @@ class ExperimentHandler:
|
|
|
77
84
|
ElnExperimentStatus.Canceled]
|
|
78
85
|
"""The set of statuses that an ELN experiment could have and be considered locked."""
|
|
79
86
|
|
|
80
|
-
|
|
87
|
+
__instances: WeakValueDictionary[str, ExperimentHandler] = WeakValueDictionary()
|
|
88
|
+
__initialized: bool
|
|
89
|
+
|
|
90
|
+
def __new__(cls, context: SapioWebhookContext | SapioUser,
|
|
91
|
+
experiment: ExperimentIdentifier | SapioRecord | None = None):
|
|
92
|
+
"""
|
|
93
|
+
:param context: The current webhook context or a user object to send requests from.
|
|
94
|
+
:param experiment: If an experiment is provided that is separate from the experiment that is in the context,
|
|
95
|
+
that experiment will be used by this ExperimentHandler instead. An experiment can be provided in various
|
|
96
|
+
forms, including an ElnExperiment, ElnExperimentProtocol, an experiment record, or a notebook experiment ID.
|
|
97
|
+
"""
|
|
98
|
+
param_results = cls.__parse_params(context, experiment)
|
|
99
|
+
user = param_results[0]
|
|
100
|
+
experiment = param_results[2]
|
|
101
|
+
key = f"{user.__hash__()}:{experiment.notebook_experiment_id}"
|
|
102
|
+
obj = cls.__instances.get(key)
|
|
103
|
+
if not obj:
|
|
104
|
+
obj = object.__new__(cls)
|
|
105
|
+
obj.__initialized = False
|
|
106
|
+
cls.__instances[key] = obj
|
|
107
|
+
return obj
|
|
108
|
+
|
|
109
|
+
def __init__(self, context: SapioWebhookContext | SapioUser,
|
|
110
|
+
experiment: ExperimentIdentifier | SapioRecord | None = None):
|
|
81
111
|
"""
|
|
82
112
|
Initialization will throw an exception if there is no ELN Experiment in the provided context and no experiment
|
|
83
113
|
is provided.
|
|
84
114
|
|
|
85
|
-
:param context: The current webhook context.
|
|
115
|
+
:param context: The current webhook context or a user object to send requests from.
|
|
86
116
|
:param experiment: If an experiment is provided that is separate from the experiment that is in the context,
|
|
87
|
-
that experiment will be used by this ExperimentHandler instead.
|
|
117
|
+
that experiment will be used by this ExperimentHandler instead. An experiment can be provided in various
|
|
118
|
+
forms, including an ElnExperiment, ElnExperimentProtocol, an experiment record, or a notebook experiment ID.
|
|
88
119
|
"""
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
if context.eln_experiment == experiment:
|
|
94
|
-
experiment = None
|
|
95
|
-
self.context = context
|
|
120
|
+
param_results = self.__parse_params(context, experiment)
|
|
121
|
+
self.user = param_results[0]
|
|
122
|
+
self.context = param_results[1]
|
|
123
|
+
experiment = param_results[2]
|
|
96
124
|
|
|
97
125
|
# Get the basic information about this experiment that already exists in the context and is often used.
|
|
98
|
-
self.__eln_exp = experiment
|
|
99
|
-
self.__protocol = ElnExperimentProtocol(experiment,
|
|
126
|
+
self.__eln_exp = experiment
|
|
127
|
+
self.__protocol = ElnExperimentProtocol(experiment, self.user)
|
|
100
128
|
self.__exp_id = self.__protocol.get_id()
|
|
101
129
|
|
|
102
130
|
# Grab various managers that may be used.
|
|
103
|
-
self.__eln_man =
|
|
104
|
-
self.__inst_man = RecordModelManager(
|
|
131
|
+
self.__eln_man = DataMgmtServer.get_eln_manager(self.user)
|
|
132
|
+
self.__inst_man = RecordModelManager(self.user).instance_manager
|
|
105
133
|
|
|
106
134
|
# Create empty caches to fill when necessary.
|
|
107
135
|
self.__steps = {}
|
|
108
136
|
self.__step_options = {}
|
|
109
137
|
# CR-46330: Cache any experiment entry information that might already exist in the context.
|
|
110
138
|
self.__queried_all_steps = False
|
|
111
|
-
# We can only trust the entries in the context if
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
139
|
+
# We can only trust the entries in the context if the experiment that this handler is for is the same as the
|
|
140
|
+
# one from the context.
|
|
141
|
+
if self.context is not None and self.context.eln_experiment == experiment:
|
|
142
|
+
if self.context.experiment_entry is not None:
|
|
143
|
+
self.__steps.update({self.context.active_step.get_name(): self.context.active_step})
|
|
144
|
+
if self.context.experiment_entry_list is not None:
|
|
145
|
+
for entry in self.context.experiment_entry_list:
|
|
117
146
|
self.__steps.update({entry.entry_name: ElnEntryStep(self.__protocol, entry)})
|
|
118
147
|
|
|
148
|
+
@staticmethod
|
|
149
|
+
def __parse_params(context: SapioWebhookContext | SapioUser,
|
|
150
|
+
experiment: ExperimentIdentifier | SapioRecord | None = None) \
|
|
151
|
+
-> tuple[SapioUser, SapioWebhookContext | None, ElnExperiment]:
|
|
152
|
+
if isinstance(context, SapioWebhookContext):
|
|
153
|
+
user = context.user
|
|
154
|
+
context = context
|
|
155
|
+
else:
|
|
156
|
+
user = context
|
|
157
|
+
context = None
|
|
158
|
+
if context is not None and context.eln_experiment is not None and experiment is None:
|
|
159
|
+
experiment = context.eln_experiment
|
|
160
|
+
# FR-46495 - Allow the init function of ExperimentHandler to take in an ElnExperiment that is separate from the
|
|
161
|
+
# context.
|
|
162
|
+
# CR-37038 - Allow other experiment object types to be provided. Convert them all down to ElnExperiment.
|
|
163
|
+
if (context is None or context.eln_experiment is None) and experiment is not None:
|
|
164
|
+
eln_manager = DataMgmtServer.get_eln_manager(user)
|
|
165
|
+
# If this object is already an ElnExperiment, do nothing.
|
|
166
|
+
if isinstance(experiment, ElnExperiment):
|
|
167
|
+
pass
|
|
168
|
+
# If this object is an ElnExperimentProtocol, then we can get the ElnExperiment from the object.
|
|
169
|
+
elif isinstance(experiment, ElnExperimentProtocol):
|
|
170
|
+
experiment: ElnExperiment = experiment.eln_experiment
|
|
171
|
+
# If this object is an integer, assume it is a notebook ID that we can query the system with.
|
|
172
|
+
elif isinstance(experiment, int):
|
|
173
|
+
notebook_id: int = experiment
|
|
174
|
+
experiment: ElnExperiment = eln_manager.get_eln_experiment_by_id(notebook_id)
|
|
175
|
+
if not experiment:
|
|
176
|
+
raise SapioException(f"No experiment with notebook ID {notebook_id} located in the system.")
|
|
177
|
+
# If this object is a record, assume it is an experiment record that we can query the system with.
|
|
178
|
+
else:
|
|
179
|
+
record_id: int = AliasUtil.to_record_ids([experiment])[0]
|
|
180
|
+
experiment: ElnExperiment = eln_manager.get_eln_experiment_by_record_id(record_id)
|
|
181
|
+
if not experiment:
|
|
182
|
+
raise SapioException(f"No experiment with record ID {record_id} located in the system.")
|
|
183
|
+
if experiment is None:
|
|
184
|
+
raise SapioException("Cannot initialize ExperimentHandler. No ELN Experiment found in the provided parameters.")
|
|
185
|
+
|
|
186
|
+
return user, context, experiment
|
|
187
|
+
|
|
119
188
|
# FR-46495: Split the creation of the experiment in launch_experiment into a create_experiment function.
|
|
120
189
|
@staticmethod
|
|
121
190
|
def create_experiment(context: SapioWebhookContext,
|
|
@@ -263,7 +332,7 @@ class ExperimentHandler:
|
|
|
263
332
|
:return: The data record for this experiment. None if it has no record.
|
|
264
333
|
"""
|
|
265
334
|
if not hasattr(self, "_ExperimentHandler__exp_record"):
|
|
266
|
-
drm = self.
|
|
335
|
+
drm = DataMgmtServer.get_data_record_manager(self.user)
|
|
267
336
|
dt = self.__eln_exp.experiment_data_type_name
|
|
268
337
|
results = drm.query_data_records_by_id(dt, [self.__eln_exp.experiment_record_id]).result_list
|
|
269
338
|
# PR-46504: Set the exp_record to None if there are no results.
|
|
@@ -580,8 +649,6 @@ class ExperimentHandler:
|
|
|
580
649
|
step.set_records(AliasUtil.to_data_records(records))
|
|
581
650
|
|
|
582
651
|
# FR-46496 - Provide alias of set_step_records for use with form entries.
|
|
583
|
-
# TODO: Provide a similar aliased function for attachment entries once sapiopylib allows setting multiple
|
|
584
|
-
# attachments to an attachment step.
|
|
585
652
|
def set_form_record(self, step: Step, record: SapioRecord) -> None:
|
|
586
653
|
"""
|
|
587
654
|
Sets the record for a form entry.
|
|
@@ -599,7 +666,8 @@ class ExperimentHandler:
|
|
|
599
666
|
self.set_step_records(step, [record])
|
|
600
667
|
|
|
601
668
|
# FR-46496 - Provide functions for adding and removing rows from an ELN data type entry.
|
|
602
|
-
def add_eln_rows(self, step: Step, count: int
|
|
669
|
+
def add_eln_rows(self, step: Step, count: int, wrapper_type: type[WrappedType] | None = None) \
|
|
670
|
+
-> list[PyRecordModel | WrappedType]:
|
|
603
671
|
"""
|
|
604
672
|
Add rows to an ELNExperimentDetail or ELNSampleDetail table entry. The rows will not appear in the system
|
|
605
673
|
until a record manager store and commit has occurred.
|
|
@@ -611,15 +679,37 @@ class ExperimentHandler:
|
|
|
611
679
|
The step may be provided as either a string for the name of the step or an ElnEntryStep.
|
|
612
680
|
If given a name, throws an exception if no step of the given name exists in the experiment.
|
|
613
681
|
:param count: The number of new rows to add to the entry.
|
|
682
|
+
:param wrapper_type: Optionally wrap the ELN data type in a record model wrapper. If not provided, returns
|
|
683
|
+
an unwrapped PyRecordModel.
|
|
614
684
|
:return: A list of the newly created rows.
|
|
615
685
|
"""
|
|
616
686
|
step = self.__to_eln_step(step)
|
|
617
687
|
if step.eln_entry.entry_type != ElnEntryType.Table:
|
|
618
688
|
raise SapioException("The provided step is not a table entry.")
|
|
619
689
|
dt: str = step.get_data_type_names()[0]
|
|
620
|
-
if not
|
|
690
|
+
if not ElnBaseDataType.is_eln_type(dt):
|
|
621
691
|
raise SapioException("The provided step is not an ELN data type entry.")
|
|
622
|
-
|
|
692
|
+
records: list[PyRecordModel] = self.__inst_man.add_new_records(dt, count)
|
|
693
|
+
if wrapper_type:
|
|
694
|
+
return self.__inst_man.wrap_list(records, wrapper_type)
|
|
695
|
+
return records
|
|
696
|
+
|
|
697
|
+
def add_eln_row(self, step: Step, wrapper_type: type[WrappedType] | None = None) -> PyRecordModel | WrappedType:
|
|
698
|
+
"""
|
|
699
|
+
Add a row to an ELNExperimentDetail or ELNSampleDetail table entry. The row will not appear in the system
|
|
700
|
+
until a record manager store and commit has occurred.
|
|
701
|
+
|
|
702
|
+
If no step functions have been called before and a step is being searched for by name, queries for the
|
|
703
|
+
list of steps in the experiment and caches them.
|
|
704
|
+
|
|
705
|
+
:param step:
|
|
706
|
+
The step may be provided as either a string for the name of the step or an ElnEntryStep.
|
|
707
|
+
If given a name, throws an exception if no step of the given name exists in the experiment.
|
|
708
|
+
:param wrapper_type: Optionally wrap the ELN data type in a record model wrapper. If not provided, returns
|
|
709
|
+
an unwrapped PyRecordModel.
|
|
710
|
+
:return: The newly created row.
|
|
711
|
+
"""
|
|
712
|
+
return self.add_eln_rows(step, 1, wrapper_type)[0]
|
|
623
713
|
|
|
624
714
|
def remove_eln_rows(self, step: Step, records: list[SapioRecord]) -> None:
|
|
625
715
|
"""
|
|
@@ -641,7 +731,7 @@ class ExperimentHandler:
|
|
|
641
731
|
"""
|
|
642
732
|
step = self.__to_eln_step(step)
|
|
643
733
|
dt: str = step.get_data_type_names()[0]
|
|
644
|
-
if not
|
|
734
|
+
if not ElnBaseDataType.is_eln_type(dt):
|
|
645
735
|
raise SapioException("The provided step is not an ELN data type entry.")
|
|
646
736
|
if any([x.data_type_name != dt for x in records]):
|
|
647
737
|
raise SapioException("Not all of the provided records match the data type of the step.")
|
|
@@ -658,16 +748,59 @@ class ExperimentHandler:
|
|
|
658
748
|
for record in record_models:
|
|
659
749
|
record.delete()
|
|
660
750
|
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
751
|
+
def remove_eln_row(self, step: Step, record: SapioRecord) -> None:
|
|
752
|
+
"""
|
|
753
|
+
Remove a row from an ELNExperimentDetail or ELNSampleDetail table entry. ELN data type table entries display all
|
|
754
|
+
records in the system that match the entry's data type. This means that removing rows from an ELN data type
|
|
755
|
+
table entry is equivalent to deleting the records for the rows.
|
|
756
|
+
|
|
757
|
+
The row will not be deleted in the system until a record manager store and commit has occurred.
|
|
758
|
+
|
|
759
|
+
If no step functions have been called before and a step is being searched for by name, queries for the
|
|
760
|
+
list of steps in the experiment and caches them.
|
|
761
|
+
|
|
762
|
+
:param step:
|
|
763
|
+
The step may be provided as either a string for the name of the step or an ElnEntryStep.
|
|
764
|
+
If given a name, throws an exception if no step of the given name exists in the experiment.
|
|
765
|
+
:param record:
|
|
766
|
+
The record to remove from the given step.
|
|
767
|
+
The record may be provided as either a DataRecord, PyRecordModel, or WrappedRecordModel.
|
|
768
|
+
"""
|
|
769
|
+
self.remove_eln_row(step, [record])
|
|
770
|
+
|
|
771
|
+
def add_sample_details(self, step: Step, samples: list[RecordModel], wrapper_type: type[WrappedType]) \
|
|
772
|
+
-> list[PyRecordModel | WrappedType]:
|
|
773
|
+
"""
|
|
774
|
+
Add sample details to a sample details entry while relating them to the input sample records.
|
|
775
|
+
|
|
776
|
+
:param step:
|
|
777
|
+
The step may be provided as either a string for the name of the step or an ElnEntryStep.
|
|
778
|
+
If given a name, throws an exception if no step of the given name exists in the experiment.
|
|
779
|
+
:param samples: The sample records to add the sample details to.
|
|
780
|
+
:param wrapper_type: Optionally wrap the sample details in a record model wrapper. If not provided, returns
|
|
781
|
+
an unwrapped PyRecordModel.
|
|
782
|
+
:return: The newly created sample details. The indices of the samples in the input list match the index of the
|
|
783
|
+
sample details in this list that they are related to.
|
|
784
|
+
"""
|
|
785
|
+
step = self.__to_eln_step(step)
|
|
786
|
+
if step.eln_entry.entry_type != ElnEntryType.Table:
|
|
787
|
+
raise SapioException("The provided step is not a table entry.")
|
|
788
|
+
dt: str = step.get_data_type_names()[0]
|
|
789
|
+
if not ElnBaseDataType.is_eln_type(dt) or ElnBaseDataType.get_base_type(dt) != ElnBaseDataType.SAMPLE_DETAIL:
|
|
790
|
+
raise SapioException("The provided step is not an ELNSampleDetail entry.")
|
|
791
|
+
records: list[PyRecordModel] = []
|
|
792
|
+
for sample in samples:
|
|
793
|
+
if sample.data_type_name != "Sample":
|
|
794
|
+
raise SapioException(f"Received a {sample.data_type_name} record when Sample records were expected.")
|
|
795
|
+
detail: PyRecordModel = sample.add(Child.of_type_name(dt))
|
|
796
|
+
detail.set_field_values({
|
|
797
|
+
"SampleId": sample.get_field_value("SampleId"),
|
|
798
|
+
"OtherSampleId": sample.get_field_value("OtherSampleId")
|
|
799
|
+
})
|
|
800
|
+
records.append(detail)
|
|
801
|
+
if wrapper_type:
|
|
802
|
+
return self.__inst_man.wrap_list(records, wrapper_type)
|
|
803
|
+
return records
|
|
671
804
|
|
|
672
805
|
def update_step(self, step: Step,
|
|
673
806
|
entry_name: str | None = None,
|
|
@@ -956,6 +1089,27 @@ class ExperimentHandler:
|
|
|
956
1089
|
step.unlock_step()
|
|
957
1090
|
step.eln_entry.entry_status = ExperimentEntryStatus.UnlockedChangesRequired
|
|
958
1091
|
|
|
1092
|
+
def disable_step(self, step: Step) -> None:
|
|
1093
|
+
"""
|
|
1094
|
+
Set the status of the input step to Disabled. This is the state that entries are in when they are waiting for
|
|
1095
|
+
entries that they are dependent upon to be submitted before they can be enabled. If you have unsubmitted an
|
|
1096
|
+
entry and want its dependent entries to be locked again, then you would use this to set their status to
|
|
1097
|
+
disabled.
|
|
1098
|
+
|
|
1099
|
+
Makes a webservice call to update the step. Checks if the step is already unlocked, and does nothing if so.
|
|
1100
|
+
|
|
1101
|
+
If no step functions have been called before and a step is being searched for by name, queries for the
|
|
1102
|
+
list of steps in the experiment and caches them.
|
|
1103
|
+
|
|
1104
|
+
:param step:
|
|
1105
|
+
The step to disable.
|
|
1106
|
+
The step may be provided as either a string for the name of the step or an ElnEntryStep.
|
|
1107
|
+
If given a name, throws an exception if no step of the given name exists in the experiment.
|
|
1108
|
+
"""
|
|
1109
|
+
step = self.__to_eln_step(step)
|
|
1110
|
+
if step.eln_entry.entry_status in self.__ENTRY_LOCKED_STATUSES:
|
|
1111
|
+
self.update_step(step, entry_status=ExperimentEntryStatus.Disabled)
|
|
1112
|
+
|
|
959
1113
|
def step_is_submitted(self, step: Step) -> bool:
|
|
960
1114
|
"""
|
|
961
1115
|
Determine if the input step has already been submitted.
|
|
@@ -23,7 +23,7 @@ class CDL:
|
|
|
23
23
|
"fileName": file_name
|
|
24
24
|
}
|
|
25
25
|
user: SapioUser = context if isinstance(context, SapioUser) else context.user
|
|
26
|
-
with io.
|
|
26
|
+
with io.BytesIO(file_data.encode() if isinstance(file_data, str) else file_data) as data_stream:
|
|
27
27
|
response = user.post_data_stream(sub_path, params=params, data_stream=data_stream)
|
|
28
28
|
user.raise_for_status(response)
|
|
29
29
|
# The response content is returned as bytes for a comma separated string of record IDs.
|
|
@@ -54,7 +54,7 @@ class FileBridge:
|
|
|
54
54
|
'Filepath': f"bridge://{bridge_name}/{file_path}"
|
|
55
55
|
}
|
|
56
56
|
user: SapioUser = context if isinstance(context, SapioUser) else context.user
|
|
57
|
-
with io.
|
|
57
|
+
with io.BytesIO(file_data.encode() if isinstance(file_data, str) else file_data) as data_stream:
|
|
58
58
|
response = user.post_data_stream(sub_path, params=params, data_stream=data_stream)
|
|
59
59
|
user.raise_for_status(response)
|
|
60
60
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from abc import abstractmethod, ABC
|
|
4
|
+
from weakref import WeakValueDictionary
|
|
4
5
|
|
|
5
6
|
from sapiopycommons.files.file_bridge import FileBridge
|
|
6
7
|
from sapiopylib.rest.User import SapioUser
|
|
@@ -23,12 +24,32 @@ class FileBridgeHandler:
|
|
|
23
24
|
__directories: dict[str, Directory]
|
|
24
25
|
"""A cache of directory file paths to Directory objects."""
|
|
25
26
|
|
|
27
|
+
__instances: WeakValueDictionary[str, FileBridgeHandler] = WeakValueDictionary()
|
|
28
|
+
__initialized: bool
|
|
29
|
+
|
|
30
|
+
def __new__(cls, context: SapioWebhookContext | SapioUser, bridge_name: str):
|
|
31
|
+
"""
|
|
32
|
+
:param context: The current webhook context or a user object to send requests from.
|
|
33
|
+
"""
|
|
34
|
+
user = context if isinstance(context, SapioUser) else context.user
|
|
35
|
+
key = f"{user.__hash__()}:{bridge_name}"
|
|
36
|
+
obj = cls.__instances.get(key)
|
|
37
|
+
if not obj:
|
|
38
|
+
obj = object.__new__(cls)
|
|
39
|
+
obj.__initialized = False
|
|
40
|
+
cls.__instances[key] = obj
|
|
41
|
+
return obj
|
|
42
|
+
|
|
26
43
|
def __init__(self, context: SapioWebhookContext | SapioUser, bridge_name: str):
|
|
27
44
|
"""
|
|
28
45
|
:param context: The current webhook context or a user object to send requests from.
|
|
29
46
|
:param bridge_name: The name of the bridge to communicate with. This is the "connection name" in the
|
|
30
47
|
file bridge configurations.
|
|
31
48
|
"""
|
|
49
|
+
if self.__initialized:
|
|
50
|
+
return
|
|
51
|
+
self.__initialized = True
|
|
52
|
+
|
|
32
53
|
self.user = context if isinstance(context, SapioUser) else context.user
|
|
33
54
|
self.__bridge = bridge_name
|
|
34
55
|
self.__file_cache = {}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import io
|
|
2
|
+
import warnings
|
|
3
|
+
import zipfile
|
|
2
4
|
|
|
3
5
|
import pandas
|
|
4
6
|
from numpy import dtype
|
|
@@ -21,7 +23,8 @@ class FileUtil:
|
|
|
21
23
|
"""
|
|
22
24
|
@staticmethod
|
|
23
25
|
def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
24
|
-
seperator: str = ",", *, encoding: str | None = None
|
|
26
|
+
seperator: str = ",", *, encoding: str | None = None, exception_on_empty: bool = True) \
|
|
27
|
+
-> tuple[list[dict[str, str]], list[list[str]]]:
|
|
25
28
|
"""
|
|
26
29
|
Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
|
|
27
30
|
must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
|
|
@@ -37,6 +40,8 @@ class FileUtil:
|
|
|
37
40
|
:param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
|
|
38
41
|
contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
|
|
39
42
|
ISO-8859-1 as the encoding.
|
|
43
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
44
|
+
the first element of the returned tuple.
|
|
40
45
|
:return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
41
46
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
42
47
|
If the header row index is 0 or None, this list will be empty.
|
|
@@ -49,11 +54,13 @@ class FileUtil:
|
|
|
49
54
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
50
55
|
# Parse the data from the file body into a list of dicts.
|
|
51
56
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
57
|
+
if exception_on_empty and not rows:
|
|
58
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
52
59
|
return rows, metadata
|
|
53
60
|
|
|
54
61
|
@staticmethod
|
|
55
|
-
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0
|
|
56
|
-
|
|
62
|
+
def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
|
|
63
|
+
*, exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
|
|
57
64
|
"""
|
|
58
65
|
Tokenize an XLSX file row by row.
|
|
59
66
|
|
|
@@ -64,6 +71,8 @@ class FileUtil:
|
|
|
64
71
|
row is returned in the metadata list. If input is None, then no row is considered to be the header row,
|
|
65
72
|
meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
|
|
66
73
|
is assumed to be the header row.
|
|
74
|
+
:param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
|
|
75
|
+
the first element of the returned tuple.
|
|
67
76
|
:return: The XLSX parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
|
|
68
77
|
that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
|
|
69
78
|
If the header row index is 0 or None, this list will be empty.
|
|
@@ -75,6 +84,8 @@ class FileUtil:
|
|
|
75
84
|
metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
|
|
76
85
|
# Parse the data from the file body into a list of dicts.
|
|
77
86
|
rows: list[dict[str, str]] = FileUtil.data_frame_to_dicts(file_body, required_headers, header_row_index)
|
|
87
|
+
if exception_on_empty and not rows:
|
|
88
|
+
raise SapioUserErrorException("The provided file contains no rows of information below the headers.")
|
|
78
89
|
return rows, metadata
|
|
79
90
|
|
|
80
91
|
@staticmethod
|
|
@@ -229,7 +240,7 @@ class FileUtil:
|
|
|
229
240
|
:param file_data: The CSV file to be converted.
|
|
230
241
|
:return: The bytes of the CSV file converted to an XLSX file.
|
|
231
242
|
"""
|
|
232
|
-
with (io.BytesIO(file_data) if isinstance(file_data,
|
|
243
|
+
with (io.BytesIO(file_data.encode() if isinstance(file_data, str) else file_data)) as csv:
|
|
233
244
|
# Setting header to false makes pandas read the CSV as-is.
|
|
234
245
|
data_frame = pandas.read_csv(csv, sep=",", header=None)
|
|
235
246
|
|
|
@@ -273,6 +284,20 @@ class FileUtil:
|
|
|
273
284
|
file_bytes: bytes = buffer.getvalue()
|
|
274
285
|
return file_bytes
|
|
275
286
|
|
|
287
|
+
@staticmethod
|
|
288
|
+
def zip_files(files: dict[str, str | bytes]) -> bytes:
|
|
289
|
+
"""
|
|
290
|
+
Create a zip file for a collection of files.
|
|
291
|
+
|
|
292
|
+
:param files: A dictionary of file name to file data as a string or bytes.
|
|
293
|
+
:return: The bytes for a zip file containing the input files.
|
|
294
|
+
"""
|
|
295
|
+
zip_buffer: io.BytesIO = io.BytesIO()
|
|
296
|
+
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
|
|
297
|
+
for file_name, file_data in files.items():
|
|
298
|
+
zip_file.writestr(file_name, file_data)
|
|
299
|
+
return zip_buffer.getvalue()
|
|
300
|
+
|
|
276
301
|
# Deprecated functions:
|
|
277
302
|
|
|
278
303
|
# FR-46097 - Add write file request shorthand functions to FileUtil.
|
|
@@ -290,6 +315,8 @@ class FileUtil:
|
|
|
290
315
|
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
291
316
|
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
292
317
|
"""
|
|
318
|
+
warnings.warn("FileUtil.write_file is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
319
|
+
DeprecationWarning)
|
|
293
320
|
return SapioWebhookResult(True, client_callback_request=WriteFileRequest(file_bytes, file_name,
|
|
294
321
|
request_context))
|
|
295
322
|
|
|
@@ -306,6 +333,8 @@ class FileUtil:
|
|
|
306
333
|
:param request_context: Context that will be returned to the webhook server in the client callback result.
|
|
307
334
|
:return: A SapioWebhookResult with the write request as its client callback request.
|
|
308
335
|
"""
|
|
336
|
+
warnings.warn("FileUtil.write_files is deprecated as of 24.5+. Use CallbackUtil.write_file instead.",
|
|
337
|
+
DeprecationWarning)
|
|
309
338
|
return SapioWebhookResult(True, client_callback_request=MultiFileRequest(files, request_context))
|
|
310
339
|
|
|
311
340
|
@staticmethod
|
|
@@ -333,6 +362,8 @@ class FileUtil:
|
|
|
333
362
|
1 - The file name of the requested file if the user provided one.
|
|
334
363
|
2 - The file bytes of the requested file if the user provided one.
|
|
335
364
|
"""
|
|
365
|
+
warnings.warn("FileUtil.request_file is deprecated as of 24.5+. Use CallbackUtil.request_file instead.",
|
|
366
|
+
DeprecationWarning)
|
|
336
367
|
client_callback = context.client_callback_result
|
|
337
368
|
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
338
369
|
# If the user cancels, terminate the interaction.
|
|
@@ -385,6 +416,8 @@ class FileUtil:
|
|
|
385
416
|
May also contain a result that will terminate the client interaction if the user canceled the prompt.
|
|
386
417
|
1 - A dictionary that maps the file names to the file bytes for each provided file.
|
|
387
418
|
"""
|
|
419
|
+
warnings.warn("FileUtil.request_files is deprecated as of 24.5+. Use CallbackUtil.request_files instead.",
|
|
420
|
+
DeprecationWarning)
|
|
388
421
|
client_callback = context.client_callback_result
|
|
389
422
|
result_context: str | None = client_callback.callback_context_data if client_callback else None
|
|
390
423
|
# If the user cancels, terminate the interaction.
|
|
@@ -427,7 +460,7 @@ class FileUtil:
|
|
|
427
460
|
if len(allowed_extensions) != 0:
|
|
428
461
|
matches: bool = False
|
|
429
462
|
for ext in allowed_extensions:
|
|
430
|
-
if file_path.endswith("." + ext):
|
|
463
|
+
if file_path.endswith("." + ext.lstrip(".")):
|
|
431
464
|
matches = True
|
|
432
465
|
break
|
|
433
466
|
if matches is False:
|
|
@@ -8,11 +8,11 @@ from sapiopylib.rest.pojo.CustomReport import RawReportTerm, RawTermOperation
|
|
|
8
8
|
from sapiopylib.rest.pojo.datatype.FieldDefinition import VeloxIntegerFieldDefinition, VeloxStringFieldDefinition, \
|
|
9
9
|
AbstractVeloxFieldDefinition
|
|
10
10
|
from sapiopylib.rest.pojo.webhook.WebhookContext import SapioWebhookContext
|
|
11
|
-
from sapiopylib.rest.pojo.webhook.WebhookResult import SapioWebhookResult
|
|
12
11
|
|
|
13
12
|
from sapiopycommons.callbacks.callback_util import CallbackUtil
|
|
14
13
|
from sapiopycommons.files.file_data_handler import FileDataHandler, FilterList
|
|
15
14
|
from sapiopycommons.general.custom_report_util import CustomReportUtil
|
|
15
|
+
from sapiopycommons.general.exceptions import SapioUserCancelledException
|
|
16
16
|
from sapiopycommons.general.time_util import TimeUtil
|
|
17
17
|
|
|
18
18
|
|
|
@@ -80,10 +80,10 @@ class FileValidator:
|
|
|
80
80
|
|
|
81
81
|
return failed_rows
|
|
82
82
|
|
|
83
|
-
def build_violation_report(self, context:
|
|
83
|
+
def build_violation_report(self, context: SapioWebhookContext | SapioUser,
|
|
84
84
|
rule_violations: dict[int, list[ValidationRule]]) -> None:
|
|
85
85
|
"""
|
|
86
|
-
|
|
86
|
+
Display a simple report of any rule violations in the file to the user as a table dialog.
|
|
87
87
|
|
|
88
88
|
:param context: The current webhook context or a user object to send requests from.
|
|
89
89
|
:param rule_violations: A dict of rule violations generated by a call to validate_file.
|
|
@@ -121,9 +121,24 @@ class FileValidator:
|
|
|
121
121
|
"Reason": violation.reason[:2000]
|
|
122
122
|
})
|
|
123
123
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
callback = CallbackUtil(context)
|
|
125
|
+
callback.table_dialog("Errors", "The following rule violations were encountered in the provided file.",
|
|
126
|
+
columns, rows)
|
|
127
|
+
|
|
128
|
+
def validate_and_report_errors(self, context: SapioWebhookContext | SapioUser) -> None:
|
|
129
|
+
"""
|
|
130
|
+
Validate the file. If any rule violations are found, display a simple report of any rule violations in the file
|
|
131
|
+
to the user as a table dialog and throw a SapioUserCancelled exception after the user acknowledges the dialog
|
|
132
|
+
to end the webhook interaction.
|
|
133
|
+
|
|
134
|
+
Shorthand for calling validate_file() and then build_violation_report() if there are any errors.
|
|
135
|
+
|
|
136
|
+
:param context: The current webhook context or a user object to send requests from.
|
|
137
|
+
"""
|
|
138
|
+
violations = self.validate_file()
|
|
139
|
+
if violations:
|
|
140
|
+
self.build_violation_report(context, violations)
|
|
141
|
+
raise SapioUserCancelledException()
|
|
127
142
|
|
|
128
143
|
|
|
129
144
|
class ValidationRule:
|