sapiopycommons 2025.2.6a421__tar.gz → 2025.2.11a427__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sapiopycommons might be problematic. Click here for more details.

Files changed (83) hide show
  1. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/PKG-INFO +1 -1
  2. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/pyproject.toml +1 -1
  3. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/elain/tool_of_tools.py +60 -35
  4. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_util.py +36 -11
  5. sapiopycommons-2025.2.11a427/src/sapiopycommons/samples/aliquot.py +48 -0
  6. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/webhook/webhook_handlers.py +2 -2
  7. sapiopycommons-2025.2.11a427/tests/aliquot_test.py +47 -0
  8. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/.gitignore +0 -0
  9. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/LICENSE +0 -0
  10. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/README.md +0 -0
  11. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/__init__.py +0 -0
  12. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/callbacks/__init__.py +0 -0
  13. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/callbacks/callback_util.py +0 -0
  14. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/callbacks/field_builder.py +0 -0
  15. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/chem/IndigoMolecules.py +0 -0
  16. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/chem/Molecules.py +0 -0
  17. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/chem/__init__.py +0 -0
  18. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/customreport/__init__.py +0 -0
  19. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/customreport/auto_pagers.py +0 -0
  20. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/customreport/column_builder.py +0 -0
  21. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/customreport/custom_report_builder.py +0 -0
  22. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/customreport/term_builder.py +0 -0
  23. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/datatype/__init__.py +0 -0
  24. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/datatype/attachment_util.py +0 -0
  25. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/datatype/data_fields.py +0 -0
  26. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/datatype/pseudo_data_types.py +0 -0
  27. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/elain/__init__.py +0 -0
  28. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/eln/__init__.py +0 -0
  29. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/eln/experiment_handler.py +0 -0
  30. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/eln/experiment_report_util.py +0 -0
  31. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/eln/plate_designer.py +0 -0
  32. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/__init__.py +0 -0
  33. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/complex_data_loader.py +0 -0
  34. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_bridge.py +0 -0
  35. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_bridge_handler.py +0 -0
  36. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_data_handler.py +0 -0
  37. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_validator.py +0 -0
  38. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/files/file_writer.py +0 -0
  39. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/flowcyto/flow_cyto.py +0 -0
  40. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/flowcyto/flowcyto_data.py +0 -0
  41. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/__init__.py +0 -0
  42. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/accession_service.py +0 -0
  43. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/aliases.py +0 -0
  44. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/audit_log.py +0 -0
  45. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/custom_report_util.py +0 -0
  46. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/directive_util.py +0 -0
  47. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/exceptions.py +0 -0
  48. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/popup_util.py +0 -0
  49. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/sapio_links.py +0 -0
  50. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/storage_util.py +0 -0
  51. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/general/time_util.py +0 -0
  52. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/multimodal/multimodal.py +0 -0
  53. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/multimodal/multimodal_data.py +0 -0
  54. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/processtracking/__init__.py +0 -0
  55. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/processtracking/custom_workflow_handler.py +0 -0
  56. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/processtracking/endpoints.py +0 -0
  57. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/recordmodel/__init__.py +0 -0
  58. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/recordmodel/record_handler.py +0 -0
  59. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/rules/__init__.py +0 -0
  60. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/rules/eln_rule_handler.py +0 -0
  61. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/rules/on_save_rule_handler.py +0 -0
  62. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/sftpconnect/__init__.py +0 -0
  63. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/sftpconnect/sftp_builder.py +0 -0
  64. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/webhook/__init__.py +0 -0
  65. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/webhook/webhook_context.py +0 -0
  66. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/src/sapiopycommons/webhook/webservice_handlers.py +0 -0
  67. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/AF-A0A009IHW8-F1-model_v4.cif +0 -0
  68. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/_do_not_add_init_py_here +0 -0
  69. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/accession_test.py +0 -0
  70. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/bio_reg_test.py +0 -0
  71. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/chem_test.py +0 -0
  72. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/chem_test_curation_queue.py +0 -0
  73. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/curation_queue_test.sdf +0 -0
  74. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/data_type_models.py +0 -0
  75. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto/101_DEN084Y5_15_E01_008_clean.fcs +0 -0
  76. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto/101_DEN084Y5_15_E03_009_clean.fcs +0 -0
  77. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto/101_DEN084Y5_15_E05_010_clean.fcs +0 -0
  78. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto/8_color_ICS.wsp +0 -0
  79. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto/COVID19_W_001_O.fcs +0 -0
  80. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/flowcyto_test.py +0 -0
  81. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/kappa.chains.fasta +0 -0
  82. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/mafft_test.py +0 -0
  83. {sapiopycommons-2025.2.6a421 → sapiopycommons-2025.2.11a427}/tests/test.gb +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: sapiopycommons
3
- Version: 2025.2.6a421
3
+ Version: 2025.2.11a427
4
4
  Summary: Official Sapio Python API Utilities Package
5
5
  Project-URL: Homepage, https://github.com/sapiosciences
6
6
  Author-email: Jonathan Steck <jsteck@sapiosciences.com>, Yechen Qiao <yqiao@sapiosciences.com>
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "sapiopycommons"
7
- version='2025.02.06a421'
7
+ version='2025.02.11a427'
8
8
  authors = [
9
9
  { name="Jonathan Steck", email="jsteck@sapiosciences.com" },
10
10
  { name="Yechen Qiao", email="yqiao@sapiosciences.com" },
@@ -8,13 +8,13 @@ from sapiopylib.rest.ELNService import ElnManager
8
8
  from sapiopylib.rest.User import SapioUser
9
9
  from sapiopylib.rest.pojo.DataRecord import DataRecord
10
10
  from sapiopylib.rest.pojo.chartdata.DashboardDefinition import GaugeChartDefinition
11
- from sapiopylib.rest.pojo.chartdata.DashboardEnums import ChartGroupingType, ChartOperationType, ChartType
11
+ from sapiopylib.rest.pojo.chartdata.DashboardEnums import ChartGroupingType, ChartOperationType
12
12
  from sapiopylib.rest.pojo.chartdata.DashboardSeries import GaugeChartSeries
13
13
  from sapiopylib.rest.pojo.datatype.FieldDefinition import AbstractVeloxFieldDefinition, FieldType
14
14
  from sapiopylib.rest.pojo.eln.ElnExperiment import ElnExperiment
15
15
  from sapiopylib.rest.pojo.eln.ExperimentEntry import ExperimentEntry
16
16
  from sapiopylib.rest.pojo.eln.ExperimentEntryCriteria import ElnEntryCriteria, ElnFormEntryUpdateCriteria, \
17
- ElnDashboardEntryUpdateCriteria, AbstractElnEntryUpdateCriteria
17
+ ElnDashboardEntryUpdateCriteria, ElnTextEntryUpdateCriteria
18
18
  from sapiopylib.rest.pojo.eln.SapioELNEnums import ElnEntryType, ElnBaseDataType
19
19
  from sapiopylib.rest.pojo.eln.eln_headings import ElnExperimentTabAddCriteria, ElnExperimentTab
20
20
  from sapiopylib.rest.pojo.eln.field_set import ElnFieldSetInfo
@@ -22,6 +22,7 @@ from sapiopylib.rest.utils.ProtocolUtils import ELNStepFactory
22
22
  from sapiopylib.rest.utils.Protocols import ElnEntryStep, ElnExperimentProtocol
23
23
 
24
24
  from sapiopycommons.callbacks.field_builder import FieldBuilder
25
+ from sapiopycommons.general.aliases import AliasUtil, SapioRecord
25
26
  from sapiopycommons.general.exceptions import SapioException
26
27
  from sapiopycommons.general.time_util import TimeUtil
27
28
 
@@ -126,33 +127,28 @@ def create_experiment_details_from_json(user: SapioUser,
126
127
  fb = FieldBuilder()
127
128
  fields: list[AbstractVeloxFieldDefinition] = []
128
129
  fields_by_name: dict[str, AbstractVeloxFieldDefinition] = {}
129
- valid_keys: list[str] = []
130
- display_to_field_name: dict[str, str] = {}
131
130
  for key, value in json_list[0].items():
132
131
  field_name: str = key.replace(" ", "_")
133
- display_to_field_name[key] = field_name
134
132
  if isinstance(value, str):
135
133
  field = fb.string_field(field_name, display_name=key)
136
134
  fields.append(field)
137
- fields_by_name[field_name] = field
138
- valid_keys.append(key)
135
+ fields_by_name[key] = field
139
136
  elif isinstance(value, (int, float)):
140
137
  field = fb.double_field(field_name, display_name=key, precision=3)
141
138
  fields.append(field)
142
- fields_by_name[field_name] = field
143
- valid_keys.append(key)
139
+ fields_by_name[key] = field
144
140
 
145
141
  # Extract the valid field values from the JSON.
146
142
  field_maps: list[dict[str, Any]] = []
147
143
  for json_dict in json_list:
148
144
  field_map: dict[str, Any] = {}
149
- for key in valid_keys:
145
+ for key, field in fields_by_name.items():
150
146
  # Watch out for NaN values or other special values.
151
147
  val: Any = json_dict.get(key)
152
- if (fields_by_name[key].data_field_type == FieldType.DOUBLE
148
+ if (field.data_field_type == FieldType.DOUBLE
153
149
  and (not isinstance(val, (int, float))) or (isinstance(val, float) and math.isnan(val))):
154
150
  val = None
155
- field_map[display_to_field_name[key]] = val
151
+ field_map[field.data_field_name] = val
156
152
  field_maps.append(field_map)
157
153
 
158
154
  detail_entry = ElnEntryCriteria(ElnEntryType.Table, entry_name,
@@ -172,6 +168,43 @@ def tab_next_entry_order(user: SapioUser, exp_id: int, tab: ElnExperimentTab) ->
172
168
  return max_order + 1
173
169
 
174
170
 
171
+ def set_text_entry(timestamp: str, description: str, text_entry: ExperimentEntry, exp_id: int, user: SapioUser) -> None:
172
+ """
173
+ Set the text of a text entry.
174
+
175
+ :param timestamp: The timestamp to display at the top of the text entry.
176
+ :param description: The description to display in the text entry.
177
+ :param text_entry: The text entry to set the text of.
178
+ :param exp_id: The ID of the experiment that the text entry is in.
179
+ :param user: The user to send the request from.
180
+ """
181
+ description: str = f"""<p><span style="color: rgb(35, 111, 161); font-size: 12pt; font-weight:500;">{timestamp}</span>
182
+ <br><span style="font-size: 15pt;">{description}</span></p>"""
183
+ protocol = ElnExperimentProtocol(ElnExperiment(exp_id, "", 0), user)
184
+ step = ElnEntryStep(protocol, text_entry)
185
+ text_record: DataRecord = step.get_records()[0]
186
+ text_record.set_field_value(ElnBaseDataType.get_text_entry_data_field_name(), description)
187
+ DataRecordManager(user).commit_data_records([text_record])
188
+
189
+
190
+ def add_to_text_entry(description: str, text_entry: ExperimentEntry, exp_id: int, user: SapioUser) -> None:
191
+ """
192
+ Add to the text of a text entry.
193
+
194
+ :param description: The text to add to the text entry.
195
+ :param text_entry: The text entry to add the text to.
196
+ :param exp_id: The ID of the experiment that the text entry is in.
197
+ :param user: The user to send the request from.
198
+ """
199
+ protocol = ElnExperimentProtocol(ElnExperiment(exp_id, "", 0), user)
200
+ step = ElnEntryStep(protocol, text_entry)
201
+ text_record: DataRecord = step.get_records()[0]
202
+ update: str = text_record.get_field_value(ElnBaseDataType.get_text_entry_data_field_name())
203
+ update += f"""<p style="padding-top: 10px;"><span style="font-size: 15pt;">{description}</span></p>"""
204
+ text_record.set_field_value(ElnBaseDataType.get_text_entry_data_field_name(), update)
205
+ DataRecordManager(user).commit_data_records([text_record])
206
+
207
+
175
208
  class ToolOfToolsHelper:
176
209
  """
177
210
  A class with helper methods utilized by the Tool of Tools for the creation and updating of experiment tabs that
@@ -320,7 +353,7 @@ class ToolOfToolsHelper:
320
353
  self.description_record = text_entry.get_records()[0]
321
354
 
322
355
  # Shrink the text entry by one column.
323
- text_update_crit = _ElnTextEntryUpdateCriteria()
356
+ text_update_crit = ElnTextEntryUpdateCriteria()
324
357
  text_update_crit.column_span = 2
325
358
  self.eln_man.update_experiment_entry(self.exp_id, self.description_entry.get_id(), text_update_crit)
326
359
 
@@ -380,6 +413,16 @@ class ToolOfToolsHelper:
380
413
  self.progress_record.set_field_value("StatusMsg", status_msg)
381
414
  self.dr_man.commit_data_records([self.progress_record])
382
415
 
416
+ def add_results(self, results: list[SapioRecord]) -> None:
417
+ """
418
+ Add the results of the tool to the results entry.
419
+
420
+ :param results: The result records to add to the results entry.
421
+ """
422
+ if not self._initialized:
423
+ raise SapioException("The tab for this tool has not been initialized.")
424
+ self.results_entry.add_records(AliasUtil.to_data_records(results))
425
+
383
426
  def add_results_bar_chart(self, x_axis: str, y_axis: str) -> ExperimentEntry:
384
427
  """
385
428
  Create a bar chart entry for the results of the tool.
@@ -451,7 +494,7 @@ class ToolOfToolsHelper:
451
494
  file_contents: bytes = f.read()
452
495
  return self.add_attachment_entry(file_path, file_contents, entry_name, tab)
453
496
 
454
- # TODO: Remove this once pylib has a gauge chart function in ElnStepFactory.
497
+ # TODO: Remove this once pylib's gauge chart definition is up to date.
455
498
  @staticmethod
456
499
  def _create_gauge_chart(protocol: ElnExperimentProtocol, data_source_step: ElnEntryStep, step_name: str,
457
500
  field_name: str, status_field: str, group_by_field_name: str = "DataRecordName") \
@@ -473,19 +516,16 @@ class ToolOfToolsHelper:
473
516
  chart.grouping_type = ChartGroupingType.GROUP_BY_FIELD
474
517
  chart.grouping_type_data_type_name = data_type_name
475
518
  chart.grouping_type_data_field_name = group_by_field_name
476
- dashboard, step = ELNStepFactory._create_dashboard_step_from_chart(chart, data_source_step, protocol, step_name)
519
+ dashboard, step = ELNStepFactory._create_dashboard_step_from_chart(chart, data_source_step, protocol, step_name,
520
+ None)
477
521
  protocol.invalidate()
478
522
  return step
479
523
 
480
524
 
481
- # TODO: This is only here because the get_chart_type function in pylib is wrong. Remove this once pylib is fixed.
482
- # Also using this to set the new status field setting.
525
+ # TODO: Using this to set the new status field setting.
483
526
  class _GaugeChartDefinition(GaugeChartDefinition):
484
527
  status_field: str
485
528
 
486
- def get_chart_type(self) -> ChartType:
487
- return ChartType.GAUGE_CHART
488
-
489
529
  def to_json(self) -> dict[str, Any]:
490
530
  result = super().to_json()
491
531
  result["statusValueField"] = {
@@ -493,18 +533,3 @@ class _GaugeChartDefinition(GaugeChartDefinition):
493
533
  "dataFieldName": self.status_field
494
534
  }
495
535
  return result
496
-
497
-
498
- # TODO: Remove once the ElnTextEntryUpdateCriteria is fixed.
499
- class _ElnTextEntryUpdateCriteria(AbstractElnEntryUpdateCriteria):
500
- """
501
- Text Entry Update Data Payload
502
- Create this payload object and set the attributes you want to update before sending the request.
503
- """
504
-
505
- def __init__(self):
506
- super().__init__(ElnEntryType.Text)
507
-
508
- def to_json(self) -> dict[str, Any]:
509
- ret = super().to_json()
510
- return ret
@@ -21,10 +21,14 @@ class FileUtil:
21
21
  Utilities for the handling of files, including the requesting of files from the user and the parsing of files into
22
22
  tokenized lists. Makes use of Pandas DataFrames for any file parsing purposes.
23
23
  """
24
+ # PR-47433: Add a keep_default_na argument to FileUtil.tokenize_csv and FileUtil.tokenize_xlsx so that N/A values
25
+ # don't get returned as NoneType, and add **kwargs in case any other Pandas input parameters need changed by the
26
+ # caller.
24
27
  @staticmethod
25
28
  def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
26
29
  seperator: str = ",", *, encoding: str | None = None, encoding_error: str | None = "strict",
27
- exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
30
+ exception_on_empty: bool = True, keep_default_na: bool = False, **kwargs) \
31
+ -> tuple[list[dict[str, str]], list[list[str]]]:
28
32
  """
29
33
  Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
30
34
  must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -46,6 +50,9 @@ class FileUtil:
46
50
  https://docs.python.org/3/library/codecs.html#error-handlers
47
51
  :param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
48
52
  the first element of the returned tuple.
53
+ :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
54
+ If True, these values will be converted to a NoneType value.
55
+ :param kwargs: Additional arguments to be passed to the pandas read_csv function.
49
56
  :return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
50
57
  that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
51
58
  If the header row index is 0 or None, this list will be empty.
@@ -53,7 +60,8 @@ class FileUtil:
53
60
  # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
54
61
  # while the second is the body of the file below the header row.
55
62
  file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
56
- encoding=encoding, encoding_error=encoding_error)
63
+ encoding=encoding, encoding_error=encoding_error,
64
+ keep_default_na=keep_default_na, **kwargs)
57
65
  # Parse the metadata from above the header row index into a list of lists.
58
66
  metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
59
67
  # Parse the data from the file body into a list of dicts.
@@ -64,7 +72,8 @@ class FileUtil:
64
72
 
65
73
  @staticmethod
66
74
  def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
67
- *, exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
75
+ *, exception_on_empty: bool = True, keep_default_na: bool = False, **kwargs) \
76
+ -> tuple[list[dict[str, str]], list[list[str]]]:
68
77
  """
69
78
  Tokenize an XLSX file row by row.
70
79
 
@@ -77,13 +86,17 @@ class FileUtil:
77
86
  is assumed to be the header row.
78
87
  :param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
79
88
  the first element of the returned tuple.
89
+ :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
90
+ If True, these values will be converted to a NoneType value.
91
+ :param kwargs: Additional arguments to be passed to the pandas read_excel function.
80
92
  :return: The XLSX parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
81
93
  that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
82
94
  If the header row index is 0 or None, this list will be empty.
83
95
  """
84
96
  # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
85
97
  # while the second is the body of the file below the header row.
86
- file_body, file_metadata = FileUtil.xlsx_to_data_frames(file_bytes, header_row_index)
98
+ file_body, file_metadata = FileUtil.xlsx_to_data_frames(file_bytes, header_row_index,
99
+ keep_default_na=keep_default_na, **kwargs)
87
100
  # Parse the metadata from above the header row index into a list of lists.
88
101
  metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
89
102
  # Parse the data from the file body into a list of dicts.
@@ -94,7 +107,8 @@ class FileUtil:
94
107
 
95
108
  @staticmethod
96
109
  def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
97
- *, encoding: str | None = None, encoding_error: str | None = "strict") \
110
+ *, encoding: str | None = None, encoding_error: str | None = "strict",
111
+ keep_default_na: bool = False, **kwargs) \
98
112
  -> tuple[DataFrame, DataFrame | None]:
99
113
  """
100
114
  Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
@@ -113,6 +127,9 @@ class FileUtil:
113
127
  is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
114
128
  characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
115
129
  https://docs.python.org/3/library/codecs.html#error-handlers
130
+ :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
131
+ If True, these values will be converted to a NoneType value.
132
+ :param kwargs: Additional arguments to be passed to the pandas read_csv function.
116
133
  :return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
117
134
  metadata from above the header row, or None if there is no metadata.
118
135
  """
@@ -125,19 +142,21 @@ class FileUtil:
125
142
  file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
126
143
  skiprows=lambda x: x >= header_row_index,
127
144
  skip_blank_lines=False, sep=seperator, encoding=encoding,
128
- encoding_errors=encoding_error)
145
+ encoding_errors=encoding_error, keep_default_na=keep_default_na,
146
+ **kwargs)
129
147
  with io.BytesIO(file_bytes) as file_io:
130
148
  # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
131
149
  # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
132
150
  # string.
133
151
  file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
134
- skip_blank_lines=False, sep=seperator, encoding=encoding)
152
+ skip_blank_lines=False, sep=seperator, encoding=encoding,
153
+ keep_default_na=keep_default_na, **kwargs)
135
154
 
136
155
  return file_body, file_metadata
137
156
 
138
157
  @staticmethod
139
- def xlsx_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0) \
140
- -> tuple[DataFrame, DataFrame | None]:
158
+ def xlsx_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, *, keep_default_na: bool = False,
159
+ **kwargs) -> tuple[DataFrame, DataFrame | None]:
141
160
  """
142
161
  Parse the file bytes for an XLSX into DataFrames.
143
162
 
@@ -146,6 +165,9 @@ class FileUtil:
146
165
  row is returned in the metadata list. If input is None, then no row is considered to be the header row,
147
166
  meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
148
167
  is assumed to be the header row.
168
+ :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
169
+ If True, these values will be converted to a NoneType value.
170
+ :param kwargs: Additional arguments to be passed to the pandas read_excel function.
149
171
  :return: A tuple of two DataFrames. The first is the frame for the XLSX table body, while the second is for the
150
172
  metadata from above the header row, or None if there is no metadata.
151
173
  """
@@ -155,12 +177,14 @@ class FileUtil:
155
177
  # The metadata DataFrame has no headers and only consists of the rows above the header row index.
156
178
  # Therefore, we skip every row including and past the header.
157
179
  file_metadata = pandas.read_excel(file_io, header=None, dtype=dtype(str),
158
- skiprows=lambda x: x >= header_row_index)
180
+ skiprows=lambda x: x >= header_row_index,
181
+ keep_default_na=keep_default_na, **kwargs)
159
182
  with io.BytesIO(file_bytes) as file_io:
160
183
  # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
161
184
  # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
162
185
  # string.
163
- file_body: DataFrame = pandas.read_excel(file_io, header=header_row_index, dtype=dtype(str))
186
+ file_body: DataFrame = pandas.read_excel(file_io, header=header_row_index, dtype=dtype(str),
187
+ keep_default_na=keep_default_na, **kwargs)
164
188
 
165
189
  return file_body, file_metadata
166
190
 
@@ -255,6 +279,7 @@ class FileUtil:
255
279
  data_frame = pandas.read_csv(csv, sep=",", header=None)
256
280
 
257
281
  with io.BytesIO() as output:
282
+ # noinspection PyTypeChecker
258
283
  with pandas.ExcelWriter(output, engine='xlsxwriter') as writer:
259
284
  # Setting header and index to false makes the CSV convert to an XLSX as-is.
260
285
  data_frame.to_excel(writer, sheet_name='Sheet1', header=False, index=False)
@@ -0,0 +1,48 @@
1
+ from sapiopycommons.general.aliases import SapioRecord
2
+ from sapiopylib.rest.User import SapioUser
3
+ from sapiopylib.rest.utils.MultiMap import SetMultimap
4
+
5
+ # FR-47421 Added module
6
+
7
+ def create_aliquot_for_samples(parent_sample_to_num_aliquots_map: dict[SapioRecord, int], user: SapioUser) -> SetMultimap[SapioRecord, int]:
8
+ """"
9
+ Ask server to create aliquot records for provided sample parent records.
10
+ :param parent_sample_to_num_aliquots_map: The dictionary containing (parent sample record) -> (number of aliquots to create) mapping.
11
+ :return: The dictionary containing (parent sample record) -> (list of new aliquot record ids) mapping.
12
+ """
13
+ # throw error if at least one record id is blank
14
+ has_negative_record_ids = any([record.record_id < 0 for record in parent_sample_to_num_aliquots_map.keys()])
15
+ if has_negative_record_ids:
16
+ raise ValueError("At least one record requested for aliquot has a negative record ID. "
17
+ "You should have stored record model changes first.")
18
+ has_blank_record_ids = any([record.record_id is None for record in parent_sample_to_num_aliquots_map.keys()])
19
+ if has_blank_record_ids:
20
+ raise ValueError("At least one record requested for aliquot does not currently have a record ID.")
21
+ record_id_to_sapio_record_map = {record.record_id: record for record in parent_sample_to_num_aliquots_map.keys()}
22
+ parent_record_id_to_num_aliquots_map = {record.record_id: num_aliquots for record, num_aliquots in parent_sample_to_num_aliquots_map.items()}
23
+ aliquot_result: SetMultimap[int, int] = create_aliquot_for_samples_record_ids(parent_record_id_to_num_aliquots_map, user)
24
+ ret: SetMultimap[SapioRecord, int] = SetMultimap()
25
+ for parent_record_id in aliquot_result.keys():
26
+ parent_record = record_id_to_sapio_record_map[parent_record_id]
27
+ for aliquot_record_id in aliquot_result.get(parent_record_id):
28
+ ret.put(parent_record, aliquot_record_id)
29
+ return ret
30
+
31
+
32
+ def create_aliquot_for_samples_record_ids(parent_record_id_to_num_aliquots_map: dict[int, int], user: SapioUser) -> SetMultimap[int, int]:
33
+ """
34
+ Ask the server to create aliquot records for the provided sample record IDs.
35
+ :param sample_record_id_list: The dictionary containing (parent sample record id) -> (number of aliquots to create) mapping.
36
+ :return: The dictionary containing (parent sample record id) -> (list of new aliquot record ids) mapping.
37
+ """
38
+ if not parent_record_id_to_num_aliquots_map:
39
+ return SetMultimap()
40
+ endpoint_path = 'sample/aliquot'
41
+ response = user.plugin_put(endpoint_path, payload=parent_record_id_to_num_aliquots_map)
42
+ user.raise_for_status(response)
43
+ response_map: dict[int, list[int]] = response.json()
44
+ ret: SetMultimap[int, int] = SetMultimap()
45
+ for parent_record_id, aliquot_record_ids in response_map.items():
46
+ for aliquot_record_id in aliquot_record_ids:
47
+ ret.put(int(parent_record_id), int(aliquot_record_id))
48
+ return ret
@@ -73,7 +73,7 @@ class CommonsWebhookHandler(AbstractWebhookHandler):
73
73
  """A class for making requests to the accession webservice endpoints."""
74
74
  fnd_acc_man: FoundationAccessionManager
75
75
  """A class for making requests to the Foundations accession webservice endpoints."""
76
- custom_report_man: CustomReportManager
76
+ report_man: CustomReportManager
77
77
  """A class for making requests to the custom report webservice endpoints."""
78
78
  dash_man: DashboardManager
79
79
  """A class for making requests to the dashboard management webservice endpoints."""
@@ -179,7 +179,7 @@ class CommonsWebhookHandler(AbstractWebhookHandler):
179
179
 
180
180
  # Initialize basic manager classes from sapiopylib.
181
181
  self.acc_man = DataMgmtServer.get_accession_manager(self.user)
182
- self.custom_report_man = DataMgmtServer.get_custom_report_manager(self.user)
182
+ self.report_man = DataMgmtServer.get_custom_report_manager(self.user)
183
183
  self.dash_man = DataMgmtServer.get_dashboard_manager(self.user)
184
184
  self.xml_data_man = DataMgmtServer.get_data_manager(self.user)
185
185
  self.dr_man = context.data_record_manager
@@ -0,0 +1,47 @@
1
+ import unittest
2
+
3
+ from sapiopylib.rest.DataMgmtService import DataMgmtServer
4
+ from sapiopylib.rest.DataRecordManagerService import DataRecordManager
5
+ from sapiopylib.rest.User import SapioUser
6
+ from sapiopylib.rest.utils.recordmodel.RecordModelManager import RecordModelManager, RecordModelInstanceManager, \
7
+ RecordModelRelationshipManager
8
+ from sapiopylib.rest.utils.recordmodel.properties import Children
9
+
10
+ from sapiopycommons.samples.aliquot import create_aliquot_for_samples
11
+ from sapiopycommons.general.accession_service import AccessionService
12
+ from data_type_models import *
13
+
14
+ # FR-47421 Added module
15
+
16
+ user = SapioUser(url="https://linux-vm:8443/webservice/api", verify_ssl_cert=False,
17
+ guid="66c2bea5-7cb2-4bfc-a413-304a3f4c3f33",
18
+ username="yqiao_api", password="Password1!")
19
+ data_record_manager: DataRecordManager = DataMgmtServer.get_data_record_manager(user)
20
+ rec_man: RecordModelManager = RecordModelManager(user)
21
+ inst_man: RecordModelInstanceManager = rec_man.instance_manager
22
+ relationship_man: RecordModelRelationshipManager = rec_man.relationship_manager
23
+ accession_service = AccessionService(user)
24
+
25
+ class AliquotTest(unittest.TestCase):
26
+ def test_aliquot_samples(self):
27
+ parent_sample_1 = inst_man.add_new_record_of_type(SampleModel)
28
+ parent_sample_2 = inst_man.add_new_record_of_type(SampleModel)
29
+ id_list = accession_service.accession_with_config(SampleModel.DATA_TYPE_NAME, SampleModel.SAMPLEID__FIELD_NAME.field_name, 2)
30
+ parent_sample_1.set_SampleId_field(id_list[0])
31
+ parent_sample_2.set_SampleId_field(id_list[1])
32
+ aliquot_request = {parent_sample_1: 2, parent_sample_2: 3}
33
+ rec_man.store_and_commit()
34
+ aliquot_map = create_aliquot_for_samples(aliquot_request, user)
35
+ self.assertTrue(len(aliquot_map) == 2)
36
+ self.assertTrue(parent_sample_1 in aliquot_map and parent_sample_2 in aliquot_map)
37
+ self.assertTrue(len(aliquot_map.get(parent_sample_1)) == 2)
38
+ self.assertTrue(len(aliquot_map.get(parent_sample_2)) == 3)
39
+
40
+ relationship_man.load_children_of_type([parent_sample_1, parent_sample_2], SampleModel)
41
+ child_id_list_sample_1 = [child.get_SampleId_field() for child in parent_sample_1.get(Children.of_type(SampleModel))]
42
+ child_id_list_sample_2 = [child.get_SampleId_field() for child in parent_sample_2.get(Children.of_type(SampleModel))]
43
+ # Check the format of all ids of parent sample id "_" number is in the child list.
44
+ for i in range(2):
45
+ self.assertTrue(f"{parent_sample_1.get_SampleId_field()}_{i+1}" in child_id_list_sample_1)
46
+ for i in range(3):
47
+ self.assertTrue(f"{parent_sample_2.get_SampleId_field()}_{i+1}" in child_id_list_sample_2)