sapiopycommons 2025.4.9a150__py3-none-any.whl → 2025.4.9a476__py3-none-any.whl

sapiopycommons/files/file_util.py

@@ -21,10 +21,14 @@ class FileUtil:
     Utilities for the handling of files, including the requesting of files from the user and the parsing of files into
     tokenized lists. Makes use of Pandas DataFrames for any file parsing purposes.
     """
+    # PR-47433: Add a keep_default_na argument to FileUtil.tokenize_csv and FileUtil.tokenize_xlsx so that N/A values
+    # don't get returned as NoneType, and add **kwargs in case any other Pandas input parameters need changed by the
+    # caller.
     @staticmethod
     def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
                      seperator: str = ",", *, encoding: str | None = None, encoding_error: str | None = "strict",
-                     exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
+                     exception_on_empty: bool = True, keep_default_na: bool = False, **kwargs) \
+            -> tuple[list[dict[str, str]], list[list[str]]]:
         """
         Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
         must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -46,6 +50,9 @@ class FileUtil:
             https://docs.python.org/3/library/codecs.html#error-handlers
         :param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
             the first element of the returned tuple.
+        :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
+            If True, these values will be converted to a NoneType value.
+        :param kwargs: Additional arguments to be passed to the pandas read_csv function.
         :return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
             that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
             If the header row index is 0 or None, this list will be empty.
@@ -53,7 +60,8 @@ class FileUtil:
         # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
         # while the second is the body of the file below the header row.
         file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
-                                                               encoding=encoding, encoding_error=encoding_error)
+                                                               encoding=encoding, encoding_error=encoding_error,
+                                                               keep_default_na=keep_default_na, **kwargs)
         # Parse the metadata from above the header row index into a list of lists.
         metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
         # Parse the data from the file body into a list of dicts.
@@ -64,7 +72,8 @@ class FileUtil:
 
     @staticmethod
     def tokenize_xlsx(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
-                      *, exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
+                      *, exception_on_empty: bool = True, keep_default_na: bool = False, **kwargs) \
+            -> tuple[list[dict[str, str]], list[list[str]]]:
         """
         Tokenize an XLSX file row by row.
 
@@ -77,13 +86,17 @@ class FileUtil:
             is assumed to be the header row.
         :param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
             the first element of the returned tuple.
+        :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
+            If True, these values will be converted to a NoneType value.
+        :param kwargs: Additional arguments to be passed to the pandas read_excel function.
         :return: The XLSX parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
             that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
             If the header row index is 0 or None, this list will be empty.
         """
         # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
         # while the second is the body of the file below the header row.
-        file_body, file_metadata = FileUtil.xlsx_to_data_frames(file_bytes, header_row_index)
+        file_body, file_metadata = FileUtil.xlsx_to_data_frames(file_bytes, header_row_index,
+                                                                keep_default_na=keep_default_na, **kwargs)
         # Parse the metadata from above the header row index into a list of lists.
         metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
         # Parse the data from the file body into a list of dicts.
@@ -94,7 +107,8 @@ class FileUtil:
 
     @staticmethod
     def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
-                           *, encoding: str | None = None, encoding_error: str | None = "strict") \
+                           *, encoding: str | None = None, encoding_error: str | None = "strict",
+                           keep_default_na: bool = False, **kwargs) \
             -> tuple[DataFrame, DataFrame | None]:
         """
         Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
@@ -113,6 +127,9 @@ class FileUtil:
             is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
             characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
             https://docs.python.org/3/library/codecs.html#error-handlers
+        :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
+            If True, these values will be converted to a NoneType value.
+        :param kwargs: Additional arguments to be passed to the pandas read_csv function.
         :return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
             metadata from above the header row, or None if there is no metadata.
         """
@@ -125,19 +142,21 @@ class FileUtil:
                 file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
                                                 skiprows=lambda x: x >= header_row_index,
                                                 skip_blank_lines=False, sep=seperator, encoding=encoding,
-                                                encoding_errors=encoding_error)
+                                                encoding_errors=encoding_error, keep_default_na=keep_default_na,
+                                                **kwargs)
         with io.BytesIO(file_bytes) as file_io:
             # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
             # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
             # string.
             file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
-                                                   skip_blank_lines=False, sep=seperator, encoding=encoding)
+                                                   skip_blank_lines=False, sep=seperator, encoding=encoding,
+                                                   keep_default_na=keep_default_na, **kwargs)
 
         return file_body, file_metadata
 
     @staticmethod
-    def xlsx_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0) \
-            -> tuple[DataFrame, DataFrame | None]:
+    def xlsx_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, *, keep_default_na: bool = False,
+                            **kwargs) -> tuple[DataFrame, DataFrame | None]:
         """
         Parse the file bytes for an XLSX into DataFrames.
 
@@ -146,6 +165,9 @@ class FileUtil:
             row is returned in the metadata list. If input is None, then no row is considered to be the header row,
             meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
             is assumed to be the header row.
+        :param keep_default_na: If False, values that are recognized as NaN (e.g. N/A, NA, NaN) will remain as strings.
+            If True, these values will be converted to a NoneType value.
+        :param kwargs: Additional arguments to be passed to the pandas read_excel function.
         :return: A tuple of two DataFrames. The first is the frame for the XLSX table body, while the second is for the
             metadata from above the header row, or None if there is no metadata.
         """
@@ -155,12 +177,14 @@ class FileUtil:
                 # The metadata DataFrame has no headers and only consists of the rows above the header row index.
                 # Therefore, we skip every row including and past the header.
                 file_metadata = pandas.read_excel(file_io, header=None, dtype=dtype(str),
-                                                  skiprows=lambda x: x >= header_row_index)
+                                                  skiprows=lambda x: x >= header_row_index,
+                                                  keep_default_na=keep_default_na, **kwargs)
         with io.BytesIO(file_bytes) as file_io:
             # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
             # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
             # string.
-            file_body: DataFrame = pandas.read_excel(file_io, header=header_row_index, dtype=dtype(str))
+            file_body: DataFrame = pandas.read_excel(file_io, header=header_row_index, dtype=dtype(str),
+                                                     keep_default_na=keep_default_na, **kwargs)
 
         return file_body, file_metadata
 
@@ -255,6 +279,7 @@ class FileUtil:
             data_frame = pandas.read_csv(csv, sep=",", header=None)
 
         with io.BytesIO() as output:
+            # noinspection PyTypeChecker
             with pandas.ExcelWriter(output, engine='xlsxwriter') as writer:
                 # Setting header and index to false makes the CSV convert to an XLSX as-is.
                 data_frame.to_excel(writer, sheet_name='Sheet1', header=False, index=False)
@@ -302,10 +327,10 @@ class FileUtil:
         :param files: A dictionary of file name to file data as a string or bytes.
         :return: The bytes for a zip file containing the input files.
         """
-        zip_buffer: io.BytesIO = io.BytesIO()
-        with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
-            for file_name, file_data in files.items():
-                zip_file.writestr(file_name, file_data)
+        with io.BytesIO() as zip_buffer:
+            with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+                for file_name, file_data in files.items():
+                    zip_file.writestr(file_name, file_data)
         return zip_buffer.getvalue()
 
     # Deprecated functions:

sapiopycommons/files/file_validator.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from abc import abstractmethod
 from typing import Any
 
@@ -9,9 +10,9 @@ from sapiopylib.rest.pojo.datatype.FieldDefinition import VeloxIntegerFieldDefin
     AbstractVeloxFieldDefinition
 
 from sapiopycommons.callbacks.callback_util import CallbackUtil
+from sapiopycommons.customreport.auto_pagers import QuickReportDictAutoPager
 from sapiopycommons.files.file_data_handler import FileDataHandler, FilterList
 from sapiopycommons.general.aliases import UserIdentifier, AliasUtil
-from sapiopycommons.general.custom_report_util import CustomReportUtil
 from sapiopycommons.general.exceptions import SapioUserCancelledException
 from sapiopycommons.general.time_util import TimeUtil
 
@@ -311,8 +312,8 @@ class MatchesPatternRule(ColumnRule):
     """
     pattern: str
 
-    def __init__(self, header: str, pattern: str, *, reason: str | None = None, whitelist: FilterList = None,
-                 blacklist: FilterList = None):
+    def __init__(self, header: str, pattern: str | re.Pattern[str], *, reason: str | None = None,
+                 whitelist: FilterList = None, blacklist: FilterList = None):
         """
         :param header: The header that this rule acts upon.
         :param pattern: A regex pattern.
@@ -529,7 +530,7 @@ class UniqueSystemValueRule(ColumnRule):
         # Run a quick report for all records of this type that match these field values.
         term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
                              "{" + ",".join(values) + "}")
-        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        results: list[dict[str, Any]] = QuickReportDictAutoPager(self.user, term).get_all_at_once()
         existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
         return file_handler.get_in_list(self.header, existing_values)
 
@@ -563,6 +564,6 @@ class ExistingSystemValueRule(ColumnRule):
         # Run a quick report for all records of this type that match these field values.
         term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
                              "{" + ",".join(values) + "}")
-        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        results: list[dict[str, Any]] = QuickReportDictAutoPager(self.user, term).get_all_at_once()
         existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
         return file_handler.get_not_in_list(self.header, existing_values)

sapiopycommons/files/file_writer.py

@@ -307,7 +307,7 @@ class FieldColumn(ColumnDef):
         elif self.search_order == FieldSearchOrder.BUNDLE_ONLY:
             return row.fields.get(self.field_name)
         elif self.search_order == FieldSearchOrder.RECORD_FIRST:
-            fields: dict[str, Any] = AliasUtil.to_field_map_lists([record])[0] if record else {}
+            fields: dict[str, Any] = AliasUtil.to_field_map(record) if record else {}
             if self.field_name not in fields or (self.skip_none_values and fields.get(self.field_name) is None):
                 return row.fields.get(self.field_name)
             return fields.get(self.field_name)

sapiopycommons/flowcyto/flow_cyto.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from weakref import WeakValueDictionary
 
-from sapiopylib.rest.User import SapioUser
 from databind.json import dumps
+from sapiopylib.rest.User import SapioUser
 
 from sapiopycommons.flowcyto.flowcyto_data import FlowJoWorkspaceInputJson, UploadFCSInputJson, \
     ComputeFlowStatisticsInputJson

sapiopycommons/general/accession_service.py

@@ -95,7 +95,7 @@ class AccessionWithPrefixSuffix(AbstractAccessionServiceOperator):
 
     @property
     def default_accessor_name(self):
-        return "PREFIX_AND_SUFFIX" + "(" + self.prefix + "," + self.suffix + ")";
+        return "PREFIX_AND_SUFFIX" + "(" + self.prefix + "," + self.suffix + ")"
 
 
 class AccessionGlobalPrefixSuffix(AbstractAccessionServiceOperator):
@@ -199,7 +199,7 @@ class AccessionRequestId(AbstractAccessionServiceOperator):
 
     Properties:
         numberOfCharacters: Number of characters maximum in the request ID.
-        accessorName: This is a legacy variable from drum.getNextIdListByMapName(), which allows setting different "accessorName" from old system. We need this for compability patch for converting these to the new preference format.
+        accessorName: This is a legacy variable from drum.getNextIdListByMapName(), which allows setting different "accessorName" from old system. We need this for compatibility patch for converting these to the new preference format.
     """
     _num_of_characters: int
     _accessor_name: str
@@ -341,7 +341,7 @@ class AccessionService:
     def get_affixed_id_in_batch(self, data_type_name: str, data_field_name: str, num_ids: int, prefix: str | None,
                                 suffix: str | None, num_digits: int | None, start_num: int = 1) -> list[str]:
         """
-        Get the batch affixed IDs that are maximal in cache and contiguious for a particular datatype.datafield under a given format.
+        Get the batch affixed IDs that are maximal in cache and contiguous for a particular datatype.datafield under a given format.
         :param data_type_name: The datatype name to look for max ID
         :param data_field_name: The datafield name to look for max ID
         :param num_ids: The number of IDs to accession.

sapiopycommons/general/aliases.py

@@ -1,50 +1,53 @@
 from collections.abc import Iterable
-from typing import Any
+from typing import Any, TypeAlias
 
 from sapiopylib.rest.User import SapioUser
 from sapiopylib.rest.pojo.DataRecord import DataRecord
-from sapiopylib.rest.pojo.datatype.FieldDefinition import FieldType
+from sapiopylib.rest.pojo.datatype.FieldDefinition import FieldType, AbstractVeloxFieldDefinition
 from sapiopylib.rest.pojo.eln.ElnExperiment import ElnExperiment
 from sapiopylib.rest.pojo.eln.ExperimentEntry import ExperimentEntry
 from sapiopylib.rest.pojo.eln.SapioELNEnums import ElnBaseDataType
+from sapiopylib.rest.pojo.eln.eln_headings import ElnExperimentTab
 from sapiopylib.rest.pojo.webhook.WebhookContext import SapioWebhookContext
 from sapiopylib.rest.utils.Protocols import ElnExperimentProtocol, ElnEntryStep
-from sapiopylib.rest.utils.recordmodel.PyRecordModel import PyRecordModel
+from sapiopylib.rest.utils.recordmodel.PyRecordModel import PyRecordModel, AbstractRecordModel
 from sapiopylib.rest.utils.recordmodel.RecordModelWrapper import WrappedRecordModel, WrappedType, WrapperField
 
 from sapiopycommons.general.exceptions import SapioException
 
-FieldValue = int | float | str | bool | None
+FieldValue: TypeAlias = int | float | str | bool | None
 """Allowable values for fields in the system."""
-RecordModel = PyRecordModel | WrappedRecordModel
+RecordModel: TypeAlias = PyRecordModel | AbstractRecordModel | WrappedRecordModel
 """Different forms that a record model could take."""
-SapioRecord = DataRecord | RecordModel
+SapioRecord: TypeAlias = DataRecord | RecordModel
 """A record could be provided as either a DataRecord, PyRecordModel, or WrappedRecordModel (WrappedType)."""
-RecordIdentifier = SapioRecord | int
+RecordIdentifier: TypeAlias = SapioRecord | int
 """A RecordIdentifier is either a record type or an integer for the record's record ID."""
-DataTypeIdentifier = SapioRecord | type[WrappedType] | str
+DataTypeIdentifier: TypeAlias = SapioRecord | type[WrappedType] | str
 """A DataTypeIdentifier is either a SapioRecord, a record model wrapper type, or a string."""
-FieldIdentifier = WrapperField | str | tuple[str, FieldType]
+FieldIdentifier: TypeAlias = AbstractVeloxFieldDefinition | WrapperField | str | tuple[str, FieldType]
 """A FieldIdentifier is either wrapper field from a record model wrapper, a string, or a tuple of string
 and field type."""
-FieldIdentifierKey = WrapperField | str
+FieldIdentifierKey: TypeAlias = WrapperField | str
 """A FieldIdentifierKey is a FieldIdentifier, except it can't be a tuple, s tuples can't be used as keys in
 dictionaries.."""
-HasFieldWrappers = type[WrappedType] | WrappedRecordModel
+HasFieldWrappers: TypeAlias = type[WrappedType] | WrappedRecordModel
 """An identifier for classes that have wrapper fields."""
-ExperimentIdentifier = ElnExperimentProtocol | ElnExperiment | int
+ExperimentIdentifier: TypeAlias = ElnExperimentProtocol | ElnExperiment | int
 """An ExperimentIdentifier is either an experiment protocol, experiment, or an integer for the experiment's notebook
 ID."""
-ExperimentEntryIdentifier = ElnEntryStep | ExperimentEntry | int
+ExperimentEntryIdentifier: TypeAlias = ElnEntryStep | ExperimentEntry | int
 """An ExperimentEntryIdentifier is either an ELN entry step, experiment entry, or an integer for the entry's ID."""
-FieldMap = dict[str, FieldValue]
+TabIdentifier: TypeAlias = int | ElnExperimentTab
+"""A TabIdentifier is either an integer for the tab's ID or an ElnExperimentTab object."""
+FieldMap: TypeAlias = dict[str, FieldValue]
 """A field map is simply a dict of data field names to values. The purpose of aliasing this is to help distinguish
 any random dict in a webhook from one which is explicitly used for record fields."""
-FieldIdentifierMap = dict[FieldIdentifierKey, FieldValue]
+FieldIdentifierMap: TypeAlias = dict[FieldIdentifierKey, FieldValue]
 """A field identifier map is the same thing as a field map, except the keys can be field identifiers instead
 of just strings. Note that although one of the allowed field identifiers is a tuple, you can't use tuples as
 keys in a dictionary."""
-UserIdentifier = SapioWebhookContext | SapioUser
+UserIdentifier: TypeAlias = SapioWebhookContext | SapioUser
 """An identifier for classes from which a user object can be used for sending requests."""
 
 
@@ -142,23 +145,25 @@ class AliasUtil:
     @staticmethod
     def to_data_field_name(value: FieldIdentifier) -> str:
         """
-        Convert a string or WrapperField to a data field name string.
+        Convert an object that can be used to identify a data field to a data field name string.
 
-        :param value: A string or WrapperField.
+        :param value: An object that can be used to identify a data field.
         :return: A string of the data field name of the input value.
         """
         if isinstance(value, tuple):
             return value[0]
         if isinstance(value, WrapperField):
             return value.field_name
+        if isinstance(value, AbstractVeloxFieldDefinition):
+            return value.data_field_name
         return value
 
     @staticmethod
     def to_data_field_names(values: Iterable[FieldIdentifier]) -> list[str]:
         """
-        Convert an iterable of strings or WrapperFields to a list of data field name strings.
+        Convert an iterable of objects that can be used to identify data fields to a list of data field name strings.
 
-        :param values: An iterable of strings or WrapperFields.
+        :param values: An iterable of objects that can be used to identify a data field.
         :return: A list of strings of the data field names of the input values.
         """
         return [AliasUtil.to_data_field_name(x) for x in values]
@@ -205,20 +210,38 @@ class AliasUtil:
                              f"field with the name \"{field}\",")
 
     @staticmethod
-    def to_field_map_lists(records: Iterable[SapioRecord]) -> list[FieldMap]:
+    def to_field_map(record: SapioRecord, include_record_id: bool = False) -> FieldMap:
         """
-        Convert a list of variables that could either be DataRecords, PyRecordModels,
-        or WrappedRecordModels to a list of their field maps.
+        Convert a given record value to a field map.
+
+        :param record: A record which is a DataRecord, PyRecordModel, or WrappedRecordModel.
+        :param include_record_id: If true, include the record ID of the record in the field map using the RecordId key.
+        :return: The field map for the input record.
+        """
+        if isinstance(record, DataRecord):
+            # noinspection PyTypeChecker
+            fields: FieldMap = record.get_fields()
+        else:
+            fields: FieldMap = record.fields.copy_to_dict()
+        # PR-47457: Only include the record ID if the caller requests it, since including the record ID can break
+        # callbacks in certain circumstances if the record ID is negative.
+        if include_record_id:
+            fields["RecordId"] = AliasUtil.to_record_id(record)
+        return fields
+
+    @staticmethod
+    def to_field_map_list(records: Iterable[SapioRecord], include_record_id: bool = False) -> list[FieldMap]:
+        """
+        Convert a list of variables that could either be DataRecords, PyRecordModels, or WrappedRecordModels
+        to a list of their field maps. This includes the given RecordId of the given records.
 
+        :param records: An iterable of records which are DataRecords, PyRecordModels, or WrappedRecordModels.
+        :param include_record_id: If true, include the record ID of the records in the field map using the RecordId key.
         :return: A list of field maps for the input records.
         """
         field_map_list: list[FieldMap] = []
         for record in records:
-            if isinstance(record, DataRecord):
-                # noinspection PyTypeChecker
-                field_map_list.append(record.get_fields())
-            else:
-                field_map_list.append(record.fields.copy_to_dict())
+            field_map_list.append(AliasUtil.to_field_map(record, include_record_id))
         return field_map_list
 
     @staticmethod

sapiopycommons/general/audit_log.py

@@ -3,11 +3,11 @@ from enum import Enum
 from sapiopylib.rest.User import SapioUser
 from sapiopylib.rest.pojo.CustomReport import ReportColumn, CustomReportCriteria
 
+from sapiopycommons.customreport.auto_pagers import CustomReportDictAutoPager
 from sapiopycommons.customreport.column_builder import ColumnBuilder
 from sapiopycommons.customreport.term_builder import TermBuilder
 from sapiopycommons.datatype.pseudo_data_types import AuditLogPseudoDef
 from sapiopycommons.general.aliases import RecordIdentifier, AliasUtil, UserIdentifier, FieldIdentifier, FieldValue
-from sapiopycommons.general.custom_report_util import CustomReportUtil
 
 
 class EventType(Enum):
@@ -164,7 +164,7 @@ class AuditLogUtil:
         criteria = AuditLogUtil.create_data_record_audit_log_report(records, fields)
 
         # Then we must run the custom report using that criteria.
-        raw_report_data: list[dict[str, FieldValue]] = CustomReportUtil.run_custom_report(self.user, criteria)
+        raw_report_data: list[dict[str, FieldValue]] = CustomReportDictAutoPager(self.user, criteria).get_all_at_once()
 
         # This section will prepare a map matching the original RecordIdentifier by record id.
         # This is because the audit log entries will have record ids, but we want the keys in our result map

sapiopycommons/general/custom_report_util.py

@@ -1,3 +1,4 @@
+import warnings
 from collections.abc import Iterable
 
 from sapiopylib.rest.DataMgmtService import DataMgmtServer
@@ -40,6 +41,7 @@ class CustomReportUtil:
             had a Sample column with a data field name of Identifier and a Request column with the same data field name,
             then the dictionary keys for these columns would be Sample.Identifier and Request.Identifier respectively.
         """
+        warnings.warn("Deprecated in favor of the SystemReportDictAutoPager class.", DeprecationWarning)
         results: tuple = CustomReportUtil._exhaust_system_report(context, report_name, page_limit,
                                                                  page_size, page_number)
         columns: list[ReportColumn] = results[0]
@@ -82,6 +84,7 @@ class CustomReportUtil:
             had a Sample column with a data field name of Identifier and a Request column with the same data field name,
             then the dictionary keys for these columns would be Sample.Identifier and Request.Identifier respectively.
         """
+        warnings.warn("Deprecated in favor of the CustomReportDictAutoPager class.", DeprecationWarning)
         results: tuple = CustomReportUtil._exhaust_custom_report(context, report_criteria, page_limit,
                                                                  page_size, page_number)
         columns: list[ReportColumn] = results[0]
@@ -117,6 +120,7 @@ class CustomReportUtil:
         :return: The results of the report listed row by row, mapping each cell to the header it is under. The header
             values in the dicts are the data field names of the columns.
         """
+        warnings.warn("Deprecated in favor of the QuickReportDictAutoPager class.", DeprecationWarning)
         results: tuple = CustomReportUtil._exhaust_quick_report(context, report_term, page_limit,
                                                                 page_size, page_number)
         columns: list[ReportColumn] = results[0]
@@ -127,7 +131,8 @@ class CustomReportUtil:
     def get_system_report_criteria(context: UserIdentifier, report_name: str) -> CustomReport:
         """
         Retrieve a custom report from the system given the name of the report. This works by querying the system report
-        with a page number and size of 1 to minimize the amount of data transfer needed to retrieve the report's config.
+        with a page number of 0 and page size of 1 to minimize the amount of data transfer needed to retrieve the
+        report's config.
 
         System reports are also known as predefined searches in the system and must be defined in the data designer for
         a specific data type. That is, saved searches created by users cannot be run using this function.
@@ -143,6 +148,24 @@ class CustomReportUtil:
         report_man = DataMgmtServer.get_custom_report_manager(user)
         return report_man.run_system_report_by_name(report_name, 1, 0)
 
+    @staticmethod
+    def get_quick_report_criteria(context: UserIdentifier, report_term: RawReportTerm) -> CustomReport:
+        """
+        Retrieve a quick report from the system given a report term. This works by making a quick report query
+        with a page number of 0 and page size of 1 to minimize the amount of data transfer needed to retrieve the
+        report's config.
+
+        Using this, you can add to the root term of the search to then run a new search, or provide it to client
+        callbacks or directives that take CustomReports.
+
+        :param context: The current webhook context or a user object to send requests from.
+        :param report_term: The raw report term to use for the quick report.
+        :return: The CustomReport object for the given report term.
+        """
+        user: SapioUser = AliasUtil.to_sapio_user(context)
+        report_man = DataMgmtServer.get_custom_report_manager(user)
+        return report_man.run_quick_report(report_term, 1, 0)
+
     @staticmethod
     def _exhaust_system_report(context: UserIdentifier,
                                report_name: str,

sapiopycommons/general/data_structure_util.py

@@ -0,0 +1,115 @@
+from enum import Enum
+from typing import Iterable, Any, Collection
+
+from sapiopycommons.general.exceptions import SapioException
+
+
+class ArrayTransformation(Enum):
+    """
+    An enumeration of the different transformations that can be applied to a 2D array.
+    """
+    ROTATE_CLOCKWISE = 0
+    ROTATE_COUNTER_CLOCKWISE = 1
+    ROTATE_180_DEGREES = 2
+    MIRROR_HORIZONTAL = 3
+    MIRROR_VERTICAL = 4
+
+
+# FR-47524: Create a DataStructureUtils class that implements various collection utility functions from our Java
+# libraries.
+class DataStructureUtil:
+    """
+    Utility class for working with data structures. Copies from ListUtil, SetUtil, and various other classes in
+    our Java library.
+    """
+    @staticmethod
+    def find_first_or_none(values: Iterable[Any]) -> Any | None:
+        """
+        Get the first value from an iterable, or None if the iterable is empty.
+
+        :param values: An iterable of values.
+        :return: The first value from the input, or None if the input is empty.
+        """
+        return next(iter(values), None)
+
+    @staticmethod
+    def remove_null_values(values: Iterable[Any]) -> list[Any]:
+        """
+        Remove null values from a list.
+
+        :param values: An iterable of values.
+        :return: A list containing all the non-null values from the input.
+        """
+        return [value for value in values if value is not None]
+
+    @staticmethod
+    def transform_2d_array(values: Collection[Collection[Any]], transformation: ArrayTransformation) \
+            -> Collection[Collection[Any]]:
+        """
+        Perform a transformation on a 2D list.
+
+        :param values: An iterable of iterables. The iterables should all be of the same size.
+        :param transformation: The transformation to apply to the input.
+        :return: A new 2D list containing the input transformed according to the specified transformation.
+        """
+        x: int = len(values)
+        for row in values:
+            y = len(row)
+            if y != x:
+                raise SapioException(f"Input must be a square 2D array. The provided array has a length of {x} but "
+                                     f"at least one row has a length of {y}.")
+
+        match transformation:
+            case ArrayTransformation.ROTATE_CLOCKWISE:
+                return [list(row) for row in zip(*values[::-1])]
+            case ArrayTransformation.ROTATE_COUNTER_CLOCKWISE:
+                return [list(row) for row in zip(*values)][::-1]
+            case ArrayTransformation.ROTATE_180_DEGREES:
+                return [row[::-1] for row in values[::-1]]
+            case ArrayTransformation.MIRROR_HORIZONTAL:
+                return [list(row[::-1]) for row in values]
+            case ArrayTransformation.MIRROR_VERTICAL:
+                return values[::-1]
+
+        raise SapioException(f"Invalid transformation: {transformation}")
+
+    @staticmethod
+    def flatten_to_list(values: Iterable[Iterable[Any]]) -> list[Any]:
+        """
+        Flatten a list of lists into a single list.
+
+        :param values: An iterable of iterables.
+        :return: A single list containing all the values from the input. Elements are in the order they appear in the
+            input.
+        """
+        return [item for sublist in values for item in sublist]
+
+    @staticmethod
+    def flatten_to_set(values: Iterable[Iterable[Any]]) -> set[Any]:
+        """
+        Flatten a list of lists into a single set.
+
+        :param values: An iterable of iterables.
+        :return: A single set containing all the values from the input. Elements are in the order they appear in the
+            input.
+        """
+        return {item for subset in values for item in subset}
+
+    @staticmethod
+    def invert_dictionary(dictionary: dict[Any, Any], list_values: bool = False) \
+            -> dict[Any, Any] | dict[Any, list[Any]]:
+        """
+        Invert a dictionary, swapping keys and values. Note that the values of the input dictionary must be hashable.
+
+        :param dictionary: A dictionary to invert.
+        :param list_values: If false, keys that share the same value in the input dictionary will be overwritten in
+            the output dictionary so that only the last key remains. If true, the values of the output dictionary will
+            be lists where input keys that share the same value will be stored together.
+        :return: A new dictionary with the keys and values swapped.
+        """
+        if list_values:
+            inverted = {}
+            for key, value in dictionary.items():
+                inverted.setdefault(value, []).append(key)
+            return inverted
+        return {value: key for key, value in dictionary.items()}