sapiopycommons 2025.8.14a703__py3-none-any.whl → 2026.1.22a847__py3-none-any.whl

sapiopycommons/callbacks/callback_util.py

@@ -858,9 +858,9 @@ class CallbackUtil:
             raise SapioException("No records provided.")
         data_type: str = AliasUtil.to_singular_data_type_name(records)
         if index_field is not None:
-            field_map_list: list[FieldMap] = self.__get_indexed_field_maps(records, index_field)
+            field_map_list: list[FieldMap] = self.__get_indexed_field_maps(records, index_field, True)
         else:
-            field_map_list: list[FieldMap] = AliasUtil.to_field_map_list(records)
+            field_map_list: list[FieldMap] = AliasUtil.to_field_map_list(records, True)
 
         # Convert the group_by parameter to a field name.
         if group_by is not None:
@@ -882,6 +882,18 @@ class CallbackUtil:
             temp_dt = self.__temp_dt_from_field_names(data_type, fields, None, default_modifier, field_modifiers)
         temp_dt.record_image_assignable = bool(image_data)
 
+        # PR-47894: If the RecordId field is not present in the layout, then it should not be included in the field
+        # maps, as otherwise selection list fields can break.
+        remove_record_id: bool = True
+        for field_def in temp_dt.get_field_def_list():
+            if field_def.data_field_name == "RecordId":
+                remove_record_id = False
+                break
+        if remove_record_id:
+            for field_map in field_map_list:
+                if "RecordId" in field_map:
+                    del field_map["RecordId"]
+
         # Send the request to the user.
         request = TableEntryDialogRequest(title, msg, temp_dt, field_map_list,
                                           record_image_data_list=image_data, group_by_field=group_by,
@@ -1928,7 +1940,8 @@ class CallbackUtil:
         self.write_file(zip_name, FileUtil.zip_files(files))
 
     @staticmethod
-    def __get_indexed_field_maps(records: Iterable[SapioRecord], index_field: str) -> list[FieldMap]:
+    def __get_indexed_field_maps(records: Iterable[SapioRecord], index_field: str, include_record_id: bool = False) \
+            -> list[FieldMap]:
         """
         For dialogs that accept multiple records, we may want to be able to match the returned results back to the
         records that they're for. In this case, we need to add an index to each record so that we can match them back
@@ -1938,12 +1951,13 @@ class CallbackUtil:
         :param records: The records to return indexed field maps of.
         :param index_field: The name of the field to use as the index. Make sure that this field doesn't exist on the
             records, as then it will overwrite the existing value.
+        :param include_record_id: Whether to include the RecordId field in the field maps.
         :return: A list of field maps for the records, with an index field added to each. The value of the index on
             each field map is the record's record ID (even if it's a record model with a negative ID).
         """
         ret_val: list[FieldMap] = []
         for record in records:
-            field_map: FieldMap = AliasUtil.to_field_map(record)
+            field_map: FieldMap = AliasUtil.to_field_map(record, include_record_id)
             field_map[index_field] = AliasUtil.to_record_id(record)
             ret_val.append(field_map)
         return ret_val
@@ -1984,7 +1998,10 @@ class CallbackUtil:
             if field_def.key_field:
                 field_def = modifier.modify_field(field_def)
             builder.add_field(field_def, column, span)
-        return builder.get_temporary_data_type()
+        # PR-47917: Set fill_view to false on the layout of temp data types created by CallbackUtil.
+        temp_dt = builder.get_temporary_data_type()
+        temp_dt.data_type_layout.fill_view = False
+        return temp_dt
 
     def __temp_dt_from_field_names(self, data_type: str, fields: Iterable[FieldIdentifier | FieldFilterCriteria],
                                    column_positions: dict[str, tuple[int, int]] | None,
@@ -2055,8 +2072,10 @@ class CallbackUtil:
                 modifier: FieldModifier = field_modifiers.get(field_name, default_modifier)
                 builder.add_field(modifier.modify_field(field_def), current_column, span)
                 current_column += span
-
-        return builder.get_temporary_data_type()
+        # PR-47917: Set fill_view to false on the layout of temp data types created by CallbackUtil.
+        temp_dt = builder.get_temporary_data_type()
+        temp_dt.data_type_layout.fill_view = False
+        return temp_dt
 
     # CR-47309: Allow layouts to be provided in place of field names for record dialogs.
     def __temp_dt_from_layout(self, data_type: str, layout: DataTypeLayoutIdentifier,

sapiopycommons/eln/experiment_handler.py

@@ -206,12 +206,11 @@ class ExperimentHandler:
         else:
             user = context
             context = None
-        if context is not None and context.eln_experiment is not None and experiment is None:
-            experiment = context.eln_experiment
         # FR-46495 - Allow the init function of ExperimentHandler to take in an ElnExperiment that is separate from the
         # context.
         # CR-37038 - Allow other experiment object types to be provided. Convert them all down to ElnExperiment.
-        if (context is None or context.eln_experiment is None) and experiment is not None:
+        # PR-47793 - Fix cases where both a SapioWebhookContext and an experiment parameter are provided.
+        if experiment is not None:
             eln_manager = DataMgmtServer.get_eln_manager(user)
             # If this object is already an ElnExperiment, do nothing.
             if isinstance(experiment, ElnExperiment):
@@ -227,13 +226,19 @@ class ExperimentHandler:
                     raise SapioException(f"No experiment with notebook ID {notebook_id} located in the system.")
             # If this object is a record, assume it is an experiment record that we can query the system with.
             else:
-                record_id: int = AliasUtil.to_record_ids([experiment])[0]
+                record_id: int = AliasUtil.to_record_id(experiment)
                 experiment: ElnExperiment = eln_manager.get_eln_experiment_by_record_id(record_id)
                 if not experiment:
                     raise SapioException(f"No experiment with record ID {record_id} located in the system.")
+        elif context is not None and context.eln_experiment is not None:
+            experiment = context.eln_experiment
+
         if experiment is None:
             raise SapioException("Cannot initialize ExperimentHandler. No ELN Experiment found in the provided "
                                  "parameters.")
+        elif not isinstance(experiment, ElnExperiment):
+            raise SapioException("Cannot initialize ExperimentHandler. The experiment variable is not an "
+                                 "ElnExperiment!")
 
         return user, context, experiment
 
@@ -1425,7 +1430,9 @@ class ExperimentHandler:
         :return: The map of options for the input step.
         """
         step: ElnEntryStep = self.get_step(step)
-        if step not in self._step_options:
+        # PR-47796: Fix the get_step_options function making a webservice query every time it is called instead of
+        # properly checking its cache of entry options.
+        if step.get_id() not in self._step_options:
             self._step_options.update(ExperimentReportUtil.get_experiment_entry_options(self.user,
                                                                                         self.get_all_steps()))
         return self._step_options[step.get_id()]

sapiopycommons/files/file_util.py

@@ -1,4 +1,7 @@
+import gzip
 import io
+import tarfile
+import time
 import warnings
 import zipfile
 
@@ -322,7 +325,7 @@ class FileUtil:
     @staticmethod
     def zip_files(files: dict[str, str | bytes]) -> bytes:
         """
-        Create a zip file for a collection of files.
+        Create a .zip file for a collection of files.
 
         :param files: A dictionary of file name to file data as a string or bytes.
         :return: The bytes for a zip file containing the input files.
@@ -335,6 +338,130 @@ class FileUtil:
             # throws an I/O exception.
             return zip_buffer.getvalue()
 
+    # FR-47422: Add a function for unzipping files that may have been zipped by the above function.
+    @staticmethod
+    def unzip_files(zip_file: bytes) -> dict[str, bytes]:
+        """
+        Decompress a .zip file from an in-memory bytes object and extracts all files into a dictionary.
+
+        :param zip_file: The bytes of the zip file to be decompressed.
+        :return: A dictionary of file name to file bytes for each file in the zip.
+        """
+        extracted_files: dict[str, bytes] = {}
+        with io.BytesIO(zip_file) as zip_buffer:
+            with zipfile.ZipFile(zip_buffer, "r") as zip_file:
+                for file_name in zip_file.namelist():
+                    with zip_file.open(file_name) as file:
+                        extracted_files[file_name] = file.read()
+        return extracted_files
+
+    # FR-47422: Add functions for compressing and decompressing .gz, .tar, and .tar.gz files.
+    @staticmethod
+    def gzip_file(file_data: bytes | str) -> bytes:
+        """
+        Create a .gz file for a single file.
+
+        :param file_data: The file data to be compressed as bytes or a string.
+        :return: The bytes of the gzip-compressed file.
+        """
+        return gzip.compress(file_data.encode() if isinstance(file_data, str) else file_data)
+
+    @staticmethod
+    def ungzip_file(gzip_file: bytes) -> bytes:
+        """
+        Decompress a .gz file.
+
+        :param gzip_file: The bytes of the gzip-compressed file.
+        :return: The decompressed file data as bytes.
+        """
+        return gzip.decompress(gzip_file)
+
+    @staticmethod
+    def tar_files(files: dict[str, str | bytes]) -> bytes:
+        """
+        Create a .tar file for a collection of files.
+
+        :param files: A dictionary of file name to file data as a string or bytes.
+        :return: The bytes for a tar file containing the input files.
+        """
+        with io.BytesIO() as tar_buffer:
+            with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
+                for name, data in files.items():
+                    if isinstance(data, str):
+                        data: bytes = data.encode('utf-8')
+
+                    tarinfo = tarfile.TarInfo(name=name)
+                    tarinfo.size = len(data)
+                    tarinfo.mtime = int(time.time())
+
+                    with io.BytesIO(data) as file:
+                        tar.addfile(tarinfo=tarinfo, fileobj=file)
+
+            tar_buffer.seek(0)
+            return tar_buffer.getvalue()
+
+    @staticmethod
+    def untar_files(tar_file: bytes) -> dict[str, bytes]:
+        """
+        Decompress a .tar file from an in-memory bytes object and extracts all files into a dictionary.
+
+        :param tar_file: The bytes of the tar file to be decompressed.
+        :return: A dictionary of file name to file bytes for each file in the tar.
+        """
+        extracted_files: dict[str, bytes] = {}
+        with io.BytesIO(tar_file) as tar_buffer:
+            with tarfile.open(fileobj=tar_buffer, mode="r") as tar:
+                for member in tar.getmembers():
+                    if member.isfile():
+                        file_obj = tar.extractfile(member)
+                        if file_obj:
+                            with file_obj:
+                                extracted_files[member.name] = file_obj.read()
+        return extracted_files
+
+    @staticmethod
+    def tar_gzip_files(files: dict[str, str | bytes]) -> bytes:
+        """
+        Create a .tar.gz file for a collection of files.
+
+        :param files: A dictionary of file name to file data as a string or bytes.
+        :return: The bytes for a tar.gz file containing the input files.
+        """
+        with io.BytesIO() as tar_buffer:
+            with tarfile.open(fileobj=tar_buffer, mode="w:gz") as tar:
+                for name, data in files.items():
+                    if isinstance(data, str):
+                        data: bytes = data.encode('utf-8')
+
+                    tarinfo = tarfile.TarInfo(name=name)
+                    tarinfo.size = len(data)
+                    tarinfo.mtime = int(time.time())
+
+                    with io.BytesIO(data) as file:
+                        tar.addfile(tarinfo=tarinfo, fileobj=file)
+
+            tar_buffer.seek(0)
+            return tar_buffer.getvalue()
+
+    @staticmethod
+    def untar_gzip_files(tar_gzip_file: bytes) -> dict[str, bytes]:
+        """
+        Decompress a .tar.gz file from an in-memory bytes object and extracts all files into a dictionary.
+
+        :param tar_gzip_file: The bytes of the tar.gz file to be decompressed.
+        :return: A dictionary of file name to file bytes for each file in the tar.gz
+        """
+        extracted_files: dict[str, bytes] = {}
+        with io.BytesIO(tar_gzip_file) as tar_buffer:
+            with tarfile.open(fileobj=tar_buffer, mode="r:gz") as tar:
+                for member in tar.getmembers():
+                    if member.isfile():
+                        file_obj = tar.extractfile(member)
+                        if file_obj:
+                            with file_obj:
+                                extracted_files[member.name] = file_obj.read()
+        return extracted_files
+
     # Deprecated functions:
 
     # FR-46097 - Add write file request shorthand functions to FileUtil.

sapiopycommons/files/temp_files.py

@@ -0,0 +1,82 @@
+import os
+import shutil
+import tempfile
+from typing import Callable, Any
+
+
+# FR-47422: Created class.
+class TempFileHandler:
+    """
+    A utility class to manage temporary files and directories.
+    """
+    directories: list[str]
+    files: list[str]
+
+    def __init__(self):
+        self.directories = []
+        self.files = []
+
+    def create_temp_directory(self) -> str:
+        """
+        Create a temporary directory.
+
+        :return: The path to a newly created temporary directory.
+        """
+        directory: str = tempfile.mkdtemp()
+        self.directories.append(directory)
+        return directory
+
+    def create_temp_file(self, data: str | bytes, suffix: str = "") -> str:
+        """
+        Create a temporary file with the specified data and optional suffix.
+
+        :param data: The data to write to the temporary file.
+        :param suffix: An optional suffix for the temporary file.
+        :return: The path to a newly created temporary file containing the provided data.
+        """
+        mode: str = 'w' if isinstance(data, str) else 'wb'
+        with tempfile.NamedTemporaryFile(mode=mode, suffix=suffix, delete=False) as tmp_file:
+            tmp_file.write(data)
+            file_path: str = tmp_file.name
+            self.files.append(file_path)
+        return file_path
+
+    def create_temp_file_from_func(self, func: Callable, params: dict[str, Any], suffix: str = "",
+                                   is_binary: bool = True) -> str:
+        """
+        Create a temporary file and populate it using the provided function. The function should accept parameters as
+        specified in the `params` dictionary.
+
+        :param func: The function to call with the temporary file path that will populate the file.
+        :param params: Keyword arguments to pass to the function. If "<NEW_FILE>" is used as a value, it will be
+            replaced with the temporary file object. If "<NEW_FILE_PATH>" is used as a value, it will be replaced with
+            the temporary file path.
+        :param suffix: An optional suffix for the temporary file.
+        :param is_binary: Whether to open the temporary file in binary mode.
+        :return: The path to the newly created temporary file.
+        """
+        mode: str = 'wb' if is_binary else 'w'
+        with tempfile.NamedTemporaryFile(mode, suffix=suffix, delete=False) as tmp_file:
+            for key, value in params.items():
+                if value == "<NEW_FILE>":
+                    params[key] = tmp_file
+                elif value == "<NEW_FILE_PATH>":
+                    params[key] = tmp_file.name
+            func(**params)
+            file_path: str = tmp_file.name
+            self.files.append(file_path)
+        return file_path
+
+    def cleanup(self) -> None:
+        """
+        Delete all temporary files and directories created by this handler.
+        """
+        for directory in self.directories:
+            if os.path.exists(directory):
+                shutil.rmtree(directory)
+        self.directories.clear()
+
+        for file_path in self.files:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+        self.files.clear()

sapiopycommons/general/aliases.py

@@ -223,9 +223,12 @@ class AliasUtil:
             # macros get translated to valid field values.
             fields: FieldMap = {f: record.fields.get(f) for f in record.fields}
         # PR-47457: Only include the record ID if the caller requests it, since including the record ID can break
-        # callbacks in certain circumstances if the record ID is negative.
+        # callbacks in certain circumstances.
+        # PR-47894: Also remove the RecordId key if it exists and the caller doesn't want it included.
         if include_record_id:
             fields["RecordId"] = AliasUtil.to_record_id(record)
+        elif "RecordId" in fields:
+            del fields["RecordId"]
         return fields
 
     @staticmethod

sapiopycommons/general/macros.py

@@ -0,0 +1,172 @@
+
+import re
+from datetime import datetime, timedelta, timezone
+
+from sapiopycommons.general.exceptions import SapioException
+
+date_macro_pattern: str = (
+    r"@(?:today|yesterday|thisweek|"
+    r"nextmonth|thismonth|lastmonth|"
+    r"nextyear|thisyear|lastyear|"
+    r"month(?:january|february|march|april|may|june|july|august|september|october|november|december)|"
+    r"last\d+days|next\d+days)"
+)
+"""A regular expression that can be used to determine whether a given value matches one of the supported date macros."""
+
+date_macro_values: list[str] = [
+    "@today", "@yesterday", "@thisweek",
+    "@nextmonth", "@thismonth", "@lastmonth",
+    "@nextyear", "@thisyear", "@lastyear",
+    "@monthjanuary", "@monthfebruary", "@monthmarch", "@monthapril", "@monthmay", "@monthjune", "@monthjuly",
+    "@monthaugust", "@monthseptember", "@monthoctober", "@monthnovember", "@monthdecember",
+    "@next_days", "@last_days"
+]
+"""A list of the supported date macros. For @next_days and @last_days, the underscore is expected to be replaced with
+an integer."""
+
+
+class MacroParser:
+    """
+    A utility class for parsing macros used in the Sapio platform.
+    """
+    _reg_month = re.compile(r"@\w*month\w*")
+    _reg_digits = re.compile(r"@\w*(\d+)\w*")
+    _reg_last_days = re.compile(r"@last(\d+)days")
+    _reg_next_days = re.compile(r"@next(\d+)days")
+
+    @staticmethod
+    def _now() -> datetime:
+        """
+        :return: A datetime object for the current time in UTC.
+        """
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def _dates_to_timestamps(a: datetime, b: datetime) -> tuple[int, int]:
+        """
+        Convert the given datetimes to epoch-millisecond timestamps on the start of the first date
+        and the end of the second date.
+
+        :param a: A datetime object.
+        :param b: A datetime object.
+        :return: A tuple containing the start and end timestamps in milliseconds since the epoch.
+        """
+        # The start of the date on the first datetime.
+        a = a.replace(hour=0, minute=0, second=0, microsecond=0)
+        # The end of the date on the second datetime.
+        b = b.replace(hour=23, minute=59, second=59, microsecond=999000)
+        return int(a.timestamp() * 1000), int(b.timestamp() * 1000)
+
+    @classmethod
+    def parse_date_macro(cls, macro: str) -> tuple[int, int]:
+        """
+        Convert a date macro string into a range of epoch millisecond timestamps.
+        All macros are considered from the current time in UTC.
+
+        :param macro: A valid date macro string. If an invalid macro is provided, an exception will be raised.
+        :return: A tuple containing the start and end timestamps in milliseconds since the epoch for the given macro.
+            The returned range is inclusive; the first value is the first millisecond of the starting date
+            (00:00:00.000) and the last value is the final millisecond of the ending date (23:59:59.999).
+        """
+        if macro is None or not macro.strip():
+            raise SapioException(f"Invalid macro. None or empty/blank string provided.")
+        macro = macro.strip().lower()
+
+        now: datetime = cls._now()
+
+        # --- @today: 00:00:00.000 to 23:59:59.999 today ---
+        if macro == "@today":
+            return cls._dates_to_timestamps(now, now)
+
+        # --- @yesterday: 00:00:00.000 to 23:59:59.999 yesterday ---
+        if macro == "@yesterday":
+            yesterday: datetime = now - timedelta(days=1)
+            return cls._dates_to_timestamps(yesterday, yesterday)
+
+        # --- @thisweek: Sunday -> Saturday (inclusive) ---
+        if macro == "@thisweek":
+            weekday: int = now.weekday()  # Monday=0 ... Sunday=6
+            # TODO: Some way to control what the first day of the week is considered?
+            #  +2 = Saturday
+            #  +1 = Sunday
+            #  +0 = Monday
+            days_since_sunday: int = (weekday + 1) % 7
+            sunday: datetime = now - timedelta(days=days_since_sunday)
+            saturday: datetime = sunday + timedelta(days=6)
+            return cls._dates_to_timestamps(sunday, saturday)
+
+        # --- last/next N days ---
+        if cls._reg_digits.fullmatch(macro):
+            # --- @lastNdays ---
+            if m := cls._reg_last_days.fullmatch(macro):
+                days = int(m.group(1))
+                return cls._dates_to_timestamps(now - timedelta(days=days), now)
+
+            # --- @nextNdays ---
+            if m := cls._reg_next_days.fullmatch(macro):
+                days = int(m.group(1))
+                return cls._dates_to_timestamps(now, now + timedelta(days=days))
+
+            raise SapioException(f"Invalid macro: {macro}")
+
+        # --- Month macros ---
+        if cls._reg_month.fullmatch(macro):
+            year: int = now.year
+            month: int = now.month
+
+            if macro == "@lastmonth":
+                month -= 1
+                if month == 0:
+                    year -= 1
+                    month = 12
+            elif macro == "@nextmonth":
+                month += 1
+                if month == 13:
+                    year += 1
+                    month = 1
+            # @thismonth uses the current month and year, so no replacement needed.
+            elif macro != "@thismonth":
+                month_map: dict[str, int] = {
+                    "@monthjanuary": 1,
+                    "@monthfebruary": 2,
+                    "@monthmarch": 3,
+                    "@monthapril": 4,
+                    "@monthmay": 5,
+                    "@monthjune": 6,
+                    "@monthjuly": 7,
+                    "@monthaugust": 8,
+                    "@monthseptember": 9,
+                    "@monthoctober": 10,
+                    "@monthnovember": 11,
+                    "@monthdecember": 12,
+                }
+                if macro in month_map:
+                    month = month_map[macro]
+                else:
+                    raise SapioException(f"Invalid macro: {macro}")
+
+            month_start: datetime = now.replace(year=year, month=month, day=1)
+            # Find the first day of next month.
+            if month == 12:
+                next_month = datetime(year + 1, 1, 1, tzinfo=timezone.utc)
+            else:
+                next_month = datetime(year, month + 1, 1, tzinfo=timezone.utc)
+            # Then subtract one day to find the last day of the start month.
+            month_end: datetime = next_month - timedelta(days=1)
+
+            return cls._dates_to_timestamps(month_start, month_end)
+
+        # --- Year macros ---
+        if macro in ("@thisyear", "@lastyear", "@nextyear"):
+            year: int = now.year
+            if macro == "@lastyear":
+                year -= 1
+            elif macro == "@nextyear":
+                year += 1
+            # No change in year needed for @thisyear.
+
+            year_start: datetime = now.replace(year=year, month=1, day=1)
+            year_end: datetime = year_start.replace(year=year_start.year + 1) - timedelta(days=1)
+            return cls._dates_to_timestamps(year_start, year_end)
+
+        raise SapioException(f"Invalid macro: {macro}")