sapiopycommons 2024.8.28a313__py3-none-any.whl → 2024.8.28a314__py3-none-any.whl

sapiopycommons/files/file_bridge.py

@@ -16,7 +16,8 @@ class FileBridge:
         Read a file from FileBridge.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to read the file from.
         :param base64_decode: If true, base64 decode the file. Files are by default base64 encoded when retrieved from
             FileBridge.
@@ -42,7 +43,8 @@ class FileBridge:
         Write a file to FileBridge.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to write the file to. If a file already exists at the given path then the file is
             overwritten.
         :param file_data: A string or bytes of the file to be written.
@@ -63,9 +65,10 @@ class FileBridge:
         List the contents of a FileBridge directory.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to read the directory from.
-        :return: A list of name of files and folders in the directory.
+        :return: A list of names of files and folders in the directory.
         """
         sub_path = '/ext/filebridge/listDirectory'
         params = {
@@ -77,7 +80,7 @@ class FileBridge:
 
         response_body: list[str] = response.json()
         path_length = len(f"bridge://{bridge_name}/")
-        return [urllib.parse.unquote(value[path_length:]) for value in response_body]
+        return [urllib.parse.unquote(value)[path_length:] for value in response_body]
 
     @staticmethod
     def create_directory(context: SapioWebhookContext | SapioUser, bridge_name: str, file_path: str) -> None:
@@ -85,7 +88,8 @@ class FileBridge:
         Create a new directory in FileBridge.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to create the directory at. If a directory already exists at the given path then an
             exception is raised.
         """
@@ -103,7 +107,8 @@ class FileBridge:
         Delete an existing file in FileBridge.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to the file to delete.
         """
         sub_path = '/ext/filebridge/deleteFile'
@@ -111,7 +116,7 @@ class FileBridge:
             'Filepath': f"bridge://{bridge_name}/{file_path}"
         }
         user: SapioUser = context if isinstance(context, SapioUser) else context.user
-        response = user.post(sub_path, params=params)
+        response = user.delete(sub_path, params=params)
         user.raise_for_status(response)
 
     @staticmethod
@@ -120,7 +125,8 @@ class FileBridge:
         Delete an existing directory in FileBridge.
 
         :param context: The current webhook context or a user object to send requests from.
-        :param bridge_name: The name of the bridge to use.
+        :param bridge_name: The name of the bridge to use. This is the "connection name" in the
+            file bridge configurations.
         :param file_path: The path to the directory to delete.
         """
         sub_path = '/ext/filebridge/deleteDirectory'
@@ -128,5 +134,5 @@ class FileBridge:
             'Filepath': f"bridge://{bridge_name}/{file_path}"
         }
         user: SapioUser = context if isinstance(context, SapioUser) else context.user
-        response = user.post(sub_path, params=params)
+        response = user.delete(sub_path, params=params)
         user.raise_for_status(response)

sapiopycommons/files/file_bridge_handler.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+from abc import abstractmethod, ABC
+
+from sapiopycommons.files.file_bridge import FileBridge
+from sapiopylib.rest.User import SapioUser
+from sapiopylib.rest.pojo.webhook.WebhookContext import SapioWebhookContext
+
+
+class FileBridgeHandler:
+    """
+    The FileBridgeHandler provides caching of the results of file bridge endpoint calls while also containing quality
+    of life functions for common file bridge actions.
+    """
+    user: SapioUser
+    __bridge: str
+    __file_cache: dict[str, bytes]
+    """A cache of file paths to file bytes."""
+    __files: dict[str, File]
+    """A cache of file paths to File objects."""
+    __dir_cache: dict[str, list[str]]
+    """A cache of directory file paths to the names of the files or nested directories within it."""
+    __directories: dict[str, Directory]
+    """A cache of directory file paths to Directory objects."""
+
+    def __init__(self, context: SapioWebhookContext | SapioUser, bridge_name: str):
+        """
+        :param context: The current webhook context or a user object to send requests from.
+        :param bridge_name: The name of the bridge to communicate with. This is the "connection name" in the
+            file bridge configurations.
+        """
+        self.user = context if isinstance(context, SapioUser) else context.user
+        self.__bridge = bridge_name
+        self.__file_cache = {}
+        self.__files = {}
+        self.__dir_cache = {}
+        self.__directories = {}
+
+    @property
+    def connection_name(self) -> str:
+        return self.__bridge
+
+    def clear_caches(self) -> None:
+        """
+        Clear the file and directory caches of this handler.
+        """
+        self.__file_cache.clear()
+        self.__files.clear()
+        self.__dir_cache.clear()
+        self.__directories.clear()
+
+    def read_file(self, file_path: str, base64_decode: bool = True) -> bytes:
+        """
+        Read a file from FileBridge. The bytes of the given file will be cached so that any subsequent reads of this
+        file will not make an additional webservice call.
+
+        :param file_path: The path to read the file from.
+        :param base64_decode: If true, base64 decode the file. Files are by default base64 encoded when retrieved from
+            FileBridge.
+        :return: The bytes of the file.
+        """
+        if file_path in self.__file_cache:
+            return self.__file_cache[file_path]
+        file_bytes: bytes = FileBridge.read_file(self.user, self.__bridge, file_path, base64_decode)
+        self.__file_cache[file_path] = file_bytes
+        return file_bytes
+
+    def write_file(self, file_path: str, file_data: bytes | str) -> None:
+        """
+        Write a file to FileBridge. The bytes of the given file will be cached so that any subsequent reads of this
+        file will not make an additional webservice call.
+
+        :param file_path: The path to write the file to. If a file already exists at the given path then the file is
+            overwritten.
+        :param file_data: A string or bytes of the file to be written.
+        """
+        FileBridge.write_file(self.user, self.__bridge, file_path, file_data)
+        self.__file_cache[file_path] = file_data if isinstance(file_data, bytes) else file_data.encode()
+
+        # Find the directory path to this file and the name of the file. Add the file name to the cached list of
+        # files for the directory, assuming we have this directory cached and the file isn't already in it.
+        last_slash: int = file_path.rfind("/")
+        dir_path: str = file_path[:last_slash]
+        file_name: str = file_path[last_slash + 1:]
+        if dir_path in self.__dir_cache and file_path not in self.__dir_cache[dir_path]:
+            self.__dir_cache[dir_path].append(file_name)
+
+    def delete_file(self, file_path: str) -> None:
+        """
+        Delete an existing file in FileBridge. If this file is in the cache, it will also be deleted from the cache.
+
+        :param file_path: The path to the file to delete.
+        """
+        FileBridge.delete_file(self.user, self.__bridge, file_path)
+        if file_path in self.__file_cache:
+            self.__file_cache.pop(file_path)
+        if file_path in self.__files:
+            self.__files.pop(file_path)
+
+    def list_directory(self, file_path: str) -> list[str]:
+        """
+        List the contents of a FileBridge directory. The contents of this directory will be cached so that any
+        subsequent lists of this directory will not make an additional webservice call.
+
+        :param file_path: The path to read the directory from.
+        :return: A list of names of files and folders in the directory.
+        """
+        if file_path in self.__dir_cache:
+            return self.__dir_cache[file_path]
+        files: list[str] = FileBridge.list_directory(self.user, self.__bridge, file_path)
+        self.__dir_cache[file_path] = files
+        return files
+
+    def create_directory(self, file_path: str) -> None:
+        """
+        Create a new directory in FileBridge. This new directory will be added to the cache as empty so that listing
+        the same directory does not make an additional webservice call.
+
+        :param file_path: The path to create the directory at. If a directory already exists at the given path then an
+            exception is raised.
+        """
+        FileBridge.create_directory(self.user, self.__bridge, file_path)
+        # This directory was just created, so we know it's empty.
+        self.__dir_cache[file_path] = []
+
+    def delete_directory(self, file_path: str) -> None:
+        """
+        Delete an existing directory in FileBridge. If this directory is in the cache, it will also be deleted
+        from the cache.
+
+        :param file_path: The path to the directory to delete.
+        """
+        FileBridge.delete_directory(self.user, self.__bridge, file_path)
+        if file_path in self.__dir_cache:
+            self.__dir_cache.pop(file_path)
+        if file_path in self.__directories:
+            self.__directories.pop(file_path)
+
+    def is_file(self, file_path: str) -> bool:
+        """
+        Determine if the given file path points to a file or a directory. This is achieved by trying to call
+        list_directory on the given file path. If an exception is thrown, that's because the function was called
+        on a file. If no exception is thrown, then we know that this is a directory, and we have now also cached
+        the contents of that directory if it wasn't cached already.
+
+        :param file_path: A file path.
+        :return: True if the file path points to a file. False if it points to a directory.
+        """
+        try:
+            self.list_directory(file_path)
+            return False
+        except Exception:
+            return True
+
+    def move_file(self, move_from: str, move_to: str, old_name: str, new_name: str | None = None) -> None:
+        """
+        Move a file from one location to another within File Bridge. This is done be reading the file into memory,
+        writing a copy of the file in the new location, then deleting the original file.
+
+        :param move_from: The path to the current location of the file.
+        :param move_to: The path to move the file to.
+        :param old_name: The current name of the file.
+        :param new_name: The name that the file should have after it is moved. if this is not provided, then the new
+            name will be the same as the old name.
+        """
+        if not new_name:
+            new_name = old_name
+
+        # Read the file into memory.
+        file_bytes: bytes = self.read_file(move_from + "/" + old_name)
+        # Write the file into the new location.
+        self.write_file(move_to + "/" + new_name, file_bytes)
+        # Delete the file from the old location. We do this last in case the write call fails.
+        self.delete_file(move_from + "/" + old_name)
+
+    def get_file_object(self, file_path: str) -> File:
+        """
+        Get a File object from a file path. This object can be used to get the contents of the file at this path
+        and traverse up the file hierarchy to the directory that the file is contained within.
+
+        There is no guarantee that this file actually exists within the current file bridge connection when it is
+        constructed. If the file doesn't exist, then retrieving its contents will fail.
+
+        :param file_path: A file path.
+        :return: A File object constructed form the given file path.
+        """
+        if file_path in self.__files:
+            return self.__files[file_path]
+        file = File(self, file_path)
+        self.__files[file_path] = file
+        return file
+
+    def get_directory_object(self, file_path: str) -> Directory | None:
+        """
+        Get a Directory object from a file path. This object can be used to traverse up and down the file hierarchy
+        by going up to the parent directory that this directory is contained within or going down to the contents of
+        this directory.
+
+        There is no guarantee that this directory actually exists within the current file bridge connection when it is
+        constructed. If the directory doesn't exist, then retrieving its contents will fail.
+
+        :param file_path: A file path.
+        :return: A Directory object constructed form the given file path.
+        """
+        if file_path is None:
+            return None
+        if file_path in self.__directories:
+            return self.__directories[file_path]
+        directory = Directory(self, file_path)
+        self.__directories[file_path] = directory
+        return directory
+
+
+class FileBridgeObject(ABC):
+    """
+    A FileBridgeObject is either a file or a directory that is contained within file bridge. Every object has a
+    name and a parent directory that it is contained within (unless the object is located in the bridge root, in
+    which case the parent is None). From the name and the parent, a path can be constructed to that object.
+    """
+    _handler: FileBridgeHandler
+    name: str
+    parent: Directory | None
+
+    def __init__(self, handler: FileBridgeHandler, file_path: str):
+        self._handler = handler
+
+        name, root = split_path(file_path)
+        self.name = name
+        self.parent = handler.get_directory_object(root)
+
+    @abstractmethod
+    def is_file(self) -> bool:
+        """
+        :return: True if this object is a file. False if it is a directory.
+        """
+        pass
+
+    def get_path(self) -> str:
+        """
+        :return: The file path that leads to this object.
+        """
+        if self.parent is None:
+            return self.name
+        return self.parent.get_path() + "/" + self.name
+
+
+class File(FileBridgeObject):
+    def __init__(self, handler: FileBridgeHandler, file_path: str):
+        """
+        :param handler: A FileBridgeHandler for the connection that this file came from.
+        :param file_path: The path to this file.
+        """
+        super().__init__(handler, file_path)
+
+    @property
+    def contents(self) -> bytes:
+        """
+        :return: The bytes of this file.
+            This pulls from the cache of this object's related FileBridgeHandler.
+        """
+        return self._handler.read_file(self.get_path())
+
+    def is_file(self) -> bool:
+        return True
+
+
+class Directory(FileBridgeObject):
+    def __init__(self, handler: FileBridgeHandler, file_path: str):
+        """
+        :param handler: A FileBridgeHandler for the connection that this directory came from.
+        :param file_path: The path to this directory.
+        """
+        super().__init__(handler, file_path)
+
+    @property
+    def contents(self) -> dict[str, FileBridgeObject]:
+        """
+        :return: A dictionary of object names to the objects (Files or Directories) contained within this Directory.
+            This pulls from the cache of this object's related FileBridgeHandler.
+        """
+        contents: dict[str, FileBridgeObject] = {}
+        path: str = self.get_path()
+        for name in self._handler.list_directory(path):
+            file_path: str = path + "/" + name
+            if self._handler.is_file(file_path):
+                contents[name] = self._handler.get_file_object(file_path)
+            else:
+                contents[name] = self._handler.get_directory_object(file_path)
+        return contents
+
+    def is_file(self) -> bool:
+        return False
+
+    def get_files(self) -> dict[str, File]:
+        """
+        :return: A mapping of file name to File for every file in this Directory.
+            This pulls from the cache of this object's related FileBridgeHandler.
+        """
+        return {x: y for x, y in self.contents.items() if y.is_file()}
+
+    def get_directories(self) -> dict[str, Directory]:
+        """
+        :return: A mapping of directory name to Directory for every directory in this Directory.
+            This pulls from the cache of this object's related FileBridgeHandler.
+        """
+        return {x: y for x, y in self.contents.items() if not y.is_file()}
+
+
+def split_path(file_path: str) -> (str, str):
+    """
+    :param file_path: A file path where directories are separated the "/" characters.
+    :return: A tuple of two strings that splits the path on its last slash. The first string is the name of the
+        file/directory at the given file path and the second string is the location to that file.
+    """
+    last_slash: int = file_path.rfind("/")
+    if last_slash == -1:
+        return file_path, None
+    return file_path[last_slash + 1:], file_path[:last_slash]

sapiopycommons/files/file_util.py

@@ -21,7 +21,7 @@ class FileUtil:
     """
     @staticmethod
     def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
-                     seperator: str = ",") -> tuple[list[dict[str, str]], list[list[str]]]:
+                     seperator: str = ",", *, encoding: str | None = None) -> tuple[list[dict[str, str]], list[list[str]]]:
         """
         Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
         must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -34,13 +34,17 @@ class FileUtil:
             meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
             is assumed to be the header row.
         :param seperator: The character that separates cells in the table.
+        :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
+            contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
+            ISO-8859-1 as the encoding.
         :return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
             that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
             If the header row index is 0 or None, this list will be empty.
         """
         # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
         # while the second is the body of the file below the header row.
-        file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator)
+        file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
+                                                               encoding=encoding)
         # Parse the metadata from above the header row index into a list of lists.
         metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
         # Parse the data from the file body into a list of dicts.
@@ -74,8 +78,8 @@ class FileUtil:
         return rows, metadata
 
     @staticmethod
-    def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",") \
-            -> tuple[DataFrame, DataFrame | None]:
+    def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
+                           *, encoding: str | None = None) -> tuple[DataFrame, DataFrame | None]:
         """
         Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
         cells, all the rows in the file must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -86,6 +90,9 @@ class FileUtil:
             meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
             is assumed to be the header row.
         :param seperator: The character that separates cells in the table.
+        :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
+            contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
+            ISO-8859-1 as the encoding.
         :return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
             metadata from above the header row, or None if there is no metadata.
         """
@@ -97,13 +104,13 @@ class FileUtil:
                 # can throw off the header row index.
                 file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
                                                 skiprows=lambda x: x >= header_row_index,
-                                                skip_blank_lines=False, sep=seperator)
+                                                skip_blank_lines=False, sep=seperator, encoding=encoding)
         with io.BytesIO(file_bytes) as file_io:
             # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
             # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
             # string.
             file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
-                                                   skip_blank_lines=False, sep=seperator)
+                                                   skip_blank_lines=False, sep=seperator, encoding=encoding)
 
         return file_body, file_metadata
 

sapiopycommons/files/file_validator.py

@@ -4,12 +4,15 @@ from abc import abstractmethod
 from typing import Any
 
 from sapiopylib.rest.User import SapioUser
+from sapiopylib.rest.pojo.CustomReport import RawReportTerm, RawTermOperation
 from sapiopylib.rest.pojo.datatype.FieldDefinition import VeloxIntegerFieldDefinition, VeloxStringFieldDefinition, \
     AbstractVeloxFieldDefinition
+from sapiopylib.rest.pojo.webhook.WebhookContext import SapioWebhookContext
 from sapiopylib.rest.pojo.webhook.WebhookResult import SapioWebhookResult
 
 from sapiopycommons.callbacks.callback_util import CallbackUtil
 from sapiopycommons.files.file_data_handler import FileDataHandler, FilterList
+from sapiopycommons.general.custom_report_util import CustomReportUtil
 from sapiopycommons.general.time_util import TimeUtil
 
 
@@ -480,3 +483,71 @@ class ContainsSubstringFromCellRule(RowRule):
 
     def validate(self, row: dict[str, Any]) -> bool:
         return row.get(self.second) in row.get(self.first)
+
+
+class UniqueSystemValueRule(ColumnRule):
+    """
+    Requires that every cell in the column has a value that is not already in use in the system for a given data type
+    and field name.
+    """
+    user: SapioUser
+    data_type_name: str
+    data_field_name: str
+
+    def __init__(self, context: SapioWebhookContext | SapioUser, header: str, data_type_name: str,
+                 data_field_name: str):
+        """
+        :param context: The current webhook context or a user object to send requests from.
+        :param header: The header that this rule acts upon.
+        :param data_type_name: The data type name to search on.
+        :param data_field_name: The data field name to search on. This is expected to be a string field.
+        """
+        self.user = context.user if isinstance(context, SapioWebhookContext) else context
+        self.data_type_name = data_type_name
+        self.data_field_name = data_field_name
+        super().__init__(header, f"This value already exists in the system.")
+
+    def validate(self, rows: list[dict[str, Any]]) -> list[int]:
+        file_handler = FileDataHandler(rows)
+        values: list[str] = file_handler.get_values_list(self.header)
+
+        # Run a quick report for all records of this type that match these field values.
+        term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
+                             "{" + ",".join(values) + "}")
+        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
+        return file_handler.get_in_list(self.header, existing_values)
+
+
+class ExistingSystemValueRule(ColumnRule):
+    """
+    Requires that every cell in the column has a value that is already in use in the system for a given data type
+    and field name.
+    """
+    user: SapioUser
+    data_type_name: str
+    data_field_name: str
+
+    def __init__(self, context: SapioWebhookContext | SapioUser, header: str, data_type_name: str,
+                 data_field_name: str):
+        """
+        :param context: The current webhook context or a user object to send requests from.
+        :param header: The header that this rule acts upon.
+        :param data_type_name: The data type name to search on.
+        :param data_field_name: The data field name to search on. This is expected to be a string field.
+        """
+        self.user = context.user if isinstance(context, SapioWebhookContext) else context
+        self.data_type_name = data_type_name
+        self.data_field_name = data_field_name
+        super().__init__(header, f"This value doesn't exist in the system.")
+
+    def validate(self, rows: list[dict[str, Any]]) -> list[int]:
+        file_handler = FileDataHandler(rows)
+        values: list[str] = file_handler.get_values_list(self.header)
+
+        # Run a quick report for all records of this type that match these field values.
+        term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
+                             "{" + ",".join(values) + "}")
+        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
+        return file_handler.get_not_in_list(self.header, existing_values)