sapiopycommons 2024.6.6a248__tar.gz → 2024.6.11a252__tar.gz

{sapiopycommons-2024.6.6a248 → sapiopycommons-2024.6.11a252}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sapiopycommons
-Version: 2024.6.6a248
+Version: 2024.6.11a252
 Summary: Official Sapio Python API Utilities Package
 Project-URL: Homepage, https://github.com/sapiosciences
 Author-email: Jonathan Steck <jsteck@sapiosciences.com>, Yechen Qiao <yqiao@sapiosciences.com>

{sapiopycommons-2024.6.6a248 → sapiopycommons-2024.6.11a252}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sapiopycommons"
-version='2024.06.06a248'
+version='2024.06.11a252'
 authors = [
     { name="Jonathan Steck", email="jsteck@sapiosciences.com" },
     { name="Yechen Qiao", email="yqiao@sapiosciences.com" },

{sapiopycommons-2024.6.6a248 → sapiopycommons-2024.6.11a252}/src/sapiopycommons/files/file_util.py

@@ -21,7 +21,7 @@ class FileUtil:
     """
     @staticmethod
     def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
-                     seperator: str = ",") -> tuple[list[dict[str, str]], list[list[str]]]:
+                     seperator: str = ",", *, encoding: str | None = None) -> tuple[list[dict[str, str]], list[list[str]]]:
         """
         Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
         must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -34,13 +34,17 @@ class FileUtil:
             meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
             is assumed to be the header row.
         :param seperator: The character that separates cells in the table.
+        :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
+            contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
+            ISO-8859-1 as the encoding.
         :return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
             that row. Also returns a list of each row above the headers (the metadata), parsed into a list of each cell.
             If the header row index is 0 or None, this list will be empty.
         """
         # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
         # while the second is the body of the file below the header row.
-        file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator)
+        file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
+                                                               encoding=encoding)
         # Parse the metadata from above the header row index into a list of lists.
         metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
         # Parse the data from the file body into a list of dicts.
@@ -74,8 +78,8 @@ class FileUtil:
         return rows, metadata
 
     @staticmethod
-    def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",") \
-            -> tuple[DataFrame, DataFrame | None]:
+    def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
+                           *, encoding: str | None = None) -> tuple[DataFrame, DataFrame | None]:
         """
         Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
         cells, all the rows in the file must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -86,6 +90,9 @@ class FileUtil:
             meaning that required headers are also ignored if any are provided. By default, the first row (0th index)
             is assumed to be the header row.
         :param seperator: The character that separates cells in the table.
+        :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
+            contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
+            ISO-8859-1 as the encoding.
         :return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
             metadata from above the header row, or None if there is no metadata.
         """
@@ -97,13 +104,13 @@ class FileUtil:
                 # can throw off the header row index.
                 file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
                                                 skiprows=lambda x: x >= header_row_index,
-                                                skip_blank_lines=False, sep=seperator)
+                                                skip_blank_lines=False, sep=seperator, encoding=encoding)
         with io.BytesIO(file_bytes) as file_io:
             # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
             # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a
             # string.
             file_body: DataFrame = pandas.read_csv(file_io, header=header_row_index, dtype=dtype(str),
-                                                   skip_blank_lines=False, sep=seperator)
+                                                   skip_blank_lines=False, sep=seperator, encoding=encoding)
 
         return file_body, file_metadata
 

{sapiopycommons-2024.6.6a248 → sapiopycommons-2024.6.11a252}/src/sapiopycommons/files/file_validator.py

@@ -4,12 +4,15 @@ from abc import abstractmethod
 from typing import Any
 
 from sapiopylib.rest.User import SapioUser
+from sapiopylib.rest.pojo.CustomReport import RawReportTerm, RawTermOperation
 from sapiopylib.rest.pojo.datatype.FieldDefinition import VeloxIntegerFieldDefinition, VeloxStringFieldDefinition, \
     AbstractVeloxFieldDefinition
+from sapiopylib.rest.pojo.webhook.WebhookContext import SapioWebhookContext
 from sapiopylib.rest.pojo.webhook.WebhookResult import SapioWebhookResult
 
 from sapiopycommons.callbacks.callback_util import CallbackUtil
 from sapiopycommons.files.file_data_handler import FileDataHandler, FilterList
+from sapiopycommons.general.custom_report_util import CustomReportUtil
 from sapiopycommons.general.time_util import TimeUtil
 
 
@@ -480,3 +483,71 @@ class ContainsSubstringFromCellRule(RowRule):
 
     def validate(self, row: dict[str, Any]) -> bool:
         return row.get(self.second) in row.get(self.first)
+
+
+class UniqueSystemValueRule(ColumnRule):
+    """
+    Requires that every cell in the column has a value that is not already in use in the system for a given data type
+    and field name.
+    """
+    user: SapioUser
+    data_type_name: str
+    data_field_name: str
+
+    def __init__(self, context: SapioWebhookContext | SapioUser, header: str, data_type_name: str,
+                 data_field_name: str):
+        """
+        :param context: The current webhook context or a user object to send requests from.
+        :param header: The header that this rule acts upon.
+        :param data_type_name: The data type name to search on.
+        :param data_field_name: The data field name to search on. This is expected to be a string field.
+        """
+        self.user = context.user if isinstance(context, SapioWebhookContext) else context
+        self.data_type_name = data_type_name
+        self.data_field_name = data_field_name
+        super().__init__(header, f"This value already exists in the system.")
+
+    def validate(self, rows: list[dict[str, Any]]) -> list[int]:
+        file_handler = FileDataHandler(rows)
+        values: list[str] = file_handler.get_values_list(self.header)
+
+        # Run a quick report for all records of this type that match these field values.
+        term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
+                             "{" + ",".join(values) + "}")
+        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
+        return file_handler.get_in_list(self.header, existing_values)
+
+
+class ExistingSystemValueRule(ColumnRule):
+    """
+    Requires that every cell in the column has a value that is already in use in the system for a given data type
+    and field name.
+    """
+    user: SapioUser
+    data_type_name: str
+    data_field_name: str
+
+    def __init__(self, context: SapioWebhookContext | SapioUser, header: str, data_type_name: str,
+                 data_field_name: str):
+        """
+        :param context: The current webhook context or a user object to send requests from.
+        :param header: The header that this rule acts upon.
+        :param data_type_name: The data type name to search on.
+        :param data_field_name: The data field name to search on. This is expected to be a string field.
+        """
+        self.user = context.user if isinstance(context, SapioWebhookContext) else context
+        self.data_type_name = data_type_name
+        self.data_field_name = data_field_name
+        super().__init__(header, f"This value doesn't exist in the system.")
+
+    def validate(self, rows: list[dict[str, Any]]) -> list[int]:
+        file_handler = FileDataHandler(rows)
+        values: list[str] = file_handler.get_values_list(self.header)
+
+        # Run a quick report for all records of this type that match these field values.
+        term = RawReportTerm(self.data_type_name, self.data_field_name, RawTermOperation.EQUAL_TO_OPERATOR,
+                             "{" + ",".join(values) + "}")
+        results: list[dict[str, Any]] = CustomReportUtil.run_quick_report(self.user, term)
+        existing_values: list[Any] = [x.get(self.data_field_name) for x in results]
+        return file_handler.get_not_in_list(self.header, existing_values)