sapiopycommons 2024.9.25a334__tar.gz → 2024.9.30a335__tar.gz

{sapiopycommons-2024.9.25a334 → sapiopycommons-2024.9.30a335}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sapiopycommons
-Version: 2024.9.25a334
+Version: 2024.9.30a335
 Summary: Official Sapio Python API Utilities Package
 Project-URL: Homepage, https://github.com/sapiosciences
 Author-email: Jonathan Steck <jsteck@sapiosciences.com>, Yechen Qiao <yqiao@sapiosciences.com>

{sapiopycommons-2024.9.25a334 → sapiopycommons-2024.9.30a335}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "sapiopycommons"
-version='2024.09.25a334'
+version='2024.09.30a335'
 authors = [
     { name="Jonathan Steck", email="jsteck@sapiosciences.com" },
     { name="Yechen Qiao", email="yqiao@sapiosciences.com" },

{sapiopycommons-2024.9.25a334 → sapiopycommons-2024.9.30a335}/src/sapiopycommons/files/file_util.py

@@ -23,8 +23,8 @@ class FileUtil:
     """
     @staticmethod
     def tokenize_csv(file_bytes: bytes, required_headers: list[str] | None = None, header_row_index: int | None = 0,
-                     seperator: str = ",", *, encoding: str | None = None, exception_on_empty: bool = True) \
-            -> tuple[list[dict[str, str]], list[list[str]]]:
+                     seperator: str = ",", *, encoding: str | None = None, encoding_error: str | None = "strict",
+                     exception_on_empty: bool = True) -> tuple[list[dict[str, str]], list[list[str]]]:
         """
         Tokenize a CSV file. The provided file must be uniform. That is, if row 1 has 10 cells, all the rows in the file
         must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -39,7 +39,11 @@ class FileUtil:
         :param seperator: The character that separates cells in the table.
         :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
             contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
-            ISO-8859-1 as the encoding.
+            ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
+        :param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
+            is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
+            characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
+            https://docs.python.org/3/library/codecs.html#error-handlers
         :param exception_on_empty: Throw a user error exception if the provided file bytes result in an empty list in
             the first element of the returned tuple.
         :return: The CSV parsed into a list of dicts where each dict is a row, mapping the headers to the cells for
@@ -49,7 +53,7 @@ class FileUtil:
         # Parse the file bytes into two DataFrames. The first is metadata of the file located above the header row,
         # while the second is the body of the file below the header row.
         file_body, file_metadata = FileUtil.csv_to_data_frames(file_bytes, header_row_index, seperator,
-                                                               encoding=encoding)
+                                                               encoding=encoding, encoding_error=encoding_error)
         # Parse the metadata from above the header row index into a list of lists.
         metadata: list[list[str]] = FileUtil.data_frame_to_lists(file_metadata)
         # Parse the data from the file body into a list of dicts.
@@ -90,7 +94,8 @@ class FileUtil:
 
     @staticmethod
     def csv_to_data_frames(file_bytes: bytes, header_row_index: int | None = 0, seperator: str = ",",
-                           *, encoding: str | None = None) -> tuple[DataFrame, DataFrame | None]:
+                           *, encoding: str | None = None, encoding_error: str | None = "strict") \
+            -> tuple[DataFrame, DataFrame | None]:
         """
         Parse the file bytes for a CSV into DataFrames. The provided file must be uniform. That is, if row 1 has 10
         cells, all the rows in the file must have 10 cells. Otherwise, the Pandas parser throws a tokenizer exception.
@@ -103,7 +108,11 @@ class FileUtil:
         :param seperator: The character that separates cells in the table.
         :param encoding: The encoding used to read the given file bytes. If not provided, uses utf-8. If your file
             contains a non-utf-8 character, then a UnicodeDecodeError will be thrown. If this happens, consider using
-            ISO-8859-1 as the encoding.
+            ISO-8859-1 as the encoding, or investigate what encoding would handle the characters in your file.
+        :param encoding_error: The error handling behavior if an encoding error is encountered. By default, the behavior
+            is "strict", meaning that encoding errors raise an exception. Change this to "ignore" to skip over invalid
+            characters or "replace" to replace invalid characters with a ? character. For a full list of options, see
+            https://docs.python.org/3/library/codecs.html#error-handlers
         :return: A tuple of two DataFrames. The first is the frame for the CSV table body, while the second is for the
             metadata from above the header row, or None if there is no metadata.
         """
@@ -115,7 +124,8 @@ class FileUtil:
                 # can throw off the header row index.
                 file_metadata = pandas.read_csv(file_io, header=None, dtype=dtype(str),
                                                 skiprows=lambda x: x >= header_row_index,
-                                                skip_blank_lines=False, sep=seperator, encoding=encoding)
+                                                skip_blank_lines=False, sep=seperator, encoding=encoding,
+                                                encoding_errors=encoding_error)
         with io.BytesIO(file_bytes) as file_io:
             # The use of the dtype argument is to ensure that everything from the file gets read as a string. Added
             # because some numerical values would get ".0" appended to them, even when casting the DataFrame cell to a