data-prep-toolkit 0.2.2.dev2__py3-none-any.whl → 0.2.3.dev0__py3-none-any.whl

{data_prep_toolkit-0.2.2.dev2.dist-info → data_prep_toolkit-0.2.3.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_toolkit
-Version: 0.2.2.dev2
+Version: 0.2.3.dev0
 Summary: Data Preparation Toolkit Library for Ray and Python
 Author-email: Maroun Touma <touma@us.ibm.com>
 License: Apache-2.0
@@ -13,6 +13,7 @@ Requires-Dist: boto3==1.34.69
 Requires-Dist: argparse
 Requires-Dist: mmh3
 Requires-Dist: psutil
+Requires-Dist: polars>=1.9.0
 Provides-Extra: dev
 Requires-Dist: twine; extra == "dev"
 Requires-Dist: pytest>=7.3.2; extra == "dev"

{data_prep_toolkit-0.2.2.dev2.dist-info → data_prep_toolkit-0.2.3.dev0.dist-info}/RECORD

@@ -47,7 +47,7 @@ data_processing/utils/params_utils.py,sha256=oAKY3wC8b17rDUJGqX19-rAQHDc9SQn1ksT
 data_processing/utils/pipinstaller.py,sha256=PxFNwEy8v4FqjwYgrPhH0UTrCgsJvM5WAE2fKylsk2Q,2511
 data_processing/utils/transform_configuration.json,sha256=6YBw0Hk2mokY6JBn1kR6L9AkV_yivbFrpSoHecAJp9o,4562
 data_processing/utils/transform_configurator.py,sha256=9OHSCQ8rFSoDdMW6ZCHYdNe6thRwV9zOaRPnLkWNMYE,3601
-data_processing/utils/transform_utils.py,sha256=KGNioN35B1i1h-MIsfm3QvXLlU1aGXimheva7NbUhMM,8496
+data_processing/utils/transform_utils.py,sha256=1IEowOYQA6HOGEalqujbDVatrBaImnuY5OKmUYGaGwI,9068
 data_processing/utils/unrecoverable.py,sha256=cbF74AGK1IdRor_L1w_hPwglV_b2blP6Ad4ET79xrl0,831
 data_processing_ray/runtime/ray/__init__.py,sha256=vjQOvb_OJNq3c1F_tG3WjO-pciY77Z1lETO2Ha_GVbw,784
 data_processing_ray/runtime/ray/execution_configuration.py,sha256=C9YFixlATr7PPpkVQ0WzjCCPTWFuP80W2rnzY1bbp5I,4628
@@ -72,7 +72,7 @@ data_processing_spark/runtime/spark/transform_runtime.py,sha256=je27rTRdd-5Wtd8n
 data_processing_spark/test_support/transform/__init__.py,sha256=FQJyj7z1hXQynngMVQlCTJxTh2bdc4jN4220CBmLTqE,872
 data_processing_spark/test_support/transform/noop_folder_transform.py,sha256=z0jXCVKJYHPqB9ZTfUxnQkUVDnmfWjvss4_I3QZ8JZ4,2187
 data_processing_spark/test_support/transform/noop_transform.py,sha256=0FR3o-LnXf-UFS5gU0j-i4LVlw1mHDxGaPI40dkkIKY,1694
-data_prep_toolkit-0.2.2.dev2.dist-info/METADATA,sha256=XgskYjPA5pddqDgaBrPpe1IeqOpHPB2WscNM4dRh7XQ,2240
-data_prep_toolkit-0.2.2.dev2.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-data_prep_toolkit-0.2.2.dev2.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
-data_prep_toolkit-0.2.2.dev2.dist-info/RECORD,,
+data_prep_toolkit-0.2.3.dev0.dist-info/METADATA,sha256=u2UV51dNeRPKa4R0bEOI6udMGEodA8JzlCffyt-xXt8,2269
+data_prep_toolkit-0.2.3.dev0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+data_prep_toolkit-0.2.3.dev0.dist-info/top_level.txt,sha256=XGMDmY55_pe5KeRWvO0un9a640e2v99tzbBBtjNybPM,58
+data_prep_toolkit-0.2.3.dev0.dist-info/RECORD,,

{data_prep_toolkit-0.2.2.dev2.dist-info → data_prep_toolkit-0.2.3.dev0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

data_processing/utils/transform_utils.py

@@ -11,6 +11,7 @@
 ################################################################################
 
 import hashlib
+import io
 import os
 import string
 import sys
@@ -144,8 +145,21 @@ class TransformUtils:
             table = pq.read_table(reader, schema=schema)
             return table
         except Exception as e:
-            logger.error(f"Failed to convert byte array to arrow table, exception {e}. Skipping it")
-            return None
+            logger.warning(f"Could not convert bytes to pyarrow: {e}")
+
+        # We have seen this exception before when using pyarrow, but polars does not throw it.
+        # "Nested data conversions not implemented for chunked array outputs"
+        # See issue 816 https://github.com/IBM/data-prep-kit/issues/816.
+        logger.info(f"Attempting read of pyarrow Table using polars")
+        try:
+            import polars
+
+            df = polars.read_parquet(io.BytesIO(data))
+            table = df.to_arrow()
+        except Exception as e:
+            logger.error(f"Could not convert bytes to pyarrow using polars: {e}. Skipping.")
+            table = None
+        return table
 
     @staticmethod
     def convert_arrow_to_binary(table: pa.Table) -> bytes: