deltacat 1.1.31__py3-none-any.whl → 1.1.32__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.31"
+__version__ = "1.1.32"
 
 
 __all__ = [

deltacat/tests/utils/test_pyarrow.py

@@ -2,9 +2,12 @@ from unittest import TestCase
 from deltacat.utils.pyarrow import (
     s3_partial_parquet_file_to_table,
     pyarrow_read_csv,
+    ContentTypeValidationError,
     content_type_to_reader_kwargs,
     _add_column_kwargs,
+    logger,
     s3_file_to_table,
+    s3_file_to_parquet,
     ReadKwargsProviderPyArrowSchemaOverride,
     RAISE_ON_EMPTY_CSV_KWARG,
     RAISE_ON_DECIMAL_OVERFLOW,
@@ -435,7 +438,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowInvalid,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_PRECISION_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -479,7 +482,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowInvalid,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_SCALE_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -590,7 +593,7 @@ class TestReadCSV(TestCase):
             pa.lib.ArrowNotImplementedError,
             lambda: pyarrow_read_csv(
                 OVERFLOWING_DECIMAL_SCALE_UTSV_PATH,
-                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True}
+                **{**kwargs, RAISE_ON_DECIMAL_OVERFLOW: True},
             ),
         )
 
@@ -818,8 +821,11 @@ class TestS3FileToTable(TestCase):
         schema = pa.schema(
             [("is_active", pa.string()), ("ship_datetime_utc", pa.timestamp("us"))]
         )
-
         # OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG has no effect on uTSV files
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
         pa_kwargs_provider = lambda content_type, kwargs: {
             "reader_type": "pyarrow",
             OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG: ContentEncoding.IDENTITY.value,
@@ -864,3 +870,99 @@ class TestS3FileToTable(TestCase):
         schema = result.schema
         schema_index = schema.get_field_index("n_legs")
         self.assertEqual(schema.field(schema_index).type, "int64")
+
+
+class TestS3FileToParquet(TestCase):
+    def test_s3_file_to_parquet_sanity(self):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.IDENTITY.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            result_parquet_file: ParquetFile = s3_file_to_parquet(
+                test_s3_url,
+                test_content_type,
+                test_content_encoding,
+                ["n_legs", "animal"],
+                ["n_legs"],
+                pa_read_func_kwargs_provider=pa_kwargs_provider,
+            )
+        log_message_log_args = cm.records[0].getMessage()
+        log_message_presanitize_kwargs = cm.records[1].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )
+        self.assertIn("{'reader_type': 'pyarrow'}", log_message_presanitize_kwargs)
+        for index, field in enumerate(result_parquet_file.schema_arrow):
+            self.assertEqual(
+                field.name, result_parquet_file.schema_arrow.field(index).name
+            )
+        self.assertEqual(result_parquet_file.schema_arrow.field(0).type, "int64")
+
+    def test_s3_file_to_parquet_when_parquet_gzip_encoding_and_overridden_returns_success(
+        self,
+    ):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.GZIP.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG: ContentEncoding.IDENTITY.value,
+            **kwargs,
+        }
+        with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+            result_parquet_file: ParquetFile = s3_file_to_parquet(
+                test_s3_url,
+                test_content_type,
+                test_content_encoding,
+                ["n_legs", "animal"],
+                ["n_legs"],
+                pa_read_func_kwargs_provider=pa_kwargs_provider,
+            )
+        log_message_log_args = cm.records[0].getMessage()
+        log_message_log_new_content_encoding = cm.records[1].getMessage()
+        log_message_presanitize_kwargs = cm.records[2].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )
+        self.assertIn(
+            f"Overriding {test_s3_url} content encoding from {ContentEncoding.GZIP.value} to {ContentEncoding.IDENTITY.value}",
+            log_message_log_new_content_encoding,
+        )
+        self.assertIn("{'reader_type': 'pyarrow'}", log_message_presanitize_kwargs)
+        for index, field in enumerate(result_parquet_file.schema_arrow):
+            self.assertEqual(
+                field.name, result_parquet_file.schema_arrow.field(index).name
+            )
+        self.assertEqual(result_parquet_file.schema_arrow.field(0).type, "int64")
+
+    def test_s3_file_to_parquet_when_parquet_gzip_encoding_not_overridden_throws_error(
+        self,
+    ):
+        test_s3_url = PARQUET_FILE_PATH
+        test_content_type = ContentType.PARQUET.value
+        test_content_encoding = ContentEncoding.GZIP.value
+        pa_kwargs_provider = lambda content_type, kwargs: {
+            "reader_type": "pyarrow",
+            **kwargs,
+        }
+        with self.assertRaises(ContentTypeValidationError):
+            with self.assertLogs(logger=logger.name, level="DEBUG") as cm:
+                s3_file_to_parquet(
+                    test_s3_url,
+                    test_content_type,
+                    test_content_encoding,
+                    ["n_legs", "animal"],
+                    ["n_legs"],
+                    pa_read_func_kwargs_provider=pa_kwargs_provider,
+                )
+        log_message_log_args = cm.records[0].getMessage()
+        self.assertIn(
+            f"Reading {test_s3_url} to PyArrow ParquetFile. Content type: {test_content_type}. Encoding: {test_content_encoding}",
+            log_message_log_args,
+        )

deltacat/utils/pyarrow.py

@@ -617,7 +617,18 @@ def s3_file_to_parquet(
         f"Reading {s3_url} to PyArrow ParquetFile. "
         f"Content type: {content_type}. Encoding: {content_encoding}"
     )
+    kwargs = {}
+    if pa_read_func_kwargs_provider:
+        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
 
+    if OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG in kwargs:
+        new_content_encoding = kwargs.pop(OVERRIDE_CONTENT_ENCODING_FOR_PARQUET_KWARG)
+        if content_type == ContentType.PARQUET.value:
+            logger.debug(
+                f"Overriding {s3_url} content encoding from {content_encoding} "
+                f"to {new_content_encoding}"
+            )
+            content_encoding = new_content_encoding
     if (
         content_type != ContentType.PARQUET.value
         or content_encoding != ContentEncoding.IDENTITY
@@ -630,15 +641,10 @@ def s3_file_to_parquet(
     if s3_client_kwargs is None:
         s3_client_kwargs = {}
 
-    kwargs = {}
-
     if s3_url.startswith("s3://"):
         s3_file_system = create_s3_file_system(s3_client_kwargs)
         kwargs["filesystem"] = s3_file_system
 
-    if pa_read_func_kwargs_provider:
-        kwargs = pa_read_func_kwargs_provider(content_type, kwargs)
-
     logger.debug(f"Pre-sanitize kwargs for {s3_url}: {kwargs}")
 
     kwargs = sanitize_kwargs_to_callable(ParquetFile.__init__, kwargs)

{deltacat-1.1.31.dist-info → deltacat-1.1.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.31
+Version: 1.1.32
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.31.dist-info → deltacat-1.1.32.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=gdOpCNy03T2HEQIQqSqopv0b0UL5pwXWa4McRHxMlAw,1778
+deltacat/__init__.py,sha256=amNk91Zxauag8dm3s8SuUKinWdeAA2EaiWG9_SdboQE,1778
 deltacat/constants.py,sha256=TUJLXUJ9xq1Ryil72yLkKR8EDH_Irp5wUg56QstbRNE,2181
 deltacat/exceptions.py,sha256=7sjk3BuMY5Oo-6OvAfHncZx_OcvtEL47BblWr2F7waE,12740
 deltacat/logs.py,sha256=EQSDin1deehzz5xlLV1_TrFJrO_IBZ9Ahp7MdL-4cK8,9363
@@ -180,7 +180,7 @@ deltacat/tests/utils/test_cloudpickle.py,sha256=J0pnBY3-PxlUh6MamZAN1PuquKQPr2iy
 deltacat/tests/utils/test_daft.py,sha256=kY8lkXoQvyWunok8UvOsh1An297rb3jcnstTuIAyAlc,8232
 deltacat/tests/utils/test_metrics.py,sha256=Ym9nOz1EtB180pLmvugihj1sDTNDMb5opIjjr5Nmcls,16339
 deltacat/tests/utils/test_placement.py,sha256=g61wVOMkHe4YJeR9Oxg_BOVQ6bhHHbC3IBYv8YhUu94,597
-deltacat/tests/utils/test_pyarrow.py,sha256=JmhcuphXD8B2SLnOgrPgrqCcdHg_BL6IjFAiNRmuA1I,32790
+deltacat/tests/utils/test_pyarrow.py,sha256=tuh6HzQOuAHPFxK5Mhgjjdm76Z9Z72H3MZPcJ4RnZn8,37372
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
 deltacat/tests/utils/test_resources.py,sha256=HtpvDrfPZQNtGDXUlsIzc_yd7Vf1cDscZ3YbN0oTvO8,2560
 deltacat/tests/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -201,7 +201,7 @@ deltacat/utils/numpy.py,sha256=SpHKKvC-K8NINTWGVfTZ5-gBFTGYqaXjjgKFhsdUjwg,2049
 deltacat/utils/pandas.py,sha256=q99mlRB7tymICMcNbfGLfLqFu_C-feyPZKZm2CWJJVc,9574
 deltacat/utils/performance.py,sha256=7ZLaMkS1ehPSIhT5uOQVBHvjC70iKHzoFquFo-KL0PI,645
 deltacat/utils/placement.py,sha256=Lj20fb-eq8rgMdm_M2MBMfDLwhDM1sS1nJj2DvIK56s,12060
-deltacat/utils/pyarrow.py,sha256=9Dggs8waJrbgP62NG4ssZsl-9fl3cJ4fjYLsJ1HjhHQ,34847
+deltacat/utils/pyarrow.py,sha256=MFCsHJKapqrhaaBeVAvwR2F1MglsNNhVZeCbk7YIdyI,35266
 deltacat/utils/resources.py,sha256=Ax1OgLLbZI4oYpp4Ki27OLaST-7I-AJgZwU87FVfY8g,8253
 deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
 deltacat/utils/schema.py,sha256=m4Wm4ZQcpttzOUxex4dVneGlHy1_E36HspTcjNYzvVM,1564
@@ -211,8 +211,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.31.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.31.dist-info/METADATA,sha256=JrWYw0uKVprpH34i-_cOUYjWI3egRQx0rhCn--OnE_0,1733
-deltacat-1.1.31.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-deltacat-1.1.31.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.31.dist-info/RECORD,,
+deltacat-1.1.32.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.32.dist-info/METADATA,sha256=KqU11gn6r8cnfoyKq4_C8widB7w_wdmfN_ikhHjSZfI,1733
+deltacat-1.1.32.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+deltacat-1.1.32.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.32.dist-info/RECORD,,