deltacat 1.1.11__py3-none-any.whl → 1.1.12__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.11"
+__version__ = "1.1.12"
 
 
 __all__ = [

deltacat/tests/utils/test_daft.py

@@ -104,6 +104,44 @@ class TestDaftS3FileToTable(unittest.TestCase):
         self.assertEqual(table.schema.field("MISSING").type, pa.string())
         self.assertEqual(table.num_rows, 100)
 
+    def test_read_from_s3_single_column_with_schema_extra_cols_column_names(self):
+        schema = pa.schema([("a", pa.int8()), ("MISSING", pa.string())])
+        pa_read_func_kwargs_provider = ReadKwargsProviderPyArrowSchemaOverride(
+            schema=schema
+        )
+        table = daft_s3_file_to_table(
+            self.MVP_PATH,
+            content_encoding=ContentEncoding.IDENTITY.value,
+            content_type=ContentType.PARQUET.value,
+            column_names=["a", "MISSING"],
+            pa_read_func_kwargs_provider=pa_read_func_kwargs_provider,
+        )
+        self.assertEqual(
+            table.schema.names, ["a", "MISSING"]
+        )  # NOTE: "MISSING" is padded as a null array
+        self.assertEqual(table.schema.field("a").type, pa.int8())
+        self.assertEqual(table.schema.field("MISSING").type, pa.string())
+        self.assertEqual(table.num_rows, 100)
+
+    def test_read_from_s3_single_column_with_schema_only_missing_col(self):
+        schema = pa.schema([("a", pa.int8()), ("MISSING", pa.string())])
+        pa_read_func_kwargs_provider = ReadKwargsProviderPyArrowSchemaOverride(
+            schema=schema
+        )
+        table = daft_s3_file_to_table(
+            self.MVP_PATH,
+            content_encoding=ContentEncoding.IDENTITY.value,
+            content_type=ContentType.PARQUET.value,
+            include_columns=["MISSING"],
+            column_names=["a", "MISSING"],
+            pa_read_func_kwargs_provider=pa_read_func_kwargs_provider,
+        )
+        self.assertEqual(
+            table.schema.names, ["MISSING"]
+        )  # NOTE: "MISSING" is padded as a null array
+        self.assertEqual(table.schema.field("MISSING").type, pa.string())
+        self.assertEqual(table.num_rows, 0)
+
     def test_read_from_s3_single_column_with_row_groups(self):
 
         metadata = pq.read_metadata(self.MVP_PATH)

deltacat/tests/utils/test_pyarrow.py

@@ -85,6 +85,43 @@ class TestS3PartialParquetFileToTable(TestCase):
         self.assertEqual(result_schema.field(2).type, "int64")
         self.assertEqual(result_schema.field(2).name, "MISSING")
 
+    def test_s3_partial_parquet_file_to_table_when_schema_missing_columns(self):
+
+        pq_file = ParquetFile(PARQUET_FILE_PATH)
+        partial_parquet_params = PartialParquetParameters.of(
+            pq_metadata=pq_file.metadata
+        )
+        # only first row group to be downloaded
+        partial_parquet_params.row_groups_to_download.pop()
+
+        schema = pa.schema(
+            [
+                pa.field("n_legs", pa.string()),
+                pa.field("animal", pa.string()),
+                # NOTE: This field is not in the parquet file, but will be added on as an all-null column
+                pa.field("MISSING", pa.int64()),
+            ]
+        )
+
+        pa_kwargs_provider = lambda content_type, kwargs: {"schema": schema}
+
+        result = s3_partial_parquet_file_to_table(
+            PARQUET_FILE_PATH,
+            ContentType.PARQUET.value,
+            ContentEncoding.IDENTITY.value,
+            pa_read_func_kwargs_provider=pa_kwargs_provider,
+            partial_file_download_params=partial_parquet_params,
+            column_names=["n_legs", "animal", "MISSING"],
+            include_columns=["MISSING"],
+        )
+
+        self.assertEqual(len(result), 0)
+        self.assertEqual(len(result.column_names), 1)
+
+        result_schema = result.schema
+        self.assertEqual(result_schema.field(0).type, "int64")
+        self.assertEqual(result_schema.field(0).name, "MISSING")
+
     def test_s3_partial_parquet_file_to_table_when_schema_passed_with_include_columns(
         self,
     ):
@@ -234,6 +271,32 @@ class TestReadCSV(TestCase):
             lambda: pyarrow_read_csv(NON_EMPTY_VALID_UTSV_PATH, **kwargs),
         )
 
+    def test_read_csv_when_excess_columns_included(self):
+
+        schema = pa.schema(
+            [
+                ("is_active", pa.string()),
+                ("ship_datetime_utc", pa.timestamp("us")),
+                ("MISSING", pa.string()),
+            ]
+        )
+        kwargs = content_type_to_reader_kwargs(ContentType.UNESCAPED_TSV.value)
+        _add_column_kwargs(
+            ContentType.UNESCAPED_TSV.value,
+            ["is_active", "ship_datetime_utc", "MISSING"],
+            ["is_active", "ship_datetime_utc", "MISSING"],
+            kwargs,
+        )
+
+        read_kwargs_provider = ReadKwargsProviderPyArrowSchemaOverride(schema=schema)
+
+        kwargs = read_kwargs_provider(ContentType.UNESCAPED_TSV.value, kwargs)
+
+        self.assertRaises(
+            pa.lib.ArrowInvalid,
+            lambda: pyarrow_read_csv(NON_EMPTY_VALID_UTSV_PATH, **kwargs),
+        )
+
     def test_read_csv_when_empty_csv_sanity(self):
 
         schema = pa.schema(

{deltacat-1.1.11.dist-info → deltacat-1.1.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.11
+Version: 1.1.12
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
@@ -21,13 +21,13 @@ Requires-Dist: numpy ==1.21.5
 Requires-Dist: pandas ==1.3.5
 Requires-Dist: pyarrow ==12.0.1
 Requires-Dist: pydantic ==1.10.4
-Requires-Dist: ray[default] >=2.20.0
+Requires-Dist: ray >=2.20.0
 Requires-Dist: s3fs ==2024.5.0
 Requires-Dist: tenacity ==8.1.0
 Requires-Dist: typing-extensions ==4.4.0
 Requires-Dist: pymemcache ==4.0.0
 Requires-Dist: redis ==4.6.0
-Requires-Dist: getdaft ==0.2.29
+Requires-Dist: getdaft ==0.2.31
 Requires-Dist: schedule ==1.2.0
 
 # DeltaCAT

{deltacat-1.1.11.dist-info → deltacat-1.1.12.dist-info}/RECORD

@@ -1,4 +1,4 @@
-deltacat/__init__.py,sha256=z6zdRpLnxm9AOnjLtJBmkciFK82KnMkPjmtjy85AttM,1778
+deltacat/__init__.py,sha256=jtrgQAzzZxq5YU_9RwtH0N4p8k4_ACd_vcjuWJf934Q,1778
 deltacat/constants.py,sha256=_6oRI-3yp5c8J1qKGQZrt89I9-ttT_gSSvVsJ0h8Duc,1939
 deltacat/exceptions.py,sha256=yWM4RXK7uRrQc1VgJv6Lv2UiNZWAx2wolLq7cBwjlkg,12770
 deltacat/logs.py,sha256=6g16VkEFidbaMjgenAjggE1r2l664drMVhreRs8B1IQ,8438
@@ -182,10 +182,10 @@ deltacat/tests/test_utils/storage.py,sha256=93GEn4A5WbMHWk0Ec4Bd7RxeHoSEnBfSarfW
 deltacat/tests/test_utils/utils.py,sha256=a32qEwcSSd1lvRi0aJJ4ZLnc1ZyXmoQF_K95zaQRk2M,455
 deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deltacat/tests/utils/test_cloudpickle.py,sha256=J0pnBY3-PxlUh6MamZAN1PuquKQPr2iyzjiJ7-Rcl0o,1506
-deltacat/tests/utils/test_daft.py,sha256=AIE0qz6oKhEEvBqF0VfQ5pwTtiTHqyf0EuUXduiS3t4,6487
+deltacat/tests/utils/test_daft.py,sha256=kY8lkXoQvyWunok8UvOsh1An297rb3jcnstTuIAyAlc,8232
 deltacat/tests/utils/test_metrics.py,sha256=Ym9nOz1EtB180pLmvugihj1sDTNDMb5opIjjr5Nmcls,16339
 deltacat/tests/utils/test_placement.py,sha256=g61wVOMkHe4YJeR9Oxg_BOVQ6bhHHbC3IBYv8YhUu94,597
-deltacat/tests/utils/test_pyarrow.py,sha256=eZAuYp9MUf8lmpIilH57JkURuNsTGZ3IAGC4Gm5hdrM,17307
+deltacat/tests/utils/test_pyarrow.py,sha256=AWx0DTsBA1PkQag48w_HeQdz7tlBzJsm9v7Nd6-dhEY,19607
 deltacat/tests/utils/test_record_batch_tables.py,sha256=AkG1WyljQmjnl-AxhbFWyo5LnMIKRyLScfgC2B_ES-s,11321
 deltacat/tests/utils/test_resources.py,sha256=HtpvDrfPZQNtGDXUlsIzc_yd7Vf1cDscZ3YbN0oTvO8,2560
 deltacat/tests/utils/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -216,8 +216,8 @@ deltacat/utils/ray_utils/concurrency.py,sha256=JDVwMiQWrmuSlyCWAoiq9ctoJ0XADEfDD
 deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
 deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
 deltacat/utils/ray_utils/runtime.py,sha256=rB0A-tU9WZHz0J11LzJdANYtL397YyuemcA1l-K9dAw,5029
-deltacat-1.1.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deltacat-1.1.11.dist-info/METADATA,sha256=60wuPvw4-9iEcp9v1Bz0b4fpEwSGfgjYQ_0YGMaVuVo,1757
-deltacat-1.1.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-deltacat-1.1.11.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
-deltacat-1.1.11.dist-info/RECORD,,
+deltacat-1.1.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-1.1.12.dist-info/METADATA,sha256=fqP5NYc_4wrfWcsQXO1Aht8xM22LrmDZLHfMAqq2opQ,1748
+deltacat-1.1.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+deltacat-1.1.12.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-1.1.12.dist-info/RECORD,,