adapta 4.0.0__tar.gz → 4.0.2a596.dev2__tar.gz

{adapta-4.0.0 → adapta-4.0.2a596.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adapta
-Version: 4.0.0
+Version: 4.0.2a596.dev2
 Summary: Logging, data connectors, monitoring, secret handling and general lifehacks to make data people lives easier.
 License: Apache-2.0
 License-File: LICENSE

adapta-4.0.2a596.dev2/adapta/_version.py

@@ -0,0 +1 @@
+__version__ = 'v4.0.2a596.dev2'

{adapta-4.0.0 → adapta-4.0.2a596.dev2}/adapta/dataclass_validation/validation/validation_abstract.py

@@ -72,6 +72,8 @@ class AbstractValidationClass:
         origin_dtype = get_origin(dtype)
         if origin_dtype in self._dtype_recursive_dtypes and origin_dtype == list:
             return self._dtype_recursive_dtypes[origin_dtype](self._get_expected_dtypes(dtype=get_args(dtype)[0]))
+        if origin_dtype in self._dtype_recursive_dtypes and origin_dtype == dict:
+            return self._dtype_recursive_dtypes[origin_dtype]()
 
         raise TypeError(
             f"Unsupported data type: {dtype}. Supported types are: "

{adapta-4.0.0 → adapta-4.0.2a596.dev2}/adapta/dataclass_validation/validation/validation_polars.py

@@ -23,11 +23,12 @@ class PolarsValidationClass(AbstractValidationClass):
             bool: pl.Boolean,
             datetime.date: pl.Date,
             datetime.datetime: pl.Datetime,
+            dict: pl.Struct,
         }
 
     @property
     def _dtype_recursive_dtypes(self):
-        return {list: pl.List}
+        return {list: pl.List, dict: lambda: pl.Struct}
 
     @property
     def _allowed_casts(self):

{adapta-4.0.0 → adapta-4.0.2a596.dev2}/adapta/storage/iceberg/v1/_functions.py

@@ -1,4 +1,5 @@
 """Iceberg reader (via REST Catalog)"""
+import os
 from typing import Literal
 
 import polars
@@ -53,27 +54,30 @@ def load_using_catalog(
     if row_filter:
         row_filter_expression = compile_expression(row_filter, IcebergFilterExpression)
 
-    if lazy_read:
-        return MetaFrame(
-            data=catalog.load_table(identifier=(schema, table_name))
-            .scan(
-                row_filter=row_filter_expression or ALWAYS_TRUE,
-                selected_fields=columns or ("*",),
-                limit=limit,
-                snapshot_id=version_id,
-            )
-            .to_arrow_batch_reader(),
-            convert_to_polars=lambda v: polars.scan_pyarrow_dataset(pyarrow.dataset.dataset(v)),
-            convert_to_pandas=None,
-        )
+    table = catalog.load_table(identifier=(schema, table_name))
+    if "ADAPTA__ICEBERG_REST_CATALOG__S3_ENDPOINT_OVERRIDE" in os.environ:
+        # FileIO's endpoint is taken directly from the catalog response
+        # In case it differs from `s3.endpoint` set in catalog config, align them
+        # Note that when vended credentials are used, table.config will take preference over client setting
+        # thus endpoint is updated after catalog returns creds
+        # this is necessary if your S3 service has multiple endpoints and client doesn't have access to the one used by catalog
+        table.io.properties["s3.endpoint"] = os.environ["ADAPTA__ICEBERG_REST_CATALOG__S3_ENDPOINT_OVERRIDE"]
+        table.config["s3.endpoint"] = os.environ["ADAPTA__ICEBERG_REST_CATALOG__S3_ENDPOINT_OVERRIDE"]
 
-    scanner = catalog.load_table(identifier=(schema, table_name)).scan(
+    scanner = table.scan(
         row_filter=row_filter_expression or ALWAYS_TRUE,
         selected_fields=columns or ("*",),
         limit=limit,
         snapshot_id=version_id,
     )
 
+    if lazy_read:
+        return MetaFrame(
+            data=scanner.to_arrow_batch_reader(),
+            convert_to_polars=lambda v: polars.scan_pyarrow_dataset(pyarrow.dataset.dataset(v)),
+            convert_to_pandas=None,
+        )
+
     return MetaFrame(
         data=scanner,
         # use built-in DataScan converters

{adapta-4.0.0 → adapta-4.0.2a596.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "adapta"
-version = "4.0.0"
+version = "v4.0.2a596.dev2"
 description = "Logging, data connectors, monitoring, secret handling and general lifehacks to make data people lives easier."
 authors = ["ECCO Sneaks & Data <esdsupport@ecco.com>"]
 maintainers = ['GZU <gzu@ecco.com>', 'JRB <ext-jrb@ecco.com>']

adapta-4.0.0/adapta/_version.py

@@ -1 +0,0 @@
-__version__ = '4.0.0'