acryl-datahub 1.3.0rc5__py3-none-any.whl → 1.3.0.1rc2__py3-none-any.whl

{acryl_datahub-1.3.0rc5.dist-info → acryl_datahub-1.3.0.1rc2.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.3.0rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.3.0.1rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=tQwOz8j8lgLnVKUe2eNZLleYAQzj2kRWOmo8yGoOMOk,321
+datahub/_version.py,sha256=G3S5qsHOL2-5yNrU2qQxhGJwD63r-1qLun37Mq1ejLo,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -100,7 +100,7 @@ datahub/cli/specific/structuredproperties_cli.py,sha256=rgx8fhI7WYq8QLwIttkMysX7
 datahub/cli/specific/user_cli.py,sha256=HaASGg25b82Q18hKyOn98gPoQfmj1gW0utDMs0iR5WY,1897
 datahub/configuration/__init__.py,sha256=5TN3a7CWNsLRHpdj-sv2bxKWF2IslvJwE6EpNMFrIS4,123
 datahub/configuration/_config_enum.py,sha256=ul2hr5gMmdLvBINicFkMNMi1ApmnmZSwNdUYYted5nk,1447
-datahub/configuration/common.py,sha256=HvMrc-q2ZocbAHj5KgBHUaQOjdWLvk_wRGDkTUVbhtU,11742
+datahub/configuration/common.py,sha256=i34Ec4tkTc2iy0ay82ZqPUeapUR3-ZKItR3gzI-0zTg,11730
 datahub/configuration/config_loader.py,sha256=hRzPFxkz-w9IqkpSa5vwCzSra1p49DyfeJNeyqGa8-4,6827
 datahub/configuration/connection_resolver.py,sha256=UsnV1_X8yivOykiifllkoKRn19eO6j_NTBWHC2Ob5Xg,1625
 datahub/configuration/datetimes.py,sha256=nayNc0mmlVKH6oVv9ud6C1dDUiZPGabW-YZxvrkosPg,2870
@@ -309,7 +309,7 @@ datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpi
 datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/debug/datahub_debug.py,sha256=LJmGWi2nqWiR_14nybqkIo14D4IzoJ08GaAtlcGvv98,11962
 datahub/ingestion/source/delta_lake/__init__.py,sha256=u5oqUeus81ONAtdl6o9Puw33ODSMun-0wLIamrZ4BUM,71
-datahub/ingestion/source/delta_lake/config.py,sha256=VAiZnykyFKcSXHJoNOMg_wwevXV2pUaBAOyYkAmC_X0,3526
+datahub/ingestion/source/delta_lake/config.py,sha256=fFBYOktFxidj7bdA0PNFgGD1pLqrlOK4Rn13npW82Uw,3746
 datahub/ingestion/source/delta_lake/delta_lake_utils.py,sha256=VqIDPEXepOnlk4oWMeRaneSpQBlWmlCKAa1wGUl1sfk,1525
 datahub/ingestion/source/delta_lake/report.py,sha256=uR4e4QA_jv8lL3CV-wE5t43H8pUqrGmx_ItLqN9flPI,587
 datahub/ingestion/source/delta_lake/source.py,sha256=IeYW-BAGxDy8gZzBoaLT3C2i-qbhxEEsWLZeC6Tdsfs,14191
@@ -595,7 +595,7 @@ datahub/ingestion/source_config/csv_enricher.py,sha256=IROxxfFJA56dHkmmbjjhb7h1p
 datahub/ingestion/source_config/operation_config.py,sha256=hxF2RM0jk0HUPXYiliMniXBC-wz-ZPcs90ZGLfHT8rE,3924
 datahub/ingestion/source_config/pulsar.py,sha256=zi3QTAw8CzzuwXgU-GUCuLyneT5pxHsLqZFyd15ECYs,5604
 datahub/ingestion/source_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source_report/ingestion_stage.py,sha256=0MY39QetRovYd1iBNSy0OW11YyaOsPaqhQi-1svmAcY,3106
+datahub/ingestion/source_report/ingestion_stage.py,sha256=7r_WUQ3mHH8nAlVu7QUZcj8CwTRORXRhAaATTgkVZ8c,3111
 datahub/ingestion/source_report/pulsar.py,sha256=f6CMNw8TyPp3tuSGsLLPEhSvoQLXwxtaaM6GmNvsANU,1119
 datahub/ingestion/source_report/time_window.py,sha256=9yI5l2S1DcF7ClvUHLeN8m62I5vlhV9k-aQqSZh2l7w,229
 datahub/ingestion/transformer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1019,7 +1019,7 @@ datahub/sql_parsing/sql_parsing_aggregator.py,sha256=kxxSVe3YNoz_T2OG6-F30ZuXNSX
 datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
 datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
 datahub/sql_parsing/sqlglot_lineage.py,sha256=Zli78TtN8ow-uqNl_oloWT642a5bDGn22-FF0O0tqrs,66658
-datahub/sql_parsing/sqlglot_utils.py,sha256=zH8V9tAcSVO7Y8I3sIKPhs0D_9HzdNBlranBDmk1NB4,15454
+datahub/sql_parsing/sqlglot_utils.py,sha256=2CwrnDsmjcDlwjdBbmaiTVbWZjEAm3fr4ulEvTr3cZQ,15343
 datahub/sql_parsing/tool_meta_extractor.py,sha256=5JsLPcKjuXSrPGxNIhRvX72dFPmlV33-hyvhJwlWxCY,7543
 datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/telemetry/stats.py,sha256=TwaQisQlD2Bk0uw__pP6u3Ovz9r-Ip4pCwpnto4r5e0,959
@@ -1126,8 +1126,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.3.0rc5.dist-info/METADATA,sha256=k7aciCTdmq9GKnJBJOrO9nQZt5Os9gcongq5sM9F330,184426
-acryl_datahub-1.3.0rc5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.3.0rc5.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.3.0rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.3.0rc5.dist-info/RECORD,,
+acryl_datahub-1.3.0.1rc2.dist-info/METADATA,sha256=FUZjgTu0opHK6NEx8d0r9PRHAD5OdINPUEip2y-6LII,184432
+acryl_datahub-1.3.0.1rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.3.0.1rc2.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.3.0.1rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.3.0.1rc2.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.3.0rc5"
+__version__ = "1.3.0.1rc2"
 
 
 def is_dev_mode() -> bool:

datahub/configuration/common.py

@@ -173,11 +173,11 @@ class ConnectionModel(BaseModel):
     """Represents the config associated with a connection"""
 
     class Config:
-        if PYDANTIC_VERSION_2:  # noqa: SIM108
+        if PYDANTIC_VERSION_2:
             extra = "allow"
         else:
             extra = Extra.allow
-        underscore_attrs_are_private = True
+            underscore_attrs_are_private = True
 
 
 class TransformerSemantics(ConfigEnum):

datahub/ingestion/source/delta_lake/config.py

@@ -13,8 +13,9 @@ from datahub.configuration.source_common import (
 )
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.aws.s3_util import is_s3_uri
-from datahub.ingestion.source.state.stateful_ingestion_base import (
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulIngestionConfigBase,
+    StatefulStaleMetadataRemovalConfig,
 )
 
 # hide annoying debug errors from py4j
@@ -39,9 +40,7 @@ class S3(ConfigModel):
 
 
 class DeltaLakeSourceConfig(
-    PlatformInstanceConfigMixin,
-    EnvConfigMixin,
-    StatefulIngestionConfigBase,
+    PlatformInstanceConfigMixin, EnvConfigMixin, StatefulIngestionConfigBase
 ):
     base_path: str = Field(
         description="Path to table (s3 or local file system). If path is not a delta table path "
@@ -80,6 +79,11 @@ class DeltaLakeSourceConfig(
 
     s3: Optional[S3] = Field(None)
 
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
+        default=None,
+        description="Stateful Ingestion Config with stale metadata removal",
+    )
+
     @cached_property
     def is_s3(self):
         return is_s3_uri(self.base_path or "")

datahub/ingestion/source_report/ingestion_stage.py

@@ -4,7 +4,6 @@ from contextlib import AbstractContextManager
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Tuple
 
 from datahub.utilities.perf_timer import PerfTimer
 from datahub.utilities.stats_collections import TopKDict
@@ -38,9 +37,7 @@ class IngestionStageReport:
     ingestion_high_stage_seconds: dict[IngestionHighStage, float] = field(
         default_factory=lambda: defaultdict(float)
     )
-    ingestion_stage_durations: TopKDict[Tuple[IngestionHighStage, str], float] = field(
-        default_factory=TopKDict
-    )
+    ingestion_stage_durations: TopKDict[str, float] = field(default_factory=TopKDict)
 
     def new_stage(
         self, stage: str, high_stage: IngestionHighStage = IngestionHighStage._UNDEFINED
@@ -81,9 +78,9 @@ class IngestionStageContext(AbstractContextManager):
                 f"Time spent in stage <{self._ingestion_stage}>: {elapsed} seconds",
                 stacklevel=2,
             )
-            self._report.ingestion_stage_durations[
-                (self._high_stage, self._ingestion_stage)
-            ] = elapsed
+            # Store tuple as string to avoid serialization errors
+            key = f"({self._high_stage.value}, {self._ingestion_stage})"
+            self._report.ingestion_stage_durations[key] = elapsed
         else:
             logger.info(
                 f"Time spent in stage <{self._high_stage.value}>: {elapsed} seconds",

datahub/sql_parsing/sqlglot_utils.py

@@ -40,9 +40,6 @@ def _get_dialect_str(platform: str) -> str:
         # let the fuzzy resolution logic handle it.
         # MariaDB is a fork of MySQL, so we reuse the same dialect.
         return "mysql, normalization_strategy = lowercase"
-    # Dremio is based upon drill. Not 100% compatibility
-    elif platform == "dremio":
-        return "drill"
     else:
         return platform