acryl-datahub 1.3.0.1rc1__py3-none-any.whl → 1.3.0.1rc2__py3-none-any.whl

{acryl_datahub-1.3.0.1rc1.dist-info → acryl_datahub-1.3.0.1rc2.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.3.0.1rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.3.0.1rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=R7AO3BvsCQaRcQe_NV5EtywuCZj3jLQL4SqalUnrmTc,323
+datahub/_version.py,sha256=G3S5qsHOL2-5yNrU2qQxhGJwD63r-1qLun37Mq1ejLo,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -309,7 +309,7 @@ datahub/ingestion/source/dbt/dbt_tests.py,sha256=pOZJaP4VsbaE5j4qVlE_E3ifno_KQpi
 datahub/ingestion/source/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/debug/datahub_debug.py,sha256=LJmGWi2nqWiR_14nybqkIo14D4IzoJ08GaAtlcGvv98,11962
 datahub/ingestion/source/delta_lake/__init__.py,sha256=u5oqUeus81ONAtdl6o9Puw33ODSMun-0wLIamrZ4BUM,71
-datahub/ingestion/source/delta_lake/config.py,sha256=VAiZnykyFKcSXHJoNOMg_wwevXV2pUaBAOyYkAmC_X0,3526
+datahub/ingestion/source/delta_lake/config.py,sha256=fFBYOktFxidj7bdA0PNFgGD1pLqrlOK4Rn13npW82Uw,3746
 datahub/ingestion/source/delta_lake/delta_lake_utils.py,sha256=VqIDPEXepOnlk4oWMeRaneSpQBlWmlCKAa1wGUl1sfk,1525
 datahub/ingestion/source/delta_lake/report.py,sha256=uR4e4QA_jv8lL3CV-wE5t43H8pUqrGmx_ItLqN9flPI,587
 datahub/ingestion/source/delta_lake/source.py,sha256=IeYW-BAGxDy8gZzBoaLT3C2i-qbhxEEsWLZeC6Tdsfs,14191
@@ -1126,8 +1126,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.3.0.1rc1.dist-info/METADATA,sha256=7XyrCvdsnuEzzHSrBD4-gyr723Kq-Nj2Jawvh5sOz1M,184432
-acryl_datahub-1.3.0.1rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.3.0.1rc1.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.3.0.1rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.3.0.1rc1.dist-info/RECORD,,
+acryl_datahub-1.3.0.1rc2.dist-info/METADATA,sha256=FUZjgTu0opHK6NEx8d0r9PRHAD5OdINPUEip2y-6LII,184432
+acryl_datahub-1.3.0.1rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.3.0.1rc2.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.3.0.1rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.3.0.1rc2.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.3.0.1rc1"
+__version__ = "1.3.0.1rc2"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/source/delta_lake/config.py

@@ -13,8 +13,9 @@ from datahub.configuration.source_common import (
 )
 from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig
 from datahub.ingestion.source.aws.s3_util import is_s3_uri
-from datahub.ingestion.source.state.stateful_ingestion_base import (
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StatefulIngestionConfigBase,
+    StatefulStaleMetadataRemovalConfig,
 )
 
 # hide annoying debug errors from py4j
@@ -39,9 +40,7 @@ class S3(ConfigModel):
 
 
 class DeltaLakeSourceConfig(
-    PlatformInstanceConfigMixin,
-    EnvConfigMixin,
-    StatefulIngestionConfigBase,
+    PlatformInstanceConfigMixin, EnvConfigMixin, StatefulIngestionConfigBase
 ):
     base_path: str = Field(
         description="Path to table (s3 or local file system). If path is not a delta table path "
@@ -80,6 +79,11 @@ class DeltaLakeSourceConfig(
 
     s3: Optional[S3] = Field(None)
 
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
+        default=None,
+        description="Stateful Ingestion Config with stale metadata removal",
+    )
+
     @cached_property
     def is_s3(self):
         return is_s3_uri(self.base_path or "")