acryl-datahub 1.2.0.5rc2__py3-none-any.whl → 1.2.0.6rc1__py3-none-any.whl

{acryl_datahub-1.2.0.5rc2.dist-info → acryl_datahub-1.2.0.6rc1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.5rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.6rc1.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=09t1S7gSEatlnU79aCcPBAO49NhdKd6QbnqRNsiVmM4,323
+datahub/_version.py,sha256=UpTxFRBPZj7BDFXx0obVG8eaUPk2TF8H9bqlPMVnt2A,323
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -148,7 +148,7 @@ datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPs
 datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
 datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
 datahub/ingestion/api/registry.py,sha256=LbdZr89465Lj7ptQRVB4vI1JR1igWABvQFj9-WX63bI,7454
-datahub/ingestion/api/report.py,sha256=jdAfqkAn1tlwBJBmC_t0-6KzKhKBxO8N3y_NziyBXSU,17889
+datahub/ingestion/api/report.py,sha256=-xduHhIRUgf5G51mUb3uTi6GBxVli6ZK25AS5ikXuII,18312
 datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
 datahub/ingestion/api/sink.py,sha256=GZt48PV56FAhNoma-V5EwwRZvezhb40YH_zprm8_Yo0,4961
 datahub/ingestion/api/source.py,sha256=uf0fNbiOy0bS_aKFOcNv6NvuZe0LSDIDdNza9hraP7s,21857
@@ -1104,8 +1104,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.5rc2.dist-info/METADATA,sha256=nbHrt6MYRbQZrnGSwe1h_KkxKHHRSgve70uYZnA8ob4,182056
-acryl_datahub-1.2.0.5rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.5rc2.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
-acryl_datahub-1.2.0.5rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.5rc2.dist-info/RECORD,,
+acryl_datahub-1.2.0.6rc1.dist-info/METADATA,sha256=d_JiiaPjg1a5n4u__cmZ1gBZ7FpY1qf6flPFgzZM2D8,182056
+acryl_datahub-1.2.0.6rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.6rc1.dist-info/entry_points.txt,sha256=bnGf6eX9UhiW8yVHtt6MJCVcmLErvrVQxTJAayA-PKc,9885
+acryl_datahub-1.2.0.6rc1.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.6rc1.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.5rc2"
+__version__ = "1.2.0.6rc1"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/api/report.py

@@ -23,6 +23,7 @@ from datahub.ingestion.autogenerated.lineage_helper import is_lineage_aspect
 from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     MetadataChangeProposalClass,
+    StatusClass,
     SubTypesClass,
     UpstreamLineageClass,
 )
@@ -161,6 +162,7 @@ class SourceReportSubtypes:
     entity_type: str
     subType: str = field(default="unknown")
     aspects: Dict[str, int] = field(default_factory=dict)
+    soft_deleted: bool = field(default=False)
 
 
 class ReportAttribute(BaseModel):
@@ -217,6 +219,7 @@ class ExamplesReport(Report, Closeable):
                 "entityType": lambda val: val.entity_type,
                 "subTypes": lambda val: val.subType,
                 "aspects": lambda val: json.dumps(val.aspects),
+                "soft_deleted": lambda val: val.soft_deleted,
             },
         )
 
@@ -376,6 +379,13 @@ class ExamplesReport(Report, Closeable):
                 subType=sub_type,
                 aspects=aspects_dict,
             )
+        if (
+            isinstance(mcp.aspect, StatusClass)
+            and mcp is not None
+            and mcp.aspect is not None
+        ):
+            self._file_based_dict[urn].soft_deleted = mcp.aspect.removed
+            self._file_based_dict.mark_dirty(urn)
 
     def _store_workunit_data(self, wu: MetadataWorkUnit) -> None:
         urn = wu.get_urn()
@@ -401,7 +411,8 @@ class ExamplesReport(Report, Closeable):
         query = """
         SELECT entityType, subTypes, aspects, count(*) as count
         FROM urn_aspects 
-        group by entityType, subTypes, aspects
+        WHERE soft_deleted = 0
+        GROUP BY entityType, subTypes, aspects
         """
 
         entity_subtype_aspect_counts: Dict[str, Dict[str, Dict[str, int]]] = (
@@ -410,6 +421,7 @@ class ExamplesReport(Report, Closeable):
         entity_subtype_aspect_counts_exist: Dict[str, Dict[str, Dict[str, int]]] = (
             defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
         )
+
         for row in self._file_based_dict.sql_query(query):
             entity_type = row["entityType"]
             sub_type = row["subTypes"]