acryl-datahub-gx-plugin 1.3.1.10__py3-none-any.whl → 1.4.0rc1__py3-none-any.whl

{acryl_datahub_gx_plugin-1.3.1.10.dist-info → acryl_datahub_gx_plugin-1.4.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-gx-plugin
-Version: 1.3.1.10
+Version: 1.4.0rc1
 Summary: Datahub GX plugin to capture executions and send to Datahub
 Home-page: https://docs.datahub.com/
 License: Apache-2.0
@@ -21,88 +21,88 @@ Classifier: Environment :: MacOS X
 Classifier: Topic :: Software Development
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+Requires-Dist: sqlalchemy<2,>=1.4.39
 Requires-Dist: requests_file
 Requires-Dist: great-expectations<1.0.0,>=0.17.15
 Requires-Dist: pydantic>=2.1.0
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.4.0rc1
 Requires-Dist: requests
-Requires-Dist: sqlalchemy<2,>=1.4.39
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.10
 Requires-Dist: traitlets!=5.2.2
 Provides-Extra: ignore
 Provides-Extra: dev
-Requires-Dist: coverage>=5.1; extra == "dev"
-Requires-Dist: pydantic>=2.1.0; extra == "dev"
-Requires-Dist: types-six; extra == "dev"
-Requires-Dist: jsonpickle; extra == "dev"
-Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: tox; extra == "dev"
 Requires-Dist: twine; extra == "dev"
-Requires-Dist: ruff==0.11.7; extra == "dev"
-Requires-Dist: types-click==0.1.12; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: great-expectations<1.0.0,>=0.17.15; extra == "dev"
-Requires-Dist: requests-mock; extra == "dev"
 Requires-Dist: mypy==1.17.1; extra == "dev"
-Requires-Dist: types-setuptools; extra == "dev"
-Requires-Dist: requests_file; extra == "dev"
-Requires-Dist: pytest>=6.2.2; extra == "dev"
-Requires-Dist: sqlalchemy<2,>=1.4.39; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.4.0rc1; extra == "dev"
+Requires-Dist: jsonpickle; extra == "dev"
+Requires-Dist: packaging; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
 Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
-Requires-Dist: requests; extra == "dev"
 Requires-Dist: types-tabulate; extra == "dev"
-Requires-Dist: sqlalchemy-stubs; extra == "dev"
-Requires-Dist: build; extra == "dev"
-Requires-Dist: packaging; extra == "dev"
+Requires-Dist: requests; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: sqlalchemy<2,>=1.4.39; extra == "dev"
 Requires-Dist: types-toml; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.16.0; extra == "dev"
+Requires-Dist: traitlets!=5.2.2; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
 Requires-Dist: types-dataclasses; extra == "dev"
-Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: requests_file; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: pydantic>=2.1.0; extra == "dev"
+Requires-Dist: types-six; extra == "dev"
 Requires-Dist: freezegun; extra == "dev"
-Requires-Dist: tox; extra == "dev"
-Requires-Dist: traitlets!=5.2.2; extra == "dev"
-Requires-Dist: types-pytz; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
 Requires-Dist: types-freezegun; extra == "dev"
-Requires-Dist: types-cachetools; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.10; extra == "dev"
 Requires-Dist: deepdiff!=8.0.0; extra == "dev"
-Requires-Dist: pytest-asyncio>=0.16.0; extra == "dev"
+Requires-Dist: types-pytz; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: great-expectations<1.0.0,>=0.17.15; extra == "dev"
+Requires-Dist: requests-mock; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
 Provides-Extra: integration-tests
-Requires-Dist: coverage>=5.1; extra == "integration-tests"
-Requires-Dist: pydantic>=2.1.0; extra == "integration-tests"
-Requires-Dist: types-six; extra == "integration-tests"
+Requires-Dist: tox; extra == "integration-tests"
 Requires-Dist: pyspark; extra == "integration-tests"
-Requires-Dist: acryl-datahub[testing-utils]==1.3.1.10; extra == "integration-tests"
-Requires-Dist: jsonpickle; extra == "integration-tests"
-Requires-Dist: types-python-dateutil; extra == "integration-tests"
 Requires-Dist: twine; extra == "integration-tests"
-Requires-Dist: ruff==0.11.7; extra == "integration-tests"
-Requires-Dist: types-click==0.1.12; extra == "integration-tests"
-Requires-Dist: types-PyYAML; extra == "integration-tests"
-Requires-Dist: great-expectations<1.0.0,>=0.17.15; extra == "integration-tests"
-Requires-Dist: requests-mock; extra == "integration-tests"
-Requires-Dist: psycopg2-binary; extra == "integration-tests"
 Requires-Dist: mypy==1.17.1; extra == "integration-tests"
-Requires-Dist: types-setuptools; extra == "integration-tests"
-Requires-Dist: requests_file; extra == "integration-tests"
-Requires-Dist: pytest>=6.2.2; extra == "integration-tests"
-Requires-Dist: sqlalchemy<2,>=1.4.39; extra == "integration-tests"
+Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.4.0rc1; extra == "integration-tests"
+Requires-Dist: jsonpickle; extra == "integration-tests"
+Requires-Dist: packaging; extra == "integration-tests"
+Requires-Dist: build; extra == "integration-tests"
+Requires-Dist: psycopg2-binary; extra == "integration-tests"
+Requires-Dist: types-requests; extra == "integration-tests"
 Requires-Dist: pytest-cov>=2.8.1; extra == "integration-tests"
-Requires-Dist: requests; extra == "integration-tests"
+Requires-Dist: acryl-datahub[testing-utils]==1.4.0rc1; extra == "integration-tests"
 Requires-Dist: types-tabulate; extra == "integration-tests"
-Requires-Dist: sqlalchemy-stubs; extra == "integration-tests"
-Requires-Dist: build; extra == "integration-tests"
-Requires-Dist: packaging; extra == "integration-tests"
+Requires-Dist: requests; extra == "integration-tests"
+Requires-Dist: ruff==0.11.7; extra == "integration-tests"
+Requires-Dist: types-PyYAML; extra == "integration-tests"
+Requires-Dist: sqlalchemy<2,>=1.4.39; extra == "integration-tests"
 Requires-Dist: types-toml; extra == "integration-tests"
-Requires-Dist: types-dataclasses; extra == "integration-tests"
-Requires-Dist: types-requests; extra == "integration-tests"
-Requires-Dist: freezegun; extra == "integration-tests"
-Requires-Dist: tox; extra == "integration-tests"
+Requires-Dist: types-python-dateutil; extra == "integration-tests"
+Requires-Dist: pytest-asyncio>=0.16.0; extra == "integration-tests"
 Requires-Dist: pytest-docker>=1.1.0; extra == "integration-tests"
 Requires-Dist: traitlets!=5.2.2; extra == "integration-tests"
-Requires-Dist: types-pytz; extra == "integration-tests"
+Requires-Dist: pytest>=6.2.2; extra == "integration-tests"
+Requires-Dist: types-dataclasses; extra == "integration-tests"
+Requires-Dist: requests_file; extra == "integration-tests"
+Requires-Dist: coverage>=5.1; extra == "integration-tests"
+Requires-Dist: pydantic>=2.1.0; extra == "integration-tests"
+Requires-Dist: types-six; extra == "integration-tests"
+Requires-Dist: freezegun; extra == "integration-tests"
+Requires-Dist: types-setuptools; extra == "integration-tests"
 Requires-Dist: types-freezegun; extra == "integration-tests"
-Requires-Dist: types-cachetools; extra == "integration-tests"
-Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.10; extra == "integration-tests"
 Requires-Dist: deepdiff!=8.0.0; extra == "integration-tests"
-Requires-Dist: pytest-asyncio>=0.16.0; extra == "integration-tests"
+Requires-Dist: types-pytz; extra == "integration-tests"
+Requires-Dist: types-click==0.1.12; extra == "integration-tests"
+Requires-Dist: great-expectations<1.0.0,>=0.17.15; extra == "integration-tests"
+Requires-Dist: requests-mock; extra == "integration-tests"
+Requires-Dist: sqlalchemy-stubs; extra == "integration-tests"
+Requires-Dist: types-cachetools; extra == "integration-tests"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type

acryl_datahub_gx_plugin-1.4.0rc1.dist-info/RECORD

@@ -0,0 +1,8 @@
+datahub_gx_plugin/__init__.py,sha256=EAwU2Z4u9abD5xHwcfgwo1xvK8-lSfByM-cp0IMw4mQ,69
+datahub_gx_plugin/_version.py,sha256=ioPtKcLHTY5xpvxAxlsbIJoRSU0XOZQPW9dd8N_ZjLI,136
+datahub_gx_plugin/action.py,sha256=jG8CXw6jszl8R_Oeq9Y0M8IPQp0iIdqNGxa4YI9LTVo,43401
+acryl_datahub_gx_plugin-1.4.0rc1.dist-info/METADATA,sha256=lDP9FxQ60_6Gcye8Dg4IvRHEGLh01-Q4ykRflHi_188,5645
+acryl_datahub_gx_plugin-1.4.0rc1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+acryl_datahub_gx_plugin-1.4.0rc1.dist-info/entry_points.txt,sha256=MPipKHfWSerlcRwWt8OP93TPCVxpposDMyILo9cszbM,88
+acryl_datahub_gx_plugin-1.4.0rc1.dist-info/top_level.txt,sha256=ryn3FMtO1isrM-TbO-SRYy_-31fUpgdEBmBSrAmvL1c,18
+acryl_datahub_gx_plugin-1.4.0rc1.dist-info/RECORD,,

datahub_gx_plugin/_version.py

@@ -1,3 +1,3 @@
 # Published at https://pypi.org/project/acryl-datahub-gx-plugin/.
 __package_name__ = "acryl-datahub-gx-plugin"
-__version__ = "1.3.1.10"
+__version__ = "1.4.0rc1"

datahub_gx_plugin/action.py

@@ -38,9 +38,11 @@ from sqlalchemy.engine.url import make_url
 
 import datahub.emitter.mce_builder as builder
 from datahub.cli.env_utils import get_boolean_env_variable
+from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.rest_emitter import DatahubRestEmitter
 from datahub.emitter.serialization_helper import pre_json_transform
+from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.graph.config import ClientMode
 from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import (
     get_platform_from_sqlalchemy_uri,
@@ -62,9 +64,16 @@ from datahub.metadata.com.linkedin.pegasus2avro.assertion import (
     DatasetAssertionScope,
 )
 from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance
-from datahub.metadata.schema_classes import PartitionSpecClass, PartitionTypeClass
+from datahub.metadata.schema_classes import (
+    ChangeTypeClass,
+    GenericAspectClass,
+    MetadataChangeProposalClass,
+    PartitionSpecClass,
+    PartitionTypeClass,
+)
 from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result
 from datahub.utilities.urns.dataset_urn import DatasetUrn
+from datahub.utilities.urns.urn import guess_entity_type
 
 # TODO: move this and version check used in tests to some common module
 try:
@@ -192,15 +201,16 @@ class DataHubValidationAction(ValidationAction):
             logger.info("Sending metadata to datahub ...")
             logger.info("Dataset URN - {urn}".format(urn=datasets[0]["dataset_urn"]))
 
+            graph = emitter.to_graph()
+
             for assertion in assertions:
                 logger.info(
                     "Assertion URN - {urn}".format(urn=assertion["assertionUrn"])
                 )
 
                 # Construct a MetadataChangeProposalWrapper object.
-                assertion_info_mcp = MetadataChangeProposalWrapper(
-                    entityUrn=assertion["assertionUrn"],
-                    aspect=assertion["assertionInfo"],
+                assertion_info_mcp = self._build_assertion_info_mcp(
+                    graph, assertion["assertionUrn"], assertion["assertionInfo"]
                 )
                 emitter.emit_mcp(assertion_info_mcp)
 
@@ -231,6 +241,75 @@ class DataHubValidationAction(ValidationAction):
 
         return {"datahub_notification_result": result}
 
+    def _build_assertion_info_mcp(
+        self,
+        graph: DataHubGraph,
+        assertion_urn: str,
+        assertion_info: AssertionInfo,
+    ) -> Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]:
+        try:
+            existing_info = graph.get_aspect(assertion_urn, AssertionInfo)
+        except Exception:
+            logger.warning(
+                "Failed to check existing assertionInfo. Falling back to upsert.",
+                exc_info=True,
+            )
+            existing_info = None
+
+        if existing_info is None:
+            return MetadataChangeProposalWrapper(
+                entityUrn=assertion_urn,
+                aspect=assertion_info,
+            )
+
+        return self._build_assertion_info_patch(assertion_urn, assertion_info)
+
+    def _build_assertion_info_patch(
+        self,
+        assertion_urn: str,
+        assertion_info: AssertionInfo,
+    ) -> MetadataChangeProposalClass:
+        assertion_info_obj = assertion_info.to_obj()
+        patch_ops = []
+
+        if "type" in assertion_info_obj:
+            patch_ops.append(
+                {"op": "add", "path": "/type", "value": assertion_info_obj["type"]}
+            )
+        if "datasetAssertion" in assertion_info_obj:
+            patch_ops.append(
+                {
+                    "op": "add",
+                    "path": "/datasetAssertion",
+                    "value": assertion_info_obj["datasetAssertion"],
+                }
+            )
+        custom_properties = assertion_info_obj.get("customProperties") or {}
+        expectation_suite_name = custom_properties.get("expectation_suite_name")
+        if expectation_suite_name is not None:
+            patch_ops.append(
+                {
+                    "op": "add",
+                    "path": "/customProperties/expectation_suite_name",
+                    "value": expectation_suite_name,
+                }
+            )
+
+        aspect_payload = {
+            "patch": pre_json_transform(patch_ops),
+            "forceGenericPatch": True,
+        }
+        return MetadataChangeProposalClass(
+            entityUrn=assertion_urn,
+            entityType=guess_entity_type(assertion_urn),
+            changeType=ChangeTypeClass.PATCH,
+            aspectName="assertionInfo",
+            aspect=GenericAspectClass(
+                value=json.dumps(pre_json_transform(aspect_payload)).encode(),
+                contentType=JSON_PATCH_CONTENT_TYPE,
+            ),
+        )
+
     def get_assertions_with_results(
         self,
         validation_result_suite,

acryl_datahub_gx_plugin-1.3.1.10.dist-info/RECORD

@@ -1,8 +0,0 @@
-datahub_gx_plugin/__init__.py,sha256=EAwU2Z4u9abD5xHwcfgwo1xvK8-lSfByM-cp0IMw4mQ,69
-datahub_gx_plugin/_version.py,sha256=4Fq_nLSQx3815zzf8MmBmv7QVP1Wn5smqeGsi0LNn9E,136
-datahub_gx_plugin/action.py,sha256=d6H0daSN0SrZ46ZfNPdVwldQYk1eJcSryHSC6cWnjM8,40665
-acryl_datahub_gx_plugin-1.3.1.10.dist-info/METADATA,sha256=sq-Y2Go6ayBSnitLuxgLlp0HXpiWqfj2G8QUF3HnDgo,5645
-acryl_datahub_gx_plugin-1.3.1.10.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-acryl_datahub_gx_plugin-1.3.1.10.dist-info/entry_points.txt,sha256=MPipKHfWSerlcRwWt8OP93TPCVxpposDMyILo9cszbM,88
-acryl_datahub_gx_plugin-1.3.1.10.dist-info/top_level.txt,sha256=ryn3FMtO1isrM-TbO-SRYy_-31fUpgdEBmBSrAmvL1c,18
-acryl_datahub_gx_plugin-1.3.1.10.dist-info/RECORD,,