acryl-datahub 1.2.0.6__py3-none-any.whl → 1.2.0.7rc1__py3-none-any.whl

datahub/metadata/schemas/ChartInfo.avsc

@@ -188,7 +188,8 @@
           "createdActor": "inputEdges/*/created/actor",
           "createdOn": "inputEdges/*/created/time",
           "entityTypes": [
-            "dataset"
+            "dataset",
+            "chart"
           ],
           "isLineage": true,
           "name": "Consumes",

datahub/metadata/schemas/DataHubPageModuleProperties.avsc

@@ -148,6 +148,15 @@
                     "name": "assetUrns",
                     "Urn": "Urn",
                     "urn_is_array": true
+                  },
+                  {
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "dynamicFilterJson",
+                    "default": null,
+                    "doc": "Optional dynamic filters\n\nThe stringified json representing the logical predicate built in the UI to select assets.\nThis predicate is turned into orFilters to send through graphql since graphql doesn't support\narbitrary nesting. This string is used to restore the UI for this logical predicate."
                   }
                 ],
                 "doc": "The params required if the module is type ASSET_COLLECTION"

datahub/metadata/schemas/InstitutionalMemory.avsc

@@ -75,6 +75,15 @@
               },
               "name": "createStamp",
               "doc": "Audit stamp associated with creation of this record"
+            },
+            {
+              "type": [
+                "null",
+                "com.linkedin.pegasus2avro.common.AuditStamp"
+              ],
+              "name": "updateStamp",
+              "default": null,
+              "doc": "Audit stamp associated with updation of this record"
             }
           ],
           "doc": "Metadata corresponding to a record of institutional memory."

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -332,7 +332,8 @@
                             "createdActor": "inputEdges/*/created/actor",
                             "createdOn": "inputEdges/*/created/time",
                             "entityTypes": [
-                              "dataset"
+                              "dataset",
+                              "chart"
                             ],
                             "isLineage": true,
                             "name": "Consumes",
@@ -747,6 +748,75 @@
                                 "name": "source",
                                 "default": null,
                                 "doc": "Source information for the ownership"
+                              },
+                              {
+                                "Searchable": {
+                                  "/actor": {
+                                    "fieldName": "ownerAttributionActors",
+                                    "fieldType": "URN",
+                                    "queryByDefault": false
+                                  },
+                                  "/source": {
+                                    "fieldName": "ownerAttributionSources",
+                                    "fieldType": "URN",
+                                    "queryByDefault": false
+                                  },
+                                  "/time": {
+                                    "fieldName": "ownerAttributionDates",
+                                    "fieldType": "DATETIME",
+                                    "queryByDefault": false
+                                  }
+                                },
+                                "type": [
+                                  "null",
+                                  {
+                                    "type": "record",
+                                    "name": "MetadataAttribution",
+                                    "namespace": "com.linkedin.pegasus2avro.common",
+                                    "fields": [
+                                      {
+                                        "type": "long",
+                                        "name": "time",
+                                        "doc": "When this metadata was updated."
+                                      },
+                                      {
+                                        "java": {
+                                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                                        },
+                                        "type": "string",
+                                        "name": "actor",
+                                        "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                                        "Urn": "Urn"
+                                      },
+                                      {
+                                        "java": {
+                                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                                        },
+                                        "type": [
+                                          "null",
+                                          "string"
+                                        ],
+                                        "name": "source",
+                                        "default": null,
+                                        "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                                        "Urn": "Urn"
+                                      },
+                                      {
+                                        "type": {
+                                          "type": "map",
+                                          "values": "string"
+                                        },
+                                        "name": "sourceDetail",
+                                        "default": {},
+                                        "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                                      }
+                                    ],
+                                    "doc": "Information about who, why, and how this metadata was applied"
+                                  }
+                                ],
+                                "name": "attribution",
+                                "default": null,
+                                "doc": "Information about who, why, and how this metadata was applied"
                               }
                             ],
                             "doc": "Ownership information"
@@ -883,50 +953,7 @@
                                 },
                                 "type": [
                                   "null",
-                                  {
-                                    "type": "record",
-                                    "name": "MetadataAttribution",
-                                    "namespace": "com.linkedin.pegasus2avro.common",
-                                    "fields": [
-                                      {
-                                        "type": "long",
-                                        "name": "time",
-                                        "doc": "When this metadata was updated."
-                                      },
-                                      {
-                                        "java": {
-                                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                                        },
-                                        "type": "string",
-                                        "name": "actor",
-                                        "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
-                                        "Urn": "Urn"
-                                      },
-                                      {
-                                        "java": {
-                                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                                        },
-                                        "type": [
-                                          "null",
-                                          "string"
-                                        ],
-                                        "name": "source",
-                                        "default": null,
-                                        "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
-                                        "Urn": "Urn"
-                                      },
-                                      {
-                                        "type": {
-                                          "type": "map",
-                                          "values": "string"
-                                        },
-                                        "name": "sourceDetail",
-                                        "default": {},
-                                        "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
-                                      }
-                                    ],
-                                    "doc": "Information about who, why, and how this metadata was applied"
-                                  }
+                                  "com.linkedin.pegasus2avro.common.MetadataAttribution"
                                 ],
                                 "name": "attribution",
                                 "default": null,
@@ -1107,6 +1134,15 @@
                                 "type": "com.linkedin.pegasus2avro.common.AuditStamp",
                                 "name": "createStamp",
                                 "doc": "Audit stamp associated with creation of this record"
+                              },
+                              {
+                                "type": [
+                                  "null",
+                                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                                ],
+                                "name": "updateStamp",
+                                "default": null,
+                                "doc": "Audit stamp associated with updation of this record"
                               }
                             ],
                             "doc": "Metadata corresponding to a record of institutional memory."

datahub/metadata/schemas/Ownership.avsc

@@ -162,6 +162,75 @@
               "name": "source",
               "default": null,
               "doc": "Source information for the ownership"
+            },
+            {
+              "Searchable": {
+                "/actor": {
+                  "fieldName": "ownerAttributionActors",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "/source": {
+                  "fieldName": "ownerAttributionSources",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "/time": {
+                  "fieldName": "ownerAttributionDates",
+                  "fieldType": "DATETIME",
+                  "queryByDefault": false
+                }
+              },
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "attribution",
+              "default": null,
+              "doc": "Information about who, why, and how this metadata was applied"
             }
           ],
           "doc": "Ownership information"

datahub/metadata/schemas/StructuredProperties.avsc

@@ -94,6 +94,75 @@
               "name": "lastModified",
               "default": null,
               "doc": "Audit stamp containing who last modified this relationship edge and when"
+            },
+            {
+              "Searchable": {
+                "/actor": {
+                  "fieldName": "structuredPropertyAttributionActors",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "/source": {
+                  "fieldName": "structuredPropertyAttributionSources",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "/time": {
+                  "fieldName": "structuredPropertyAttributionDates",
+                  "fieldType": "DATETIME",
+                  "queryByDefault": false
+                }
+              },
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "attribution",
+              "default": null,
+              "doc": "Information about who, why, and how this metadata was applied"
             }
           ]
         }

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -23,6 +23,9 @@
       "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
     },
     {
+      "Searchable": {
+        "fieldType": "URN"
+      },
       "UrnValidation": {
         "entityTypes": [
           "dataType"

datahub/metadata/schemas/__init__.py

@@ -15,10 +15,10 @@ import pathlib
 def _load_schema(schema_name: str) -> str:
     return (pathlib.Path(__file__).parent / f"{schema_name}.avsc").read_text()
 
-def getMetadataChangeProposalSchema() -> str:
-    return _load_schema("MetadataChangeProposal")
-
 def getMetadataChangeEventSchema() -> str:
     return _load_schema("MetadataChangeEvent")
 
+def getMetadataChangeProposalSchema() -> str:
+    return _load_schema("MetadataChangeProposal")
+
 # fmt: on

datahub/sdk/lineage_client.py

@@ -165,11 +165,7 @@ class LineageClient:
         ] = False,
         transformation_text: Optional[str] = None,
     ) -> None:
-        ...
-
-        """
-        Add dataset-to-dataset lineage with column-level mapping.
-        """
+        """Add dataset-to-dataset lineage with column-level mapping."""
 
     @overload
     def add_lineage(
@@ -178,11 +174,7 @@ class LineageClient:
         upstream: Union[DatajobUrnOrStr],
         downstream: DatasetUrnOrStr,
     ) -> None:
-        ...
-
-        """
-        Add dataset-to-datajob or dataset-to-mlmodel lineage.
-        """
+        """Add dataset-to-datajob or dataset-to-mlmodel lineage."""
 
     @overload
     def add_lineage(
@@ -191,11 +183,7 @@ class LineageClient:
         upstream: Union[DatasetUrnOrStr, DatajobUrnOrStr],
         downstream: DatajobUrnOrStr,
     ) -> None:
-        ...
-
-        """
-        Add datajob-to-dataset or datajob-to-datajob lineage.
-        """
+        """Add datajob-to-dataset or datajob-to-datajob lineage."""
 
     @overload
     def add_lineage(
@@ -204,11 +192,7 @@ class LineageClient:
         upstream: Union[DashboardUrnOrStr, DatasetUrnOrStr, ChartUrnOrStr],
         downstream: DashboardUrnOrStr,
     ) -> None:
-        ...
-
-        """
-        Add dashboard-to-dashboard or dashboard-to-dataset lineage.
-        """
+        """Add dashboard-to-dashboard or dashboard-to-dataset lineage."""
 
     @overload
     def add_lineage(
@@ -217,10 +201,7 @@ class LineageClient:
         upstream: DatasetUrnOrStr,
         downstream: ChartUrnOrStr,
     ) -> None:
-        ...
-        """
-        Add dataset-to-chart lineage.
-        """
+        """Add dataset-to-chart lineage."""
 
     # The actual implementation that handles all overloaded cases
     def add_lineage(
@@ -237,8 +218,7 @@ class LineageClient:
         ] = False,
         transformation_text: Optional[str] = None,
     ) -> None:
-        """
-        Add lineage between two entities.
+        """Add lineage between two entities.
 
         This flexible method handles different combinations of entity types:
         - dataset to dataset

datahub/sdk/main_client.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Optional, overload
+from typing import TYPE_CHECKING, Optional, overload
 
 from datahub.errors import SdkUsageError
 from datahub.ingestion.graph.client import DataHubGraph, get_default_graph
@@ -9,6 +9,9 @@ from datahub.sdk.entity_client import EntityClient
 from datahub.sdk.lineage_client import LineageClient
 from datahub.sdk.search_client import SearchClient
 
+if TYPE_CHECKING:
+    from datahub.sdk.resolver_client import ResolverClient
+
 
 class DataHubClient:
     """Main client for interacting with DataHub.
@@ -104,13 +107,14 @@ class DataHubClient:
         return EntityClient(self)
 
     @property
-    def resolve(self):  # type: ignore[report-untyped-call]  # Not available due to circular import issues
+    def resolve(self) -> "ResolverClient":
         try:
             from acryl_datahub_cloud.sdk import (  # type: ignore[import-not-found]
                 ResolverClient,
             )
         except ImportError:
-            from datahub.sdk.resolver_client import (  # type: ignore[assignment]  # If the client is not installed, use the one from the SDK
+            # If the client is not installed, use the one from the SDK.
+            from datahub.sdk.resolver_client import (  # type: ignore[assignment]
                 ResolverClient,
             )
         return ResolverClient(self)

datahub/sdk/search_filters.py

@@ -384,6 +384,21 @@ def _filter_discriminator(v: Any) -> Optional[str]:
     return None
 
 
+def _parse_and_like_filter(value: Any) -> Any:
+    # Do not parse if filter is already of type and/or/not or a custom condition
+    # also do not parse container filter if direct_descendants_only is specified
+    if (
+        isinstance(value, dict)
+        and not set(value.keys()).intersection(
+            {"and", "or", "not", "field", "condition", "direct_descendants_only"}
+        )
+        and len(value) > 1
+    ):
+        return {"and": [{k: v} for k, v in value.items()]}
+
+    return value
+
+
 if TYPE_CHECKING or not PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR:
     # The `not TYPE_CHECKING` bit is required to make the linter happy,
     # since we currently only run mypy with pydantic v1.
@@ -445,6 +460,7 @@ else:
             ],
             Discriminator(_filter_discriminator),
         ],
+        pydantic.BeforeValidator(_parse_and_like_filter),
         pydantic.BeforeValidator(_parse_json_from_string),
     ]
 

datahub/specific/aspect_helpers/siblings.py

@@ -0,0 +1,73 @@
+from typing import List
+
+from typing_extensions import Self
+
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
+from datahub.metadata.com.linkedin.pegasus2avro.common import Siblings
+
+
+class HasSiblingsPatch(MetadataPatchProposal):
+    def add_sibling(self, sibling_urn: str, primary: bool = False) -> Self:
+        """Add a sibling relationship to the entity.
+
+        Args:
+            sibling_urn: The URN of the sibling entity to add.
+            primary: Whether this entity should be marked as primary in the relationship.
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            Siblings.ASPECT_NAME,
+            "add",
+            path=("siblings", sibling_urn),
+            value=sibling_urn,
+        )
+
+        # Set primary flag if specified
+        if primary:
+            self._add_patch(
+                Siblings.ASPECT_NAME,
+                "add",
+                path=("primary",),
+                value=primary,
+            )
+
+        return self
+
+    def remove_sibling(self, sibling_urn: str) -> Self:
+        """Remove a sibling relationship from the entity.
+
+        Args:
+            sibling_urn: The URN of the sibling entity to remove.
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            Siblings.ASPECT_NAME,
+            "remove",
+            path=("siblings", sibling_urn),
+            value={},
+        )
+        return self
+
+    def set_siblings(self, sibling_urns: List[str], primary: bool = False) -> Self:
+        """Set the complete list of siblings for the entity.
+
+        This will replace all existing siblings with the new list.
+
+        Args:
+            sibling_urns: The list of sibling URNs to set.
+            primary: Whether this entity should be marked as primary.
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            Siblings.ASPECT_NAME, "add", path=("siblings",), value=sibling_urns
+        )
+
+        self._add_patch(Siblings.ASPECT_NAME, "add", path=("primary",), value=primary)
+
+        return self

datahub/specific/dataset.py

@@ -22,6 +22,7 @@ from datahub.specific.aspect_helpers.fine_grained_lineage import (
     HasFineGrainedLineagePatch,
 )
 from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
+from datahub.specific.aspect_helpers.siblings import HasSiblingsPatch
 from datahub.specific.aspect_helpers.structured_properties import (
     HasStructuredPropertiesPatch,
 )
@@ -104,6 +105,7 @@ class DatasetPatchBuilder(
     HasTagsPatch,
     HasTermsPatch,
     HasFineGrainedLineagePatch,
+    HasSiblingsPatch,
     MetadataPatchProposal,
 ):
     def __init__(

datahub/sql_parsing/sql_parsing_aggregator.py

@@ -634,6 +634,9 @@ class SqlParsingAggregator(Closeable):
             TableSwap,
         ],
     ) -> None:
+        """
+        This assumes that queries come in order of increasing timestamps.
+        """
         if isinstance(item, KnownQueryLineageInfo):
             self.add_known_query_lineage(item)
         elif isinstance(item, KnownLineageMapping):

datahub/sql_parsing/tool_meta_extractor.py

@@ -208,9 +208,7 @@ class ToolMetaExtractor:
         Returns:
             bool: whether QueryLog entry is that of hex.
         """
-        last_line = _get_last_line(entry.query_text)
-
-        if not last_line.startswith("-- Hex query metadata:"):
+        if "-- Hex query metadata:" not in entry.query_text:
             return False
 
         entry.origin = HEX_PLATFORM_URN