acryl-datahub 0.15.0.1rc17__py3-none-any.whl → 0.15.0.2__py3-none-any.whl

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -5109,6 +5109,51 @@
                         "default": null,
                         "doc": "URL where the reference exist"
                       },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob",
+                              "dataProcessInstance"
+                            ],
+                            "isLineage": true,
+                            "name": "TrainedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "trainingJobs",
+                        "default": null,
+                        "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                      },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob",
+                              "dataProcessInstance"
+                            ],
+                            "isLineage": true,
+                            "isUpstream": false,
+                            "name": "UsedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "downstreamJobs",
+                        "default": null,
+                        "doc": "List of jobs or process instances (if any) that use the model or group."
+                      },
                       {
                         "Searchable": {
                           "boostScore": 10.0,
@@ -5180,6 +5225,14 @@
                                 ],
                                 "name": "versionTag",
                                 "default": null
+                              },
+                              {
+                                "type": [
+                                  "null",
+                                  "com.linkedin.pegasus2avro.common.MetadataAttribution"
+                                ],
+                                "name": "metadataAttribution",
+                                "default": null
                               }
                             ],
                             "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
@@ -5393,54 +5446,6 @@
                         "Urn": "Urn",
                         "urn_is_array": true
                       },
-                      {
-                        "Relationship": {
-                          "/*": {
-                            "entityTypes": [
-                              "dataJob",
-                              "dataProcessInstance"
-                            ],
-                            "isLineage": true,
-                            "name": "TrainedBy"
-                          }
-                        },
-                        "type": [
-                          "null",
-                          {
-                            "type": "array",
-                            "items": "string"
-                          }
-                        ],
-                        "name": "trainingJobs",
-                        "default": null,
-                        "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.",
-                        "Urn": "Urn",
-                        "urn_is_array": true
-                      },
-                      {
-                        "Relationship": {
-                          "/*": {
-                            "entityTypes": [
-                              "dataJob"
-                            ],
-                            "isLineage": true,
-                            "isUpstream": false,
-                            "name": "UsedBy"
-                          }
-                        },
-                        "type": [
-                          "null",
-                          {
-                            "type": "array",
-                            "items": "string"
-                          }
-                        ],
-                        "name": "downstreamJobs",
-                        "default": null,
-                        "doc": "List of jobs (if any) that use the model",
-                        "Urn": "Urn",
-                        "urn_is_array": true
-                      },
                       {
                         "Relationship": {
                           "/*": {
@@ -6839,6 +6844,51 @@
                         "default": {},
                         "doc": "Custom property bag."
                       },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob",
+                              "dataProcessInstance"
+                            ],
+                            "isLineage": true,
+                            "name": "TrainedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "trainingJobs",
+                        "default": null,
+                        "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                      },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob",
+                              "dataProcessInstance"
+                            ],
+                            "isLineage": true,
+                            "isUpstream": false,
+                            "name": "UsedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "downstreamJobs",
+                        "default": null,
+                        "doc": "List of jobs or process instances (if any) that use the model or group."
+                      },
                       {
                         "Searchable": {
                           "boostScore": 10.0,
@@ -6895,29 +6945,6 @@
                         "default": null,
                         "doc": "Date when the MLModelGroup was last modified"
                       },
-                      {
-                        "Relationship": {
-                          "/*": {
-                            "entityTypes": [
-                              "dataJob"
-                            ],
-                            "isLineage": true,
-                            "name": "TrainedBy"
-                          }
-                        },
-                        "type": [
-                          "null",
-                          {
-                            "type": "array",
-                            "items": "string"
-                          }
-                        ],
-                        "name": "trainingJobs",
-                        "default": null,
-                        "doc": "List of jobs (if any) used to train the model group. Visible in Lineage.",
-                        "Urn": "Urn",
-                        "urn_is_array": true
-                      },
                       {
                         "type": [
                           "null",

datahub/metadata/schemas/VersionProperties.avsc

@@ -0,0 +1,216 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionProperties"
+  },
+  "name": "VersionProperties",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "Relationship": {
+        "entityTypes": [
+          "versionSet"
+        ],
+        "name": "VersionOf"
+      },
+      "Searchable": {
+        "queryByDefault": false
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "versionSet",
+      "doc": "The linked Version Set entity that ties multiple versioned assets together",
+      "Urn": "Urn",
+      "entityTypes": [
+        "versionSet"
+      ]
+    },
+    {
+      "Searchable": {
+        "/versionTag": {
+          "fieldName": "version",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "VersionTag",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "versionTag",
+            "default": null
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "MetadataAttribution",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "fields": [
+                  {
+                    "type": "long",
+                    "name": "time",
+                    "doc": "When this metadata was updated."
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": "string",
+                    "name": "actor",
+                    "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "source",
+                    "default": null,
+                    "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "type": {
+                      "type": "map",
+                      "values": "string"
+                    },
+                    "name": "sourceDetail",
+                    "default": {},
+                    "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                  }
+                ],
+                "doc": "Information about who, why, and how this metadata was applied"
+              }
+            ],
+            "name": "metadataAttribution",
+            "default": null
+          }
+        ],
+        "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
+      },
+      "name": "version",
+      "doc": "Label for this versioned asset, is unique within a version set"
+    },
+    {
+      "Searchable": {
+        "/*/versionTag": {
+          "fieldName": "aliases",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "com.linkedin.pegasus2avro.common.VersionTag"
+      },
+      "name": "aliases",
+      "default": [],
+      "doc": "Associated aliases for this versioned asset"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "comment",
+      "default": null,
+      "doc": "Comment documenting what this version was created for, changes, or represents"
+    },
+    {
+      "Searchable": {
+        "fieldName": "versionSortId",
+        "queryByDefault": false
+      },
+      "type": "string",
+      "name": "sortId",
+      "doc": "Sort identifier that determines where a version lives in the order of the Version Set.\nWhat this looks like depends on the Version Scheme. For sort ids generated by DataHub we use an 8 character string representation."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "AuditStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "actor",
+              "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+              "Urn": "Urn"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "impersonator",
+              "default": null,
+              "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+              "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+            }
+          ],
+          "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+        }
+      ],
+      "name": "sourceCreatedTimestamp",
+      "default": null,
+      "doc": "Timestamp reflecting when this asset version was created in the source system."
+    },
+    {
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.AuditStamp"
+      ],
+      "name": "metadataCreatedTimestamp",
+      "default": null,
+      "doc": "Timestamp reflecting when the metadata for this version was created in DataHub"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN",
+        "queryByDefault": false
+      },
+      "type": [
+        "null",
+        "boolean"
+      ],
+      "name": "isLatest",
+      "default": null,
+      "doc": "Marks whether this version is currently the latest. Set by a side effect and should not be modified by API."
+    }
+  ],
+  "doc": "Properties about a versioned asset i.e. dataset, ML Model, etc."
+}

datahub/metadata/schemas/VersionSetKey.avsc

@@ -0,0 +1,26 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionSetKey",
+    "keyForEntity": "versionSet",
+    "entityCategory": "core",
+    "entityAspects": [
+      "versionSetProperties"
+    ]
+  },
+  "name": "VersionSetKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "ID of the Version Set, generated from platform + asset id / name"
+    },
+    {
+      "type": "string",
+      "name": "entityType",
+      "doc": "Type of entities included in version set, limits to a single entity type between linked versioned entities"
+    }
+  ],
+  "doc": "Key for a Version Set entity"
+}

datahub/metadata/schemas/VersionSetProperties.avsc

@@ -0,0 +1,49 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionSetProperties"
+  },
+  "name": "VersionSetProperties",
+  "namespace": "com.linkedin.pegasus2avro.versionset",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "queryByDefault": "false"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "latest",
+      "doc": "The latest versioned entity linked to in this version set",
+      "Urn": "Urn"
+    },
+    {
+      "type": {
+        "type": "enum",
+        "name": "VersioningScheme",
+        "namespace": "com.linkedin.pegasus2avro.versionset",
+        "symbols": [
+          "ALPHANUMERIC_GENERATED_BY_DATAHUB"
+        ]
+      },
+      "name": "versioningScheme",
+      "doc": "What versioning scheme is being utilized for the versioned entities sort criterion. Static once set"
+    }
+  ]
+}

datahub/secret/datahub_secrets_client.py

@@ -11,34 +11,25 @@ class DataHubSecretsClient:
     def __init__(self, graph: DataHubGraph):
         self.graph = graph
 
+    def _cleanup_secret_name(self, secret_names: List[str]) -> List[str]:
+        """Remove empty strings from the list of secret names."""
+        return [secret_name for secret_name in secret_names if secret_name]
+
     def get_secret_values(self, secret_names: List[str]) -> Dict[str, Optional[str]]:
         if len(secret_names) == 0:
             return {}
 
-        request_json = {
-            "query": """query getSecretValues($input: GetSecretValuesInput!) {\n
-                getSecretValues(input: $input) {\n
-                    name\n
-                    value\n
-                }\n
+        res_data = self.graph.execute_graphql(
+            query="""query getSecretValues($input: GetSecretValuesInput!) {
+                getSecretValues(input: $input) {
+                    name
+                    value
+                }
             }""",
-            "variables": {"input": {"secrets": secret_names}},
-        }
-        # TODO: Use graph.execute_graphql() instead.
-
-        # Fetch secrets using GraphQL API f
-        response = self.graph._session.post(
-            f"{self.graph.config.server}/api/graphql", json=request_json
+            variables={"input": {"secrets": self._cleanup_secret_name(secret_names)}},
         )
-        response.raise_for_status()
-
-        # Verify response
-        res_data = response.json()
-        if "errors" in res_data:
-            raise Exception("Failed to retrieve secrets from DataHub.")
-
         # Convert list of name, value secret pairs into a dict and return
-        secret_value_list = res_data["data"]["getSecretValues"]
+        secret_value_list = res_data["getSecretValues"]
         secret_value_dict = dict()
         for secret_value in secret_value_list:
             secret_value_dict[secret_value["name"]] = secret_value["value"]

datahub/secret/secret_common.py

@@ -2,10 +2,7 @@ import json
 import logging
 from typing import List
 
-from datahub.configuration.config_loader import (
-    list_referenced_env_variables,
-    resolve_env_variables,
-)
+from datahub.configuration.config_loader import EnvResolver
 from datahub.secret.secret_store import SecretStore
 
 logger = logging.getLogger(__name__)
@@ -42,18 +39,27 @@ def resolve_secrets(secret_names: List[str], secret_stores: List[SecretStore]) -
     return final_secret_values
 
 
-def resolve_recipe(recipe: str, secret_stores: List[SecretStore]) -> dict:
+def resolve_recipe(
+    recipe: str, secret_stores: List[SecretStore], strict_env_syntax: bool = True
+) -> dict:
+    # Note: the default for `strict_env_syntax` is normally False, but here we override
+    # it to be true. Particularly when fetching secrets from external secret stores, we
+    # want to be more careful about not over-fetching secrets.
+
     json_recipe_raw = json.loads(recipe)
 
     # 1. Extract all secrets needing resolved.
-    secrets_to_resolve = list_referenced_env_variables(json_recipe_raw)
+    secrets_to_resolve = EnvResolver.list_referenced_variables(
+        json_recipe_raw, strict_env_syntax=strict_env_syntax
+    )
 
     # 2. Resolve secret values
     secret_values_dict = resolve_secrets(list(secrets_to_resolve), secret_stores)
 
     # 3. Substitute secrets into recipe file
-    json_recipe_resolved = resolve_env_variables(
-        json_recipe_raw, environ=secret_values_dict
+    resolver = EnvResolver(
+        environ=secret_values_dict, strict_env_syntax=strict_env_syntax
     )
+    json_recipe_resolved = resolver.resolve(json_recipe_raw)
 
     return json_recipe_resolved

datahub/specific/aspect_helpers/custom_properties.py

@@ -9,8 +9,7 @@ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
 class HasCustomPropertiesPatch(MetadataPatchProposal):
     @classmethod
     @abstractmethod
-    def _custom_properties_location(self) -> Tuple[str, PatchPath]:
-        ...
+    def _custom_properties_location(self) -> Tuple[str, PatchPath]: ...
 
     def add_custom_property(self, key: str, value: str) -> Self:
         """Add a custom property to the entity.

datahub/sql_parsing/schema_resolver.py

@@ -33,14 +33,11 @@ class GraphQLSchemaMetadata(TypedDict):
 
 class SchemaResolverInterface(Protocol):
     @property
-    def platform(self) -> str:
-        ...
+    def platform(self) -> str: ...
 
-    def includes_temp_tables(self) -> bool:
-        ...
+    def includes_temp_tables(self) -> bool: ...
 
-    def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]:
-        ...
+    def resolve_table(self, table: _TableName) -> Tuple[str, Optional[SchemaInfo]]: ...
 
     def __hash__(self) -> int:
         # Mainly to make lru_cache happy in methods that accept a schema resolver.
@@ -232,8 +229,7 @@ class SchemaResolver(Closeable, SchemaResolverInterface):
         return {
             get_simple_field_path_from_v2_field_path(field["fieldPath"]): (
                 # The actual types are more of a "nice to have".
-                field["nativeDataType"]
-                or "str"
+                field["nativeDataType"] or "str"
             )
             for field in schema["fields"]
             # TODO: We can't generate lineage to columns nested within structs yet.
@@ -289,8 +285,7 @@ def _convert_schema_field_list_to_info(
     return {
         get_simple_field_path_from_v2_field_path(col.fieldPath): (
             # The actual types are more of a "nice to have".
-            col.nativeDataType
-            or "str"
+            col.nativeDataType or "str"
         )
         for col in schema_fields
         # TODO: We can't generate lineage to columns nested within structs yet.