acryl-datahub 1.1.0.5rc3__py3-none-any.whl → 1.1.0.5rc5__py3-none-any.whl

datahub/metadata/schemas/DataHubPageModuleKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubPageModuleKey",
+    "keyForEntity": "dataHubPageModule",
+    "entityCategory": "core",
+    "entityAspects": [
+      "dataHubPageModuleProperties"
+    ]
+  },
+  "name": "DataHubPageModuleKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "Unique id for the module."
+    }
+  ],
+  "doc": "Key for a DataHubPageModule"
+}

datahub/metadata/schemas/DataHubPageModuleProperties.avsc

@@ -0,0 +1,202 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubPageModuleProperties"
+  },
+  "name": "DataHubPageModuleProperties",
+  "namespace": "com.linkedin.pegasus2avro.module",
+  "fields": [
+    {
+      "type": "string",
+      "name": "name",
+      "doc": "The display name of this module"
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "ASSET_COLLECTION": "A module with a collection of assets",
+          "DOMAINS": "Module displaying the top domains",
+          "HIERARCHY": "A module displaying a hierarchy to navigate",
+          "LINK": "Link type module",
+          "OWNED_ASSETS": "Module displaying assets owned by a user",
+          "RICH_TEXT": "Module containing rich text to be rendered",
+          "SUBSCRIBED_ASSETS": "Module displaying assets subscribed to by a given user"
+        },
+        "name": "DataHubPageModuleType",
+        "namespace": "com.linkedin.pegasus2avro.module",
+        "symbols": [
+          "LINK",
+          "RICH_TEXT",
+          "ASSET_COLLECTION",
+          "HIERARCHY",
+          "OWNED_ASSETS",
+          "SUBSCRIBED_ASSETS",
+          "DOMAINS"
+        ],
+        "doc": "Enum containing the types of page modules that there are"
+      },
+      "name": "type",
+      "doc": "The type of this module - the purpose it serves"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubPageModuleVisibility",
+        "namespace": "com.linkedin.pegasus2avro.module",
+        "fields": [
+          {
+            "Searchable": {
+              "fieldType": "KEYWORD"
+            },
+            "type": {
+              "type": "enum",
+              "symbolDocs": {
+                "GLOBAL": "This module is discoverable and can be used by any user on the platform",
+                "PERSONAL": "This module is used for individual use only"
+              },
+              "name": "PageModuleScope",
+              "namespace": "com.linkedin.pegasus2avro.module",
+              "symbols": [
+                "PERSONAL",
+                "GLOBAL"
+              ]
+            },
+            "name": "scope",
+            "doc": "Audit stamp for when and by whom this module was created"
+          }
+        ],
+        "doc": "Info about the visibility of this module"
+      },
+      "name": "visibility",
+      "doc": "Info about the visibility of this module"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubPageModuleParams",
+        "namespace": "com.linkedin.pegasus2avro.module",
+        "fields": [
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "LinkModuleParams",
+                "namespace": "com.linkedin.pegasus2avro.module",
+                "fields": [
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": "string",
+                    "name": "linkUrn",
+                    "Urn": "Urn"
+                  }
+                ]
+              }
+            ],
+            "name": "linkParams",
+            "default": null,
+            "doc": "The params required if the module is type LINK"
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "RichTextModuleParams",
+                "namespace": "com.linkedin.pegasus2avro.module",
+                "fields": [
+                  {
+                    "type": "string",
+                    "name": "content"
+                  }
+                ]
+              }
+            ],
+            "name": "richTextParams",
+            "default": null,
+            "doc": "The params required if the module is type RICH_TEXT"
+          }
+        ],
+        "doc": "The specific parameters stored for a module"
+      },
+      "name": "params",
+      "doc": "The specific parameters stored for this module"
+    },
+    {
+      "Searchable": {
+        "/actor": {
+          "fieldName": "createdBy",
+          "fieldType": "URN"
+        },
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "AuditStamp",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": "long",
+            "name": "time",
+            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "actor",
+            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "impersonator",
+            "default": null,
+            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "message",
+            "default": null,
+            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+          }
+        ],
+        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+      },
+      "name": "created",
+      "doc": "Audit stamp for when and by whom this template was created"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": "com.linkedin.pegasus2avro.common.AuditStamp",
+      "name": "lastModified",
+      "doc": "Audit stamp for when and by whom this template was last updated"
+    }
+  ],
+  "doc": "The main properties of a DataHub page module"
+}

datahub/metadata/schemas/DataHubPageTemplateKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubPageTemplateKey",
+    "keyForEntity": "dataHubPageTemplate",
+    "entityCategory": "core",
+    "entityAspects": [
+      "dataHubPageTemplateProperties"
+    ]
+  },
+  "name": "DataHubPageTemplateKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "Unique id for the template."
+    }
+  ],
+  "doc": "Key for a DataHubPageTemplate"
+}

datahub/metadata/schemas/DataHubPageTemplateProperties.avsc

@@ -0,0 +1,175 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubPageTemplateProperties"
+  },
+  "name": "DataHubPageTemplateProperties",
+  "namespace": "com.linkedin.pegasus2avro.template",
+  "fields": [
+    {
+      "Relationship": {
+        "/*/modules/*": {
+          "entityTypes": [
+            "dataHubPageModule"
+          ],
+          "name": "ContainedIn"
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "DataHubPageTemplateRow",
+          "namespace": "com.linkedin.pegasus2avro.template",
+          "fields": [
+            {
+              "type": {
+                "type": "array",
+                "items": "string"
+              },
+              "name": "modules",
+              "doc": "The modules that exist in this template row",
+              "Urn": "Urn",
+              "urn_is_array": true
+            }
+          ],
+          "doc": "A row of modules contained in a template"
+        }
+      },
+      "name": "rows",
+      "doc": "The rows of modules contained in this template"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubPageTemplateSurface",
+        "namespace": "com.linkedin.pegasus2avro.template",
+        "fields": [
+          {
+            "Searchable": {
+              "fieldType": "KEYWORD"
+            },
+            "type": {
+              "type": "enum",
+              "symbolDocs": {
+                "HOME_PAGE": "This template applies to what to display on the home page for users."
+              },
+              "name": "PageTemplateSurfaceType",
+              "namespace": "com.linkedin.pegasus2avro.template",
+              "symbols": [
+                "HOME_PAGE"
+              ]
+            },
+            "name": "surfaceType",
+            "doc": "Where exactly is this template being used"
+          }
+        ],
+        "doc": "Info about the surface area of the product that this template is deployed in"
+      },
+      "name": "surface",
+      "doc": "Info about the surface area of the product that this template is deployed in"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubPageTemplateVisibility",
+        "namespace": "com.linkedin.pegasus2avro.template",
+        "fields": [
+          {
+            "Searchable": {
+              "fieldType": "KEYWORD"
+            },
+            "type": {
+              "type": "enum",
+              "symbolDocs": {
+                "GLOBAL": "This template is used across users",
+                "PERSONAL": "This template is used for individual use only"
+              },
+              "name": "PageTemplateScope",
+              "namespace": "com.linkedin.pegasus2avro.template",
+              "symbols": [
+                "PERSONAL",
+                "GLOBAL"
+              ]
+            },
+            "name": "scope",
+            "doc": "The scope of this template and who can use/see it"
+          }
+        ],
+        "doc": "Info about the visibility of this template"
+      },
+      "name": "visibility",
+      "doc": "Info about the visibility of this template"
+    },
+    {
+      "Searchable": {
+        "/actor": {
+          "fieldName": "createdBy",
+          "fieldType": "URN"
+        },
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "AuditStamp",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": "long",
+            "name": "time",
+            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "actor",
+            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "impersonator",
+            "default": null,
+            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "message",
+            "default": null,
+            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+          }
+        ],
+        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+      },
+      "name": "created",
+      "doc": "Audit stamp for when and by whom this template was created"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": "com.linkedin.pegasus2avro.common.AuditStamp",
+      "name": "lastModified",
+      "doc": "Audit stamp for when and by whom this template was last updated"
+    }
+  ],
+  "doc": "The main properties of a DataHub page template"
+}

datahub/metadata/schemas/GlobalSettingsInfo.avsc

@@ -282,6 +282,31 @@
         "columnPropagationEnabled": true
       },
       "doc": "Settings related to the documentation propagation feature"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "GlobalHomePageSettings",
+          "namespace": "com.linkedin.pegasus2avro.settings.global",
+          "fields": [
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "defaultTemplate",
+              "doc": "The urn that will be rendered in the UI by default for all users",
+              "Urn": "Urn"
+            }
+          ],
+          "doc": "Global settings related to the home page for an instance"
+        }
+      ],
+      "name": "homePage",
+      "default": null,
+      "doc": "Global settings related to the home page for an instance"
     }
   ],
   "doc": "DataHub Global platform settings. Careful - these should not be modified by the outside world!"

datahub/sdk/datajob.py

@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Type
 
 from typing_extensions import Self
 
+import datahub.emitter.mce_builder as builder
 import datahub.metadata.schema_classes as models
 from datahub.cli.cli_utils import first_non_null
 from datahub.errors import IngestionAttributionWarning
@@ -64,7 +65,7 @@ class DataJob(
         """Get the URN type for data jobs."""
         return DataJobUrn
 
-    def __init__(
+    def __init__(  # noqa: C901
         self,
         *,
         name: str,
@@ -86,6 +87,7 @@ class DataJob(
         domain: Optional[DomainInputType] = None,
         inlets: Optional[List[DatasetUrnOrStr]] = None,
         outlets: Optional[List[DatasetUrnOrStr]] = None,
+        fine_grained_lineages: Optional[List[models.FineGrainedLineageClass]] = None,
         structured_properties: Optional[StructuredPropertyInputType] = None,
         extra_aspects: ExtraAspectsType = None,
     ):
@@ -103,12 +105,14 @@ class DataJob(
             ValueError: If neither flow nor (flow_urn and platform_instance) are provided
         """
         if flow is None:
-            if flow_urn is None or platform_instance is None:
+            if flow_urn is None:
                 raise ValueError(
                     "You must provide either: 1. a DataFlow object, or 2. a DataFlowUrn (and a platform_instance config if required)"
                 )
             flow_urn = DataFlowUrn.from_string(flow_urn)
-            if flow_urn.flow_id.startswith(f"{platform_instance}."):
+            if platform_instance and flow_urn.flow_id.startswith(
+                f"{platform_instance}."
+            ):
                 flow_name = flow_urn.flow_id[len(platform_instance) + 1 :]
             else:
                 flow_name = flow_urn.flow_id
@@ -133,8 +137,6 @@ class DataJob(
         )
         self._setdefault_aspect(job_info)
         self._ensure_datajob_props().flowUrn = str(flow.urn)
-
-        # Set properties if provided
         if description is not None:
             self.set_description(description)
         if external_url is not None:
@@ -145,8 +147,6 @@ class DataJob(
             self.set_created(created)
         if last_modified is not None:
             self.set_last_modified(last_modified)
-
-        # Set standard aspects
         if subtype is not None:
             self.set_subtype(subtype)
         if owners is not None:
@@ -159,13 +159,19 @@ class DataJob(
             self.set_terms(terms)
         if domain is not None:
             self.set_domain(domain)
+        if structured_properties is not None:
+            for key, value in structured_properties.items():
+                self.set_structured_property(property_urn=key, values=value)
         if inlets is not None:
             self.set_inlets(inlets)
         if outlets is not None:
             self.set_outlets(outlets)
-        if structured_properties is not None:
-            for key, value in structured_properties.items():
-                self.set_structured_property(property_urn=key, values=value)
+        if fine_grained_lineages is not None:
+            self.set_fine_grained_lineages(fine_grained_lineages)
+
+        if self.flow_urn.cluster.upper() in builder.ALL_ENV_TYPES:
+            env = self.flow_urn.cluster.upper()
+        self._ensure_datajob_props().env = env
 
     @classmethod
     def _new_from_graph(cls, urn: Urn, current_aspects: models.AspectBag) -> Self:
@@ -201,9 +207,7 @@ class DataJob(
     ) -> Optional[models.DataJobInputOutputClass]:
         return self._get_aspect(models.DataJobInputOutputClass)
 
-    def _ensure_datajob_inputoutput_props(
-        self,
-    ) -> models.DataJobInputOutputClass:
+    def _ensure_datajob_inputoutput_props(self) -> models.DataJobInputOutputClass:
         return self._setdefault_aspect(
             models.DataJobInputOutputClass(inputDatasets=[], outputDatasets=[])
         )
@@ -307,8 +311,6 @@ class DataJob(
                 browse_path.append(
                     models.BrowsePathEntryClass(id=entry.id, urn=entry.urn)
                 )
-
-        # Add the job itself to the path
         browse_path.append(models.BrowsePathEntryClass(id=flow.name, urn=str(flow.urn)))
         # Set the browse path aspect
         self._set_aspect(models.BrowsePathsV2Class(path=browse_path))
@@ -341,3 +343,25 @@ class DataJob(
             self._ensure_datajob_inputoutput_props().outputDatasets.append(
                 str(outlet_urn)
             )
+
+    @property
+    def fine_grained_lineages(self) -> List[models.FineGrainedLineageClass]:
+        io_aspect = self._get_datajob_inputoutput_props()
+        return (
+            io_aspect.fineGrainedLineages
+            if io_aspect and io_aspect.fineGrainedLineages
+            else []
+        )
+
+    def set_fine_grained_lineages(
+        self, lineages: List[models.FineGrainedLineageClass]
+    ) -> None:
+        io_aspect = self._ensure_datajob_inputoutput_props()
+        if io_aspect.fineGrainedLineages is None:
+            io_aspect.fineGrainedLineages = []
+        io_aspect.fineGrainedLineages.extend(lineages)
+
+    @property
+    def env(self) -> Optional[str]:
+        """Get the environment of the data job."""
+        return str(self._ensure_datajob_props().env)

datahub/specific/dataproduct.py

@@ -9,6 +9,9 @@ from datahub.metadata.schema_classes import (
 )
 from datahub.specific.aspect_helpers.custom_properties import HasCustomPropertiesPatch
 from datahub.specific.aspect_helpers.ownership import HasOwnershipPatch
+from datahub.specific.aspect_helpers.structured_properties import (
+    HasStructuredPropertiesPatch,
+)
 from datahub.specific.aspect_helpers.tags import HasTagsPatch
 from datahub.specific.aspect_helpers.terms import HasTermsPatch
 
@@ -16,6 +19,7 @@ from datahub.specific.aspect_helpers.terms import HasTermsPatch
 class DataProductPatchBuilder(
     HasOwnershipPatch,
     HasCustomPropertiesPatch,
+    HasStructuredPropertiesPatch,
     HasTagsPatch,
     HasTermsPatch,
     MetadataPatchProposal,