acryl-datahub-cloud 0.3.9rc6__py3-none-any.whl → 0.3.9rc7__py3-none-any.whl

acryl_datahub_cloud/metadata/schema.avsc

@@ -22196,7 +22196,6 @@
               "dataset",
               "mlModel"
             ],
-            "isLineage": true,
             "name": "Consumes"
           }
         },
@@ -22217,6 +22216,33 @@
         },
         "name": "inputs",
         "doc": "Input assets consumed"
+      },
+      {
+        "Relationship": {
+          "/*/destinationUrn": {
+            "createdActor": "inputEdges/*/created/actor",
+            "createdOn": "inputEdges/*/created/time",
+            "entityTypes": [
+              "dataset",
+              "mlModel"
+            ],
+            "isLineage": true,
+            "name": "DataProcessInstanceConsumes",
+            "properties": "inputEdges/*/properties",
+            "updatedActor": "inputEdges/*/lastModified/actor",
+            "updatedOn": "inputEdges/*/lastModified/time"
+          }
+        },
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.common.Edge"
+          }
+        ],
+        "name": "inputEdges",
+        "default": null,
+        "doc": "Input assets consumed by the data process instance, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the inputs field."
       }
     ],
     "doc": "Information about the inputs datasets of a Data process"
@@ -22567,8 +22593,6 @@
               "dataset",
               "mlModel"
             ],
-            "isLineage": true,
-            "isUpstream": false,
             "name": "Produces"
           }
         },
@@ -22588,7 +22612,35 @@
           "items": "string"
         },
         "name": "outputs",
-        "doc": "Output datasets to be produced"
+        "doc": "Output assets produced"
+      },
+      {
+        "Relationship": {
+          "/*/destinationUrn": {
+            "createdActor": "outputEdges/*/created/actor",
+            "createdOn": "outputEdges/*/created/time",
+            "entityTypes": [
+              "dataset",
+              "mlModel"
+            ],
+            "isLineage": true,
+            "isUpstream": false,
+            "name": "DataProcessInstanceProduces",
+            "properties": "outputEdges/*/properties",
+            "updatedActor": "outputEdges/*/lastModified/actor",
+            "updatedOn": "outputEdges/*/lastModified/time"
+          }
+        },
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.common.Edge"
+          }
+        ],
+        "name": "outputEdges",
+        "default": null,
+        "doc": "Output assets produced by the data process instance during processing, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the outputs field."
       }
     ],
     "doc": "Information about the outputs of a Data process"

acryl_datahub_cloud/metadata/schema_classes.py

@@ -13563,13 +13563,16 @@ class DataProcessInstanceInputClass(_Aspect):
 
     def __init__(self,
         inputs: List[str],
+        inputEdges: Union[None, List["EdgeClass"]]=None,
     ):
         super().__init__()
         
         self.inputs = inputs
+        self.inputEdges = inputEdges
     
     def _restore_defaults(self) -> None:
         self.inputs = list()
+        self.inputEdges = self.RECORD_SCHEMA.fields_dict["inputEdges"].default
     
     
     @property
@@ -13582,6 +13585,18 @@ class DataProcessInstanceInputClass(_Aspect):
         self._inner_dict['inputs'] = value
     
     
+    @property
+    def inputEdges(self) -> Union[None, List["EdgeClass"]]:
+        """Input assets consumed by the data process instance, with additional metadata.
+    Counts as lineage.
+    Will eventually deprecate the inputs field."""
+        return self._inner_dict.get('inputEdges')  # type: ignore
+    
+    @inputEdges.setter
+    def inputEdges(self, value: Union[None, List["EdgeClass"]]) -> None:
+        self._inner_dict['inputEdges'] = value
+    
+    
 class DataProcessInstanceOutputClass(_Aspect):
     """Information about the outputs of a Data process"""
 
@@ -13592,18 +13607,21 @@ class DataProcessInstanceOutputClass(_Aspect):
 
     def __init__(self,
         outputs: List[str],
+        outputEdges: Union[None, List["EdgeClass"]]=None,
     ):
         super().__init__()
         
         self.outputs = outputs
+        self.outputEdges = outputEdges
     
     def _restore_defaults(self) -> None:
         self.outputs = list()
+        self.outputEdges = self.RECORD_SCHEMA.fields_dict["outputEdges"].default
     
     
     @property
     def outputs(self) -> List[str]:
-        """Output datasets to be produced"""
+        """Output assets produced"""
         return self._inner_dict.get('outputs')  # type: ignore
     
     @outputs.setter
@@ -13611,6 +13629,18 @@ class DataProcessInstanceOutputClass(_Aspect):
         self._inner_dict['outputs'] = value
     
     
+    @property
+    def outputEdges(self) -> Union[None, List["EdgeClass"]]:
+        """Output assets produced by the data process instance during processing, with additional metadata.
+    Counts as lineage.
+    Will eventually deprecate the outputs field."""
+        return self._inner_dict.get('outputEdges')  # type: ignore
+    
+    @outputEdges.setter
+    def outputEdges(self, value: Union[None, List["EdgeClass"]]) -> None:
+        self._inner_dict['outputEdges'] = value
+    
+    
 class DataProcessInstancePropertiesClass(_Aspect):
     """The inputs and outputs of this data process"""
 
@@ -37834,6 +37864,7 @@ ASPECT_NAME_MAP: Dict[str, Type[_Aspect]] = {
     for aspect in ASPECT_CLASSES
 }
 
+from typing import Literal
 from typing_extensions import TypedDict
 
 class AspectBag(TypedDict, total=False):
@@ -38176,4 +38207,145 @@ KEY_ASPECTS: Dict[str, Type[_Aspect]] = {
     'telemetry': TelemetryKeyClass
 }
 
+ENTITY_TYPE_NAMES: List[str] = [
+    'dataProduct',
+    'structuredProperty',
+    'entityType',
+    'platformResource',
+    'businessAttribute',
+    'dataType',
+    'inviteToken',
+    'assertion',
+    'schemaField',
+    'corpGroup',
+    'role',
+    'dataProcessInstance',
+    'dataHubSecret',
+    'dataHubPersona',
+    'mlFeatureTable',
+    'glossaryTerm',
+    'recommendationModule',
+    'post',
+    'mlPrimaryKey',
+    'dataPlatformInstance',
+    'dataHubMetricCube',
+    'query',
+    'dataset',
+    'dataHubExecutionRequest',
+    'chart',
+    'mlModelGroup',
+    'subscription',
+    'anomaly',
+    'dataHubRemoteExecutorPool',
+    'dataJob',
+    'domain',
+    'erModelRelationship',
+    'dataHubRole',
+    'form',
+    'dataHubAccessToken',
+    'dataHubAction',
+    'dataPlatform',
+    'container',
+    'tag',
+    'dataHubPolicy',
+    'constraint',
+    'glossaryNode',
+    'dataProcess',
+    'dataHubStepState',
+    'notebook',
+    'mlFeature',
+    'dataHubUpgrade',
+    'dashboard',
+    'mlModel',
+    'incident',
+    'globalSettings',
+    'dataHubView',
+    'dataHubRetention',
+    'ownershipType',
+    'actionRequest',
+    'dataHubRemoteExecutorGlobalConfig',
+    'dataFlow',
+    'dataContract',
+    'dataHubConnection',
+    'dataHubRemoteExecutor',
+    'monitor',
+    'corpuser',
+    'versionSet',
+    'test',
+    'dataHubIngestionSource',
+    'mlModelDeployment',
+    'linkPreview',
+    'telemetry'
+]
+EntityTypeName = Literal[
+    'dataProduct',
+    'structuredProperty',
+    'entityType',
+    'platformResource',
+    'businessAttribute',
+    'dataType',
+    'inviteToken',
+    'assertion',
+    'schemaField',
+    'corpGroup',
+    'role',
+    'dataProcessInstance',
+    'dataHubSecret',
+    'dataHubPersona',
+    'mlFeatureTable',
+    'glossaryTerm',
+    'recommendationModule',
+    'post',
+    'mlPrimaryKey',
+    'dataPlatformInstance',
+    'dataHubMetricCube',
+    'query',
+    'dataset',
+    'dataHubExecutionRequest',
+    'chart',
+    'mlModelGroup',
+    'subscription',
+    'anomaly',
+    'dataHubRemoteExecutorPool',
+    'dataJob',
+    'domain',
+    'erModelRelationship',
+    'dataHubRole',
+    'form',
+    'dataHubAccessToken',
+    'dataHubAction',
+    'dataPlatform',
+    'container',
+    'tag',
+    'dataHubPolicy',
+    'constraint',
+    'glossaryNode',
+    'dataProcess',
+    'dataHubStepState',
+    'notebook',
+    'mlFeature',
+    'dataHubUpgrade',
+    'dashboard',
+    'mlModel',
+    'incident',
+    'globalSettings',
+    'dataHubView',
+    'dataHubRetention',
+    'ownershipType',
+    'actionRequest',
+    'dataHubRemoteExecutorGlobalConfig',
+    'dataFlow',
+    'dataContract',
+    'dataHubConnection',
+    'dataHubRemoteExecutor',
+    'monitor',
+    'corpuser',
+    'versionSet',
+    'test',
+    'dataHubIngestionSource',
+    'mlModelDeployment',
+    'linkPreview',
+    'telemetry'
+]
+
 # fmt: on

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceInput.avsc

@@ -13,7 +13,6 @@
             "dataset",
             "mlModel"
           ],
-          "isLineage": true,
           "name": "Consumes"
         }
       },
@@ -34,6 +33,135 @@
       "doc": "Input assets consumed",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "inputEdges/*/created/actor",
+          "createdOn": "inputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "name": "DataProcessInstanceConsumes",
+          "properties": "inputEdges/*/properties",
+          "updatedActor": "inputEdges/*/lastModified/actor",
+          "updatedOn": "inputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "inputEdges",
+      "default": null,
+      "doc": "Input assets consumed by the data process instance, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the inputs field."
     }
   ],
   "doc": "Information about the inputs datasets of a Data process"

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -13,8 +13,6 @@
             "dataset",
             "mlModel"
           ],
-          "isLineage": true,
-          "isUpstream": false,
           "name": "Produces"
         }
       },
@@ -32,9 +30,139 @@
         "items": "string"
       },
       "name": "outputs",
-      "doc": "Output datasets to be produced",
+      "doc": "Output assets produced",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "outputEdges/*/created/actor",
+          "createdOn": "outputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "isUpstream": false,
+          "name": "DataProcessInstanceProduces",
+          "properties": "outputEdges/*/properties",
+          "updatedActor": "outputEdges/*/lastModified/actor",
+          "updatedOn": "outputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "outputEdges",
+      "default": null,
+      "doc": "Output assets produced by the data process instance during processing, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the outputs field."
     }
   ],
   "doc": "Information about the outputs of a Data process"