acryl-datahub 0.15.0.1rc17__py3-none-any.whl → 0.15.0.2__py3-none-any.whl

datahub/metadata/schemas/DataFlowKey.avsc

@@ -17,6 +17,7 @@
       "glossaryTerms",
       "institutionalMemory",
       "dataPlatformInstance",
+      "container",
       "browsePathsV2",
       "structuredProperties",
       "incidentsSummary",

datahub/metadata/schemas/DataJobKey.avsc

@@ -20,6 +20,7 @@
       "glossaryTerms",
       "institutionalMemory",
       "dataPlatformInstance",
+      "container",
       "browsePathsV2",
       "structuredProperties",
       "forms",

datahub/metadata/schemas/DataProcessInstanceInput.avsc

@@ -10,8 +10,10 @@
       "Relationship": {
         "/*": {
           "entityTypes": [
-            "dataset"
+            "dataset",
+            "mlModel"
           ],
+          "isLineage": true,
           "name": "Consumes"
         }
       },
@@ -29,7 +31,7 @@
         "items": "string"
       },
       "name": "inputs",
-      "doc": "Input datasets to be consumed",
+      "doc": "Input assets consumed",
       "Urn": "Urn",
       "urn_is_array": true
     }

datahub/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -13,6 +13,8 @@
             "dataset",
             "mlModel"
           ],
+          "isLineage": true,
+          "isUpstream": false,
           "name": "Produces"
         }
       },

datahub/metadata/schemas/DatasetKey.avsc

@@ -35,7 +35,8 @@
       "access",
       "structuredProperties",
       "forms",
-      "partitionsSummary"
+      "partitionsSummary",
+      "versionProperties"
     ],
     "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
   },

datahub/metadata/schemas/MLFeatureProperties.avsc

@@ -101,6 +101,57 @@
               ],
               "name": "versionTag",
               "default": null
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "metadataAttribution",
+              "default": null
             }
           ],
           "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"

datahub/metadata/schemas/MLModelDeploymentProperties.avsc

@@ -74,6 +74,57 @@
               ],
               "name": "versionTag",
               "default": null
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "metadataAttribution",
+              "default": null
             }
           ],
           "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"

datahub/metadata/schemas/MLModelGroupProperties.avsc

@@ -21,6 +21,51 @@
       "default": {},
       "doc": "Custom property bag."
     },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataJob",
+            "dataProcessInstance"
+          ],
+          "isLineage": true,
+          "name": "TrainedBy"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "trainingJobs",
+      "default": null,
+      "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+    },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataJob",
+            "dataProcessInstance"
+          ],
+          "isLineage": true,
+          "isUpstream": false,
+          "name": "UsedBy"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "downstreamJobs",
+      "default": null,
+      "doc": "List of jobs or process instances (if any) that use the model or group."
+    },
     {
       "Searchable": {
         "boostScore": 10.0,
@@ -102,29 +147,6 @@
       "default": null,
       "doc": "Date when the MLModelGroup was last modified"
     },
-    {
-      "Relationship": {
-        "/*": {
-          "entityTypes": [
-            "dataJob"
-          ],
-          "isLineage": true,
-          "name": "TrainedBy"
-        }
-      },
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": "string"
-        }
-      ],
-      "name": "trainingJobs",
-      "default": null,
-      "doc": "List of jobs (if any) used to train the model group. Visible in Lineage.",
-      "Urn": "Urn",
-      "urn_is_array": true
-    },
     {
       "type": [
         "null",
@@ -140,6 +162,57 @@
               ],
               "name": "versionTag",
               "default": null
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "metadataAttribution",
+              "default": null
             }
           ],
           "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"

datahub/metadata/schemas/MLModelKey.avsc

@@ -29,7 +29,8 @@
       "browsePathsV2",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "versionProperties"
     ]
   },
   "name": "MLModelKey",

datahub/metadata/schemas/MLModelProperties.avsc

@@ -37,6 +37,51 @@
       "default": null,
       "doc": "URL where the reference exist"
     },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataJob",
+            "dataProcessInstance"
+          ],
+          "isLineage": true,
+          "name": "TrainedBy"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "trainingJobs",
+      "default": null,
+      "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+    },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataJob",
+            "dataProcessInstance"
+          ],
+          "isLineage": true,
+          "isUpstream": false,
+          "name": "UsedBy"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "downstreamJobs",
+      "default": null,
+      "doc": "List of jobs or process instances (if any) that use the model or group."
+    },
     {
       "Searchable": {
         "boostScore": 10.0,
@@ -133,6 +178,57 @@
               ],
               "name": "versionTag",
               "default": null
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "metadataAttribution",
+              "default": null
             }
           ],
           "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
@@ -346,54 +442,6 @@
       "Urn": "Urn",
       "urn_is_array": true
     },
-    {
-      "Relationship": {
-        "/*": {
-          "entityTypes": [
-            "dataJob",
-            "dataProcessInstance"
-          ],
-          "isLineage": true,
-          "name": "TrainedBy"
-        }
-      },
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": "string"
-        }
-      ],
-      "name": "trainingJobs",
-      "default": null,
-      "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.",
-      "Urn": "Urn",
-      "urn_is_array": true
-    },
-    {
-      "Relationship": {
-        "/*": {
-          "entityTypes": [
-            "dataJob"
-          ],
-          "isLineage": true,
-          "isUpstream": false,
-          "name": "UsedBy"
-        }
-      },
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": "string"
-        }
-      ],
-      "name": "downstreamJobs",
-      "default": null,
-      "doc": "List of jobs (if any) that use the model",
-      "Urn": "Urn",
-      "urn_is_array": true
-    },
     {
       "Relationship": {
         "/*": {

datahub/metadata/schemas/MLPrimaryKeyProperties.avsc

@@ -97,6 +97,57 @@
               ],
               "name": "versionTag",
               "default": null
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "MetadataAttribution",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When this metadata was updated."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "source",
+                      "default": null,
+                      "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": {
+                        "type": "map",
+                        "values": "string"
+                      },
+                      "name": "sourceDetail",
+                      "default": {},
+                      "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc."
+                    }
+                  ],
+                  "doc": "Information about who, why, and how this metadata was applied"
+                }
+              ],
+              "name": "metadataAttribution",
+              "default": null
             }
           ],
           "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"