acryl-datahub 0.15.0rc25__py3-none-any.whl → 0.15.0.1__py3-none-any.whl

datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -19,6 +19,8 @@ from .....schema_classes import CostCostClass
 from .....schema_classes import CostCostDiscriminatorClass
 from .....schema_classes import CostTypeClass
 from .....schema_classes import DataPlatformInstanceClass
+from .....schema_classes import DataTransformClass
+from .....schema_classes import DataTransformLogicClass
 from .....schema_classes import DeprecationClass
 from .....schema_classes import DocumentationClass
 from .....schema_classes import DocumentationAssociationClass
@@ -79,6 +81,8 @@ CostCost = CostCostClass
 CostCostDiscriminator = CostCostDiscriminatorClass
 CostType = CostTypeClass
 DataPlatformInstance = DataPlatformInstanceClass
+DataTransform = DataTransformClass
+DataTransformLogic = DataTransformLogicClass
 Deprecation = DeprecationClass
 Documentation = DocumentationClass
 DocumentationAssociation = DocumentationAssociationClass

datahub/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py

@@ -30,6 +30,7 @@ from ......schema_classes import MLModelFactorsClass
 from ......schema_classes import MLModelGroupPropertiesClass
 from ......schema_classes import MLModelPropertiesClass
 from ......schema_classes import MLPrimaryKeyPropertiesClass
+from ......schema_classes import MLTrainingRunPropertiesClass
 from ......schema_classes import MetricsClass
 from ......schema_classes import QuantitativeAnalysesClass
 from ......schema_classes import SourceCodeClass
@@ -61,6 +62,7 @@ MLModelFactors = MLModelFactorsClass
 MLModelGroupProperties = MLModelGroupPropertiesClass
 MLModelProperties = MLModelPropertiesClass
 MLPrimaryKeyProperties = MLPrimaryKeyPropertiesClass
+MLTrainingRunProperties = MLTrainingRunPropertiesClass
 Metrics = MetricsClass
 QuantitativeAnalyses = QuantitativeAnalysesClass
 SourceCode = SourceCodeClass

datahub/metadata/schema.avsc

@@ -400,6 +400,69 @@
     ],
     "doc": "Institutional memory of an entity. This is a way to link to relevant documentation and provide description of the documentation. Institutional or tribal knowledge is very important for users to leverage the entity."
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataTransformLogic"
+    },
+    "name": "DataTransformLogic",
+    "namespace": "com.linkedin.pegasus2avro.common",
+    "fields": [
+      {
+        "type": {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "DataTransform",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "QueryStatement",
+                    "namespace": "com.linkedin.pegasus2avro.query",
+                    "fields": [
+                      {
+                        "type": "string",
+                        "name": "value",
+                        "doc": "The query text"
+                      },
+                      {
+                        "type": {
+                          "type": "enum",
+                          "symbolDocs": {
+                            "SQL": "A SQL Query"
+                          },
+                          "name": "QueryLanguage",
+                          "namespace": "com.linkedin.pegasus2avro.query",
+                          "symbols": [
+                            "SQL"
+                          ]
+                        },
+                        "name": "language",
+                        "default": "SQL",
+                        "doc": "The language of the Query, e.g. SQL."
+                      }
+                    ],
+                    "doc": "A query statement against one or more data assets."
+                  }
+                ],
+                "name": "queryStatement",
+                "default": null,
+                "doc": "The data transform may be defined by a query statement"
+              }
+            ],
+            "doc": "Information about a transformation. It may be a query,"
+          }
+        },
+        "name": "transforms",
+        "doc": "List of transformations applied"
+      }
+    ],
+    "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+  },
   {
     "type": "record",
     "Aspect": {
@@ -6827,6 +6890,21 @@
                           "default": null,
                           "doc": "URL where the reference exist"
                         },
+                        {
+                          "Searchable": {
+                            "boostScore": 10.0,
+                            "enableAutocomplete": true,
+                            "fieldType": "WORD_GRAM",
+                            "queryByDefault": true
+                          },
+                          "type": [
+                            "null",
+                            "string"
+                          ],
+                          "name": "name",
+                          "default": null,
+                          "doc": "Display name of the MLModel"
+                        },
                         {
                           "Searchable": {
                             "fieldType": "TEXT",
@@ -6841,6 +6919,7 @@
                           "doc": "Documentation of the MLModel"
                         },
                         {
+                          "deprecated": true,
                           "type": [
                             "null",
                             "long"
@@ -6849,6 +6928,24 @@
                           "default": null,
                           "doc": "Date when the MLModel was developed"
                         },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "created",
+                          "default": null,
+                          "doc": "Audit stamp containing who created this and when"
+                        },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "lastModified",
+                          "default": null,
+                          "doc": "Date when the MLModel was last modified"
+                        },
                         {
                           "type": [
                             "null",
@@ -7081,7 +7178,8 @@
                           "Relationship": {
                             "/*": {
                               "entityTypes": [
-                                "dataJob"
+                                "dataJob",
+                                "dataProcessInstance"
                               ],
                               "isLineage": true,
                               "name": "TrainedBy"
@@ -7098,7 +7196,7 @@
                           ],
                           "name": "trainingJobs",
                           "default": null,
-                          "doc": "List of jobs (if any) used to train the model"
+                          "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
                         },
                         {
                           "Relationship": {
@@ -8415,6 +8513,21 @@
                           "default": {},
                           "doc": "Custom property bag."
                         },
+                        {
+                          "Searchable": {
+                            "boostScore": 10.0,
+                            "enableAutocomplete": true,
+                            "fieldType": "WORD_GRAM",
+                            "queryByDefault": true
+                          },
+                          "type": [
+                            "null",
+                            "string"
+                          ],
+                          "name": "name",
+                          "default": null,
+                          "doc": "Display name of the MLModelGroup"
+                        },
                         {
                           "Searchable": {
                             "fieldType": "TEXT",
@@ -8429,6 +8542,7 @@
                           "doc": "Documentation of the MLModelGroup"
                         },
                         {
+                          "deprecated": true,
                           "type": [
                             "null",
                             "long"
@@ -8437,6 +8551,47 @@
                           "default": null,
                           "doc": "Date when the MLModelGroup was developed"
                         },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "created",
+                          "default": null,
+                          "doc": "Time and Actor who created the MLModelGroup"
+                        },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "lastModified",
+                          "default": null,
+                          "doc": "Date when the MLModelGroup was last modified"
+                        },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob"
+                              ],
+                              "isLineage": true,
+                              "name": "TrainedBy"
+                            }
+                          },
+                          "Urn": "Urn",
+                          "urn_is_array": true,
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "trainingJobs",
+                          "default": null,
+                          "doc": "List of jobs (if any) used to train the model group. Visible in Lineage."
+                        },
                         {
                           "type": [
                             "null",
@@ -11855,35 +12010,7 @@
     "namespace": "com.linkedin.pegasus2avro.query",
     "fields": [
       {
-        "type": {
-          "type": "record",
-          "name": "QueryStatement",
-          "namespace": "com.linkedin.pegasus2avro.query",
-          "fields": [
-            {
-              "type": "string",
-              "name": "value",
-              "doc": "The query text"
-            },
-            {
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "SQL": "A SQL Query"
-                },
-                "name": "QueryLanguage",
-                "namespace": "com.linkedin.pegasus2avro.query",
-                "symbols": [
-                  "SQL"
-                ]
-              },
-              "name": "language",
-              "default": "SQL",
-              "doc": "The language of the Query, e.g. SQL."
-            }
-          ],
-          "doc": "A query statement against one or more data assets."
-        },
+        "type": "com.linkedin.pegasus2avro.query.QueryStatement",
         "name": "statement",
         "doc": "The Query Statement."
       },
@@ -12619,7 +12746,8 @@
         "Relationship": {
           "/*": {
             "entityTypes": [
-              "dataset"
+              "dataset",
+              "mlModel"
             ],
             "name": "Produces"
           }
@@ -12944,6 +13072,93 @@
     "doc": "Properties associated with a MLPrimaryKey editable from the UI"
   },
   "com.linkedin.pegasus2avro.ml.metadata.SourceCode",
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "mlTrainingRunProperties"
+    },
+    "name": "MLTrainingRunProperties",
+    "namespace": "com.linkedin.pegasus2avro.ml.metadata",
+    "fields": [
+      {
+        "Searchable": {
+          "/*": {
+            "fieldType": "TEXT",
+            "queryByDefault": true
+          }
+        },
+        "type": {
+          "type": "map",
+          "values": "string"
+        },
+        "name": "customProperties",
+        "default": {},
+        "doc": "Custom property bag."
+      },
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.url.Url",
+          "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+        },
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "externalUrl",
+        "default": null,
+        "doc": "URL where the reference exist"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "id",
+        "default": null,
+        "doc": "Run Id of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "string"
+          }
+        ],
+        "name": "outputUrls",
+        "default": null,
+        "doc": "List of URLs for the Outputs of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.ml.metadata.MLHyperParam"
+          }
+        ],
+        "name": "hyperParams",
+        "default": null,
+        "doc": "Hyperparameters of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.ml.metadata.MLMetric"
+          }
+        ],
+        "name": "trainingMetrics",
+        "default": null,
+        "doc": "Metrics of the ML Training Run"
+      }
+    ],
+    "doc": "The inputs and outputs of this training run"
+  },
   "com.linkedin.pegasus2avro.ml.metadata.EthicalConsiderations",
   "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties",
   "com.linkedin.pegasus2avro.ml.metadata.MLModelFactorPrompts",
@@ -15561,7 +15776,11 @@
         "dataProcessInstanceRelationships",
         "dataProcessInstanceRunEvent",
         "status",
-        "testResults"
+        "testResults",
+        "dataPlatformInstance",
+        "subTypes",
+        "container",
+        "mlTrainingRunProperties"
       ],
       "entityDoc": "DataProcessInstance represents an instance of a datajob/jobflow run"
     },

datahub/metadata/schemas/DataJobKey.avsc

@@ -25,7 +25,8 @@
       "forms",
       "subTypes",
       "incidentsSummary",
-      "testResults"
+      "testResults",
+      "dataTransformLogic"
     ]
   },
   "name": "DataJobKey",

datahub/metadata/schemas/DataProcessInstanceKey.avsc

@@ -11,7 +11,11 @@
       "dataProcessInstanceRelationships",
       "dataProcessInstanceRunEvent",
       "status",
-      "testResults"
+      "testResults",
+      "dataPlatformInstance",
+      "subTypes",
+      "container",
+      "mlTrainingRunProperties"
     ],
     "entityDoc": "DataProcessInstance represents an instance of a datajob/jobflow run"
   },

datahub/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -10,7 +10,8 @@
       "Relationship": {
         "/*": {
           "entityTypes": [
-            "dataset"
+            "dataset",
+            "mlModel"
           ],
           "name": "Produces"
         }

datahub/metadata/schemas/DataTransformLogic.avsc

@@ -0,0 +1,63 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataTransformLogic"
+  },
+  "name": "DataTransformLogic",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "DataTransform",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "QueryStatement",
+                  "namespace": "com.linkedin.pegasus2avro.query",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "value",
+                      "doc": "The query text"
+                    },
+                    {
+                      "type": {
+                        "type": "enum",
+                        "symbolDocs": {
+                          "SQL": "A SQL Query"
+                        },
+                        "name": "QueryLanguage",
+                        "namespace": "com.linkedin.pegasus2avro.query",
+                        "symbols": [
+                          "SQL"
+                        ]
+                      },
+                      "name": "language",
+                      "default": "SQL",
+                      "doc": "The language of the Query, e.g. SQL."
+                    }
+                  ],
+                  "doc": "A query statement against one or more data assets."
+                }
+              ],
+              "name": "queryStatement",
+              "default": null,
+              "doc": "The data transform may be defined by a query statement"
+            }
+          ],
+          "doc": "Information about a transformation. It may be a query,"
+        }
+      },
+      "name": "transforms",
+      "doc": "List of transformations applied"
+    }
+  ],
+  "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+}

datahub/metadata/schemas/MLModelGroupProperties.avsc

@@ -21,6 +21,21 @@
       "default": {},
       "doc": "Custom property bag."
     },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldType": "WORD_GRAM",
+        "queryByDefault": true
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "name",
+      "default": null,
+      "doc": "Display name of the MLModelGroup"
+    },
     {
       "Searchable": {
         "fieldType": "TEXT",
@@ -35,6 +50,7 @@
       "doc": "Documentation of the MLModelGroup"
     },
     {
+      "deprecated": true,
       "type": [
         "null",
         "long"
@@ -43,6 +59,72 @@
       "default": null,
       "doc": "Date when the MLModelGroup was developed"
     },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the event occur"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "actor",
+              "default": null,
+              "doc": "Optional: The actor urn involved in the event.",
+              "Urn": "Urn"
+            }
+          ],
+          "doc": "A standard event timestamp"
+        }
+      ],
+      "name": "created",
+      "default": null,
+      "doc": "Time and Actor who created the MLModelGroup"
+    },
+    {
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.TimeStamp"
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "Date when the MLModelGroup was last modified"
+    },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataJob"
+          ],
+          "isLineage": true,
+          "name": "TrainedBy"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "trainingJobs",
+      "default": null,
+      "doc": "List of jobs (if any) used to train the model group. Visible in Lineage.",
+      "Urn": "Urn",
+      "urn_is_array": true
+    },
     {
       "type": [
         "null",