acryl-datahub 0.15.0.6rc3__py3-none-any.whl → 1.0.0__py3-none-any.whl

datahub/metadata/schemas/AssertionInfo.avsc

@@ -2010,7 +2010,9 @@
                                               "fieldName": "glossaryTerms",
                                               "fieldType": "URN",
                                               "filterNameOverride": "Glossary Term",
-                                              "hasValuesFieldName": "hasGlossaryTerms"
+                                              "hasValuesFieldName": "hasGlossaryTerms",
+                                              "includeSystemModifiedAt": true,
+                                              "systemModifiedAtFieldName": "termsModifiedAt"
                                             },
                                             "java": {
                                               "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributeInfo.avsc

@@ -221,7 +221,9 @@
         "/terms/*/urn": {
           "boostScore": 0.5,
           "fieldName": "editedFieldGlossaryTerms",
-          "fieldType": "URN"
+          "fieldType": "URN",
+          "includeSystemModifiedAt": true,
+          "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
         }
       },
       "type": [
@@ -254,7 +256,9 @@
                         "fieldName": "glossaryTerms",
                         "fieldType": "URN",
                         "filterNameOverride": "Glossary Term",
-                        "hasValuesFieldName": "hasGlossaryTerms"
+                        "hasValuesFieldName": "hasGlossaryTerms",
+                        "includeSystemModifiedAt": true,
+                        "systemModifiedAtFieldName": "termsModifiedAt"
                       },
                       "java": {
                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributes.avsc

@@ -31,6 +31,12 @@
           "namespace": "com.linkedin.pegasus2avro.businessattribute",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "schemaFieldBusinessAttribute",
+                "includeSystemModifiedAt": true,
+                "queryByDefault": false,
+                "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt"
+              },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.BusinessAttributeUrn"
               },

datahub/metadata/schemas/ChartInfo.avsc

@@ -39,6 +39,7 @@
     },
     {
       "Searchable": {
+        "boostScore": 10.0,
         "enableAutocomplete": true,
         "fieldNameAliases": [
           "_entityName"

datahub/metadata/schemas/CorpGroupKey.avsc

@@ -14,7 +14,8 @@
       "roleMembership",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "subTypes"
     ],
     "entityDoc": "CorpGroup represents an identity of a group of users in the enterprise."
   },

datahub/metadata/schemas/CorpUserInfo.avsc

@@ -165,6 +165,19 @@
       "name": "countryCode",
       "default": null,
       "doc": "two uppercase letters country code. e.g.  US"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN",
+        "queryByDefault": false
+      },
+      "type": [
+        "boolean",
+        "null"
+      ],
+      "name": "system",
+      "default": false,
+      "doc": "Whether the corpUser is a system user."
     }
   ],
   "doc": "Linkedin corp user information"

datahub/metadata/schemas/CorpUserKey.avsc

@@ -18,7 +18,8 @@
       "roleMembership",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "subTypes"
     ],
     "entityDoc": "CorpUser represents an identity of a person (or an account) in the enterprise."
   },

datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc

@@ -16,7 +16,8 @@
     },
     {
       "Searchable": {
-        "fieldType": "TEXT_PARTIAL"
+        "fieldType": "KEYWORD",
+        "queryByDefault": false
       },
       "type": "string",
       "name": "type",
@@ -83,7 +84,9 @@
           },
           {
             "Searchable": {
-              "fieldName": "sourceExecutorId"
+              "fieldName": "sourceExecutorId",
+              "fieldType": "KEYWORD",
+              "queryByDefault": false
             },
             "type": [
               "null",
@@ -129,7 +132,9 @@
           "fields": [
             {
               "Searchable": {
-                "fieldName": "sourceType"
+                "fieldName": "sourceType",
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
               },
               "type": {
                 "type": "enum",

datahub/metadata/schemas/DataProcessInstanceInput.avsc

@@ -13,7 +13,6 @@
             "dataset",
             "mlModel"
           ],
-          "isLineage": true,
           "name": "Consumes"
         }
       },
@@ -34,6 +33,135 @@
       "doc": "Input assets consumed",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "inputEdges/*/created/actor",
+          "createdOn": "inputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "name": "DataProcessInstanceConsumes",
+          "properties": "inputEdges/*/properties",
+          "updatedActor": "inputEdges/*/lastModified/actor",
+          "updatedOn": "inputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "inputEdges",
+      "default": null,
+      "doc": "Input assets consumed by the data process instance, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the inputs field."
     }
   ],
   "doc": "Information about the inputs datasets of a Data process"

datahub/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -13,8 +13,6 @@
             "dataset",
             "mlModel"
           ],
-          "isLineage": true,
-          "isUpstream": false,
           "name": "Produces"
         }
       },
@@ -32,9 +30,139 @@
         "items": "string"
       },
       "name": "outputs",
-      "doc": "Output datasets to be produced",
+      "doc": "Output assets produced",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "outputEdges/*/created/actor",
+          "createdOn": "outputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "isUpstream": false,
+          "name": "DataProcessInstanceProduces",
+          "properties": "outputEdges/*/properties",
+          "updatedActor": "outputEdges/*/lastModified/actor",
+          "updatedOn": "outputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "outputEdges",
+      "default": null,
+      "doc": "Output assets produced by the data process instance during processing, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the outputs field."
     }
   ],
   "doc": "Information about the outputs of a Data process"

datahub/metadata/schemas/DataProcessKey.avsc

@@ -8,7 +8,8 @@
       "dataProcessInfo",
       "ownership",
       "status",
-      "testResults"
+      "testResults",
+      "subTypes"
     ]
   },
   "name": "DataProcessKey",

datahub/metadata/schemas/DataProductKey.avsc

@@ -14,7 +14,8 @@
       "status",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "subTypes"
     ]
   },
   "name": "DataProductKey",

datahub/metadata/schemas/DomainKey.avsc

@@ -10,7 +10,8 @@
       "ownership",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "displayProperties"
     ],
     "entityDoc": "A data domain within an organization."
   },

datahub/metadata/schemas/EditableSchemaMetadata.avsc

@@ -303,7 +303,9 @@
                 "/terms/*/urn": {
                   "boostScore": 0.5,
                   "fieldName": "editedFieldGlossaryTerms",
-                  "fieldType": "URN"
+                  "fieldType": "URN",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                 }
               },
               "type": [
@@ -336,7 +338,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/GlossaryNodeKey.avsc

@@ -11,7 +11,9 @@
       "status",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "subTypes",
+      "displayProperties"
     ]
   },
   "name": "GlossaryNodeKey",

datahub/metadata/schemas/GlossaryTermKey.avsc

@@ -16,7 +16,8 @@
       "browsePaths",
       "structuredProperties",
       "forms",
-      "testResults"
+      "testResults",
+      "subTypes"
     ]
   },
   "name": "GlossaryTermKey",

datahub/metadata/schemas/GlossaryTerms.avsc

@@ -26,7 +26,9 @@
                 "fieldName": "glossaryTerms",
                 "fieldType": "URN",
                 "filterNameOverride": "Glossary Term",
-                "hasValuesFieldName": "hasGlossaryTerms"
+                "hasValuesFieldName": "hasGlossaryTerms",
+                "includeSystemModifiedAt": true,
+                "systemModifiedAtFieldName": "termsModifiedAt"
               },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"