acryl-datahub-cloud 0.3.8.3rc1__py3-none-any.whl → 0.3.9__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceInput.avsc

@@ -10,7 +10,8 @@
       "Relationship": {
         "/*": {
           "entityTypes": [
-            "dataset"
+            "dataset",
+            "mlModel"
           ],
           "name": "Consumes"
         }
@@ -29,9 +30,138 @@
         "items": "string"
       },
       "name": "inputs",
-      "doc": "Input datasets to be consumed",
+      "doc": "Input assets consumed",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "inputEdges/*/created/actor",
+          "createdOn": "inputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "name": "DataProcessInstanceConsumes",
+          "properties": "inputEdges/*/properties",
+          "updatedActor": "inputEdges/*/lastModified/actor",
+          "updatedOn": "inputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "inputEdges",
+      "default": null,
+      "doc": "Input assets consumed by the data process instance, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the inputs field."
     }
   ],
   "doc": "Information about the inputs datasets of a Data process"

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -30,9 +30,139 @@
         "items": "string"
       },
       "name": "outputs",
-      "doc": "Output datasets to be produced",
+      "doc": "Output assets produced",
       "Urn": "Urn",
       "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*/destinationUrn": {
+          "createdActor": "outputEdges/*/created/actor",
+          "createdOn": "outputEdges/*/created/time",
+          "entityTypes": [
+            "dataset",
+            "mlModel"
+          ],
+          "isLineage": true,
+          "isUpstream": false,
+          "name": "DataProcessInstanceProduces",
+          "properties": "outputEdges/*/properties",
+          "updatedActor": "outputEdges/*/lastModified/actor",
+          "updatedOn": "outputEdges/*/lastModified/time"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "Edge",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "sourceUrn",
+                "default": null,
+                "doc": "Urn of the source of this relationship edge.\nIf not specified, assumed to be the entity that this aspect belongs to.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "destinationUrn",
+                "doc": "Urn of the destination of this relationship edge.",
+                "Urn": "Urn"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "AuditStamp",
+                    "namespace": "com.linkedin.pegasus2avro.common",
+                    "fields": [
+                      {
+                        "type": "long",
+                        "name": "time",
+                        "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": "string",
+                        "name": "actor",
+                        "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "java": {
+                          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "impersonator",
+                        "default": null,
+                        "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                        "Urn": "Urn"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "message",
+                        "default": null,
+                        "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                      }
+                    ],
+                    "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                  }
+                ],
+                "name": "created",
+                "default": null,
+                "doc": "Audit stamp containing who created this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  "com.linkedin.pegasus2avro.common.AuditStamp"
+                ],
+                "name": "lastModified",
+                "default": null,
+                "doc": "Audit stamp containing who last modified this relationship edge and when"
+              },
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "map",
+                    "values": "string"
+                  }
+                ],
+                "name": "properties",
+                "default": null,
+                "doc": "A generic properties bag that allows us to store specific information on this graph edge."
+              }
+            ],
+            "doc": "A common structure to represent all edges to entities when used inside aspects as collections\nThis ensures that all edges have common structure around audit-stamps and will support PATCH, time-travel automatically."
+          }
+        }
+      ],
+      "name": "outputEdges",
+      "default": null,
+      "doc": "Output assets produced by the data process instance during processing, with additional metadata.\nCounts as lineage.\nWill eventually deprecate the outputs field."
     }
   ],
   "doc": "Information about the outputs of a Data process"

acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc

@@ -11,24 +11,14 @@
       "datasetUsageStatistics",
       "operation",
       "domains",
-      "proposals",
-      "schemaProposals",
       "schemaMetadata",
       "status",
       "container",
       "deprecation",
-      "usageFeatures",
-      "storageFeatures",
-      "lineageFeatures",
       "testResults",
       "siblings",
       "embed",
       "incidentsSummary",
-      "inferredNeighbors",
-      "inferredMetadata",
-      "schemaFieldsInferredMetadata",
-      "schemaFieldsInferredNeighbors",
-      "assertionsSummary",
       "datasetProperties",
       "editableDatasetProperties",
       "datasetDeprecation",
@@ -42,16 +32,27 @@
       "browsePaths",
       "dataPlatformInstance",
       "browsePathsV2",
-      "anomaliesSummary",
       "access",
       "structuredProperties",
       "forms",
       "partitionsSummary",
+      "versionProperties",
+      "icebergCatalogInfo",
+      "inferredNeighbors",
+      "inferredMetadata",
+      "schemaFieldsInferredMetadata",
+      "schemaFieldsInferredNeighbors",
+      "assertionsSummary",
+      "usageFeatures",
+      "storageFeatures",
+      "lineageFeatures",
+      "proposals",
+      "schemaProposals",
+      "anomaliesSummary",
       "share",
       "origin",
       "documentation",
-      "entityInferenceMetadata",
-      "versionProperties"
+      "entityInferenceMetadata"
     ],
     "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
   },

acryl_datahub_cloud/metadata/schemas/EditableSchemaMetadata.avsc

@@ -305,7 +305,9 @@
                 "/terms/*/urn": {
                   "boostScore": 0.5,
                   "fieldName": "editedFieldGlossaryTerms",
-                  "fieldType": "URN"
+                  "fieldType": "URN",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                 }
               },
               "type": [
@@ -338,7 +340,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term Applied",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

acryl_datahub_cloud/metadata/schemas/ExecutionRequestInput.avsc

@@ -20,9 +20,14 @@
       "doc": "Arguments provided to the task"
     },
     {
+      "Searchable": {
+        "fieldName": "executorId",
+        "fieldType": "KEYWORD",
+        "queryByDefault": false
+      },
       "type": "string",
       "name": "executorId",
-      "doc": "Advanced: specify a specific executor to route the request to. If none is provided, a \"default\" executor is used."
+      "doc": "Advanced: specify a specific executor pool to route the request to. If none is provided, a \"default\" embedded executor is used."
     },
     {
       "type": {

acryl_datahub_cloud/metadata/schemas/ExecutionRequestSignal.avsc

@@ -18,7 +18,7 @@
       ],
       "name": "executorId",
       "default": null,
-      "doc": "Advanced: specify a specific executor to route the request to. If none is provided, a \"default\" executor is used."
+      "doc": "Advanced: specify a specific executor pool to route the request to. If none is provided, a default embedded executor is used."
     },
     {
       "type": {

acryl_datahub_cloud/metadata/schemas/FormInfo.avsc

@@ -53,6 +53,11 @@
           "namespace": "com.linkedin.pegasus2avro.form",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "promptId",
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
               "type": "string",
               "name": "id",
               "doc": "The unique id for this prompt. This must be GLOBALLY unique."

acryl_datahub_cloud/metadata/schemas/GlossaryTerms.avsc

@@ -26,7 +26,9 @@
                 "fieldName": "glossaryTerms",
                 "fieldType": "URN",
                 "filterNameOverride": "Glossary Term Applied",
-                "hasValuesFieldName": "hasGlossaryTerms"
+                "hasValuesFieldName": "hasGlossaryTerms",
+                "includeSystemModifiedAt": true,
+                "systemModifiedAtFieldName": "termsModifiedAt"
               },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

acryl_datahub_cloud/metadata/schemas/IcebergCatalogInfo.avsc

@@ -0,0 +1,28 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "icebergCatalogInfo"
+  },
+  "name": "IcebergCatalogInfo",
+  "namespace": "com.linkedin.pegasus2avro.dataset",
+  "fields": [
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "metadataPointer",
+      "default": null,
+      "doc": "When Datahub is the REST Catalog for an Iceberg Table, stores the current metadata pointer.\nIf the Iceberg table is managed by an external catalog, the metadata pointer is not set."
+    },
+    {
+      "type": [
+        "null",
+        "boolean"
+      ],
+      "name": "view",
+      "default": null
+    }
+  ],
+  "doc": "Iceberg Catalog metadata associated with an Iceberg table/view"
+}

acryl_datahub_cloud/metadata/schemas/IcebergWarehouseInfo.avsc

@@ -0,0 +1,96 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "icebergWarehouseInfo"
+  },
+  "name": "IcebergWarehouseInfo",
+  "namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
+  "fields": [
+    {
+      "type": "string",
+      "name": "dataRoot",
+      "doc": "Path of the root for the backing store of the tables in the warehouse."
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "clientId",
+      "doc": "clientId to be used to authenticate with storage hosting this warehouse",
+      "Urn": "Urn"
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "clientSecret",
+      "doc": "client secret to authenticate with storage hosting this warehouse",
+      "Urn": "Urn"
+    },
+    {
+      "type": "string",
+      "name": "region",
+      "doc": "region where the warehouse is located."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "role",
+      "default": null
+    },
+    {
+      "type": [
+        "null",
+        "int"
+      ],
+      "name": "tempCredentialExpirationSeconds",
+      "default": null
+    },
+    {
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "CORP": "Designates corporation fabrics",
+          "DEV": "Designates development fabrics",
+          "EI": "Designates early-integration fabrics",
+          "NON_PROD": "Designates non-production fabrics",
+          "PRD": "Alternative Prod spelling",
+          "PRE": "Designates pre-production fabrics",
+          "PROD": "Designates production fabrics",
+          "QA": "Designates quality assurance fabrics",
+          "RVW": "Designates review fabrics",
+          "SANDBOX": "Designates sandbox fabrics",
+          "STG": "Designates staging fabrics",
+          "TEST": "Designates testing fabrics",
+          "TST": "Alternative Test spelling",
+          "UAT": "Designates user acceptance testing fabrics"
+        },
+        "name": "FabricType",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "symbols": [
+          "DEV",
+          "TEST",
+          "QA",
+          "UAT",
+          "EI",
+          "PRE",
+          "STG",
+          "NON_PROD",
+          "PROD",
+          "CORP",
+          "RVW",
+          "PRD",
+          "TST",
+          "SANDBOX"
+        ],
+        "doc": "Fabric group type"
+      },
+      "name": "env"
+    }
+  ],
+  "doc": "An Iceberg warehouse location and credentails whose read/writes are governed by datahub catalog."
+}

acryl_datahub_cloud/metadata/schemas/IncidentActivityEvent.avsc

@@ -539,7 +539,10 @@
           },
           {
             "Searchable": {
-              "fieldType": "COUNT"
+              "/time": {
+                "fieldName": "startedAt",
+                "fieldType": "COUNT"
+              }
             },
             "type": [
               "null",

acryl_datahub_cloud/metadata/schemas/IncidentInfo.avsc

@@ -350,7 +350,10 @@
     },
     {
       "Searchable": {
-        "fieldType": "COUNT"
+        "/time": {
+          "fieldName": "startedAt",
+          "fieldType": "COUNT"
+        }
       },
       "type": [
         "null",

acryl_datahub_cloud/metadata/schemas/InputFields.avsc

@@ -553,7 +553,9 @@
                                         "fieldName": "glossaryTerms",
                                         "fieldType": "URN",
                                         "filterNameOverride": "Glossary Term Applied",
-                                        "hasValuesFieldName": "hasGlossaryTerms"
+                                        "hasValuesFieldName": "hasGlossaryTerms",
+                                        "includeSystemModifiedAt": true,
+                                        "systemModifiedAtFieldName": "termsModifiedAt"
                                       },
                                       "java": {
                                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

acryl_datahub_cloud/metadata/schemas/MLFeatureKey.avsc

@@ -8,7 +8,6 @@
       "glossaryTerms",
       "editableMlFeatureProperties",
       "domains",
-      "proposals",
       "mlFeatureProperties",
       "ownership",
       "institutionalMemory",
@@ -21,6 +20,7 @@
       "structuredProperties",
       "forms",
       "testResults",
+      "proposals",
       "share",
       "origin",
       "lineageFeatures",

acryl_datahub_cloud/metadata/schemas/MLFeatureTableKey.avsc

@@ -8,7 +8,6 @@
       "glossaryTerms",
       "editableMlFeatureTableProperties",
       "domains",
-      "proposals",
       "mlFeatureTableProperties",
       "ownership",
       "institutionalMemory",
@@ -21,6 +20,7 @@
       "structuredProperties",
       "forms",
       "testResults",
+      "proposals",
       "share",
       "origin",
       "lineageFeatures",

acryl_datahub_cloud/metadata/schemas/MLModelGroupKey.avsc

@@ -8,7 +8,6 @@
       "glossaryTerms",
       "editableMlModelGroupProperties",
       "domains",
-      "proposals",
       "mlModelGroupProperties",
       "ownership",
       "status",
@@ -20,6 +19,7 @@
       "structuredProperties",
       "forms",
       "testResults",
+      "proposals",
       "share",
       "origin",
       "lineageFeatures",

acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc

@@ -8,7 +8,6 @@
       "glossaryTerms",
       "editableMlModelProperties",
       "domains",
-      "proposals",
       "ownership",
       "mlModelProperties",
       "intendedUse",
@@ -31,12 +30,13 @@
       "structuredProperties",
       "forms",
       "testResults",
+      "versionProperties",
+      "proposals",
       "share",
       "origin",
       "lineageFeatures",
       "documentation",
-      "incidentsSummary",
-      "versionProperties"
+      "incidentsSummary"
     ]
   },
   "name": "MLModelKey",