acryl-datahub-cloud 0.3.8.2rc8__py3-none-any.whl → 0.3.9rc2__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/ChartKey.avsc

@@ -9,11 +9,8 @@
       "editableChartProperties",
       "chartQuery",
       "inputFields",
-      "usageFeatures",
-      "lineageFeatures",
       "chartUsageStatistics",
       "embed",
-      "proposals",
       "browsePaths",
       "domains",
       "container",
@@ -30,6 +27,9 @@
       "incidentsSummary",
       "forms",
       "testResults",
+      "usageFeatures",
+      "lineageFeatures",
+      "proposals",
       "share",
       "origin",
       "documentation"

acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc

@@ -18,12 +18,12 @@
       "browsePaths",
       "status",
       "domains",
-      "proposals",
       "browsePathsV2",
       "structuredProperties",
       "forms",
       "testResults",
       "access",
+      "proposals",
       "share",
       "origin",
       "documentation"

acryl_datahub_cloud/metadata/schemas/CorpGroupKey.avsc

@@ -11,11 +11,11 @@
       "ownership",
       "status",
       "origin",
-      "corpGroupSettings",
       "roleMembership",
       "structuredProperties",
       "forms",
       "testResults",
+      "corpGroupSettings",
       "share"
     ],
     "entityDoc": "CorpGroup represents an identity of a group of users in the enterprise."

acryl_datahub_cloud/metadata/schemas/DashboardKey.avsc

@@ -10,11 +10,8 @@
       "deprecation",
       "dashboardUsageStatistics",
       "inputFields",
-      "usageFeatures",
-      "lineageFeatures",
       "subTypes",
       "embed",
-      "proposals",
       "dashboardInfo",
       "editableDashboardProperties",
       "ownership",
@@ -29,6 +26,9 @@
       "incidentsSummary",
       "forms",
       "testResults",
+      "usageFeatures",
+      "lineageFeatures",
+      "proposals",
       "share",
       "origin",
       "documentation"

acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc

@@ -8,7 +8,6 @@
       "domains",
       "deprecation",
       "versionInfo",
-      "proposals",
       "dataFlowInfo",
       "editableDataFlowProperties",
       "ownership",
@@ -25,6 +24,7 @@
       "forms",
       "subTypes",
       "testResults",
+      "proposals",
       "share",
       "origin",
       "lineageFeatures",

acryl_datahub_cloud/metadata/schemas/DataHubActionInfo.avsc

@@ -55,7 +55,7 @@
             ],
             "name": "executorId",
             "default": null,
-            "doc": "The id of the executor to use to execute the automation. Defaults to 'default' (runs locally)"
+            "doc": "The id of the executor pool to use to execute the automation. Defaults to 'default' (runs locally embedded)"
           }
         ]
       },

acryl_datahub_cloud/metadata/schemas/DataHubConnectionKey.avsc

@@ -6,7 +6,8 @@
     "entityCategory": "internal",
     "entityAspects": [
       "dataHubConnectionDetails",
-      "dataPlatformInstance"
+      "dataPlatformInstance",
+      "status"
     ]
   },
   "name": "DataHubConnectionKey",

acryl_datahub_cloud/metadata/schemas/DataHubIngestionSourceInfo.avsc

@@ -16,7 +16,8 @@
     },
     {
       "Searchable": {
-        "fieldType": "TEXT_PARTIAL"
+        "fieldType": "KEYWORD",
+        "queryByDefault": false
       },
       "type": "string",
       "name": "type",
@@ -83,7 +84,9 @@
           },
           {
             "Searchable": {
-              "fieldName": "sourceExecutorId"
+              "fieldName": "sourceExecutorId",
+              "fieldType": "KEYWORD",
+              "queryByDefault": false
             },
             "type": [
               "null",
@@ -91,7 +94,7 @@
             ],
             "name": "executorId",
             "default": null,
-            "doc": "The id of the executor to use to execute the ingestion run"
+            "doc": "The id of the executor pool to use to execute the ingestion run"
           },
           {
             "type": [
@@ -129,7 +132,9 @@
           "fields": [
             {
               "Searchable": {
-                "fieldName": "sourceType"
+                "fieldName": "sourceType",
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
               },
               "type": {
                 "type": "enum",

acryl_datahub_cloud/metadata/schemas/DataHubMetricCubeDefinition.avsc

@@ -0,0 +1,185 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubMetricCubeDefinition"
+  },
+  "name": "DataHubMetricCubeDefinition",
+  "namespace": "com.linkedin.pegasus2avro.metric",
+  "fields": [
+    {
+      "Searchable": {},
+      "type": "string",
+      "name": "name",
+      "doc": " Display name of the metric cube"
+    },
+    {
+      "Searchable": {},
+      "type": "string",
+      "name": "type",
+      "default": "custom",
+      "doc": "A type or category for the metric cube. This is used to categorize the metric cube & for filtering.\n\nThis may be used to group similar types of metrics for a given entity, e.g. 'row_count', 'error_count', etc.\nthat originated in different places."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": " Optional description for the metric"
+    },
+    {
+      "Searchable": {},
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "entity",
+      "default": null,
+      "doc": " An optional URN for the entity that this metric cube is associated with.",
+      "Urn": "Urn"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubMetricCubeOrigin",
+        "namespace": "com.linkedin.pegasus2avro.metric",
+        "fields": [
+          {
+            "Searchable": {
+              "addToFilters": true,
+              "fieldName": "originType",
+              "filterNameOverride": "Origin"
+            },
+            "type": {
+              "type": "enum",
+              "symbolDocs": {
+                "ASSERTION_MONITOR": "Assertion monitor created the metric.",
+                "MANUAL": "Manually created metric, via UI or API."
+              },
+              "name": "DataHubMetricSourceType",
+              "namespace": "com.linkedin.pegasus2avro.metric",
+              "symbols": [
+                "MANUAL",
+                "ASSERTION_MONITOR"
+              ]
+            },
+            "name": "type",
+            "doc": "Message associated with the incident"
+          },
+          {
+            "Searchable": {
+              "fieldType": "URN"
+            },
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "originUrn",
+            "default": null,
+            "doc": "Reference to the source that created the metric.\nIn the case of assertion monitor, this is the URN of the assertion monitor.",
+            "Urn": "Urn"
+          }
+        ],
+        "doc": "Information about the origin of the metric cube"
+      },
+      "name": "origin",
+      "doc": "The origin of the metric cube."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "DataHubMetricCubeMeasures",
+          "namespace": "com.linkedin.pegasus2avro.metric",
+          "fields": [
+            {
+              "type": {
+                "type": "record",
+                "name": "DataHubMetricCubeMeasure",
+                "namespace": "com.linkedin.pegasus2avro.metric",
+                "fields": [
+                  {
+                    "type": "string",
+                    "name": "name",
+                    "doc": " The name of the measure"
+                  }
+                ],
+                "doc": "The definition of a metric cube measure."
+              },
+              "name": "measure",
+              "doc": " The first measure being tracked in the cube."
+            }
+          ],
+          "doc": "The definition of the measures of a metric cube.\nA measure is a metric that is being tracked in the cube."
+        }
+      ],
+      "name": "measures",
+      "default": null,
+      "doc": " Optional - The measures of the cube for display purposes."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "DataHubMetricCubeDimensions",
+          "namespace": "com.linkedin.pegasus2avro.metric",
+          "fields": [
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "DataHubMetricCubeDimension",
+                  "namespace": "com.linkedin.pegasus2avro.metric",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "name",
+                      "doc": " The name of the dimension"
+                    }
+                  ],
+                  "doc": "The definition of a metric cube dimension."
+                }
+              ],
+              "name": "dim1",
+              "default": null,
+              "doc": " The first measure being tracked in the cube."
+            },
+            {
+              "type": [
+                "null",
+                "com.linkedin.pegasus2avro.metric.DataHubMetricCubeDimension"
+              ],
+              "name": "dim2",
+              "default": null,
+              "doc": "Optional: A second measure being tracked in the cube."
+            },
+            {
+              "type": [
+                "null",
+                "com.linkedin.pegasus2avro.metric.DataHubMetricCubeDimension"
+              ],
+              "name": "dim3",
+              "default": null,
+              "doc": "Optional: A third measure being tracked in the cube."
+            }
+          ],
+          "doc": "The dimensions of the cube. This is what you filter and group by.\nThis is a record to allow for future expansion of the dimensions."
+        }
+      ],
+      "name": "dimensions",
+      "default": null,
+      "doc": "Optional - The dimensions of the cube for display purposes."
+    }
+  ],
+  "doc": "The structure of an individual metric cube in DataHub."
+}

acryl_datahub_cloud/metadata/schemas/DataHubMetricCubeEvent.avsc

@@ -0,0 +1,184 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubMetricCubeEvent",
+    "type": "timeseries"
+  },
+  "name": "DataHubMetricCubeEvent",
+  "namespace": "com.linkedin.pegasus2avro.metric",
+  "fields": [
+    {
+      "type": "long",
+      "name": "reportedTimeMillis",
+      "doc": "The event or bucket reported time field as epoch at UTC in milli seconds.\nThis must be provided in order to pass validation."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "double",
+      "name": "measure",
+      "doc": "The first measure value - Typically this is the primary metric."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "dim1",
+      "default": null,
+      "doc": "The first dimension value(s). Array type to support multi-dimensionality."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "dim2",
+      "default": null,
+      "doc": "The second dimension value(s). Array type to support multi-dimensionality."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "dim3",
+      "default": null,
+      "doc": "The third dimension value(s). Array type to support multi-dimensionality."
+    },
+    {
+      "type": "long",
+      "name": "timestampMillis",
+      "doc": "The event timestamp field as epoch at UTC in milli seconds."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeWindowSize",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "name": "CalendarInterval",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "SECOND",
+                  "MINUTE",
+                  "HOUR",
+                  "DAY",
+                  "WEEK",
+                  "MONTH",
+                  "QUARTER",
+                  "YEAR"
+                ]
+              },
+              "name": "unit",
+              "doc": "Interval unit such as minute/hour/day etc."
+            },
+            {
+              "type": "int",
+              "name": "multiple",
+              "default": 1,
+              "doc": "How many units. Defaults to 1."
+            }
+          ],
+          "doc": "Defines the size of a time window."
+        }
+      ],
+      "name": "eventGranularity",
+      "default": null,
+      "doc": "Granularity of the event if applicable"
+    },
+    {
+      "type": [
+        {
+          "type": "record",
+          "name": "PartitionSpec",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": "string",
+              "name": "partition",
+              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "TimeWindow",
+                  "namespace": "com.linkedin.pegasus2avro.timeseries",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "startTimeMillis",
+                      "doc": "Start time as epoch at UTC."
+                    },
+                    {
+                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
+                      "name": "length",
+                      "doc": "The length of the window."
+                    }
+                  ]
+                }
+              ],
+              "name": "timePartition",
+              "default": null,
+              "doc": "Time window of the partition, if we are able to extract it from the partition key."
+            },
+            {
+              "deprecated": true,
+              "type": {
+                "type": "enum",
+                "name": "PartitionType",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "FULL_TABLE",
+                  "QUERY",
+                  "PARTITION"
+                ]
+              },
+              "name": "type",
+              "default": "PARTITION",
+              "doc": "Unused!"
+            }
+          ],
+          "doc": "A reference to a specific partition in a dataset."
+        },
+        "null"
+      ],
+      "name": "partitionSpec",
+      "default": {
+        "partition": "FULL_TABLE_SNAPSHOT",
+        "type": "FULL_TABLE",
+        "timePartition": null
+      },
+      "doc": "The optional partition specification."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "messageId",
+      "default": null,
+      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
+    }
+  ],
+  "doc": "A timeseries measure event, e.g. a single observation."
+}

acryl_datahub_cloud/metadata/schemas/DataHubMetricCubeKey.avsc

@@ -0,0 +1,22 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubMetricCubeKey",
+    "keyForEntity": "dataHubMetricCube",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "dataHubMetricCubeDefinition",
+      "dataHubMetricCubeEvent"
+    ]
+  },
+  "name": "DataHubMetricCubeKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "Unique id for the cube type."
+    }
+  ],
+  "doc": "Key for a DataHub Metric Cube, e.g. an internal metric."
+}

acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc

@@ -10,7 +10,6 @@
       "domains",
       "deprecation",
       "versionInfo",
-      "proposals",
       "dataJobInfo",
       "dataJobInputOutput",
       "editableDataJobProperties",
@@ -25,15 +24,16 @@
       "browsePathsV2",
       "structuredProperties",
       "forms",
-      "anomaliesSummary",
       "subTypes",
       "incidentsSummary",
       "testResults",
+      "dataTransformLogic",
+      "proposals",
+      "anomaliesSummary",
       "share",
       "origin",
       "lineageFeatures",
-      "documentation",
-      "dataTransformLogic"
+      "documentation"
     ]
   },
   "name": "DataJobKey",

acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceKey.avsc

@@ -10,7 +10,8 @@
       "globalTags",
       "institutionalMemory",
       "deprecation",
-      "status"
+      "status",
+      "icebergWarehouseInfo"
     ]
   },
   "name": "DataPlatformInstanceKey",

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceInput.avsc

@@ -10,8 +10,10 @@
       "Relationship": {
         "/*": {
           "entityTypes": [
-            "dataset"
+            "dataset",
+            "mlModel"
           ],
+          "isLineage": true,
           "name": "Consumes"
         }
       },
@@ -29,7 +31,7 @@
         "items": "string"
       },
       "name": "inputs",
-      "doc": "Input datasets to be consumed",
+      "doc": "Input assets consumed",
       "Urn": "Urn",
       "urn_is_array": true
     }

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -13,6 +13,8 @@
             "dataset",
             "mlModel"
           ],
+          "isLineage": true,
+          "isUpstream": false,
           "name": "Produces"
         }
       },

acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc

@@ -11,24 +11,14 @@
       "datasetUsageStatistics",
       "operation",
       "domains",
-      "proposals",
-      "schemaProposals",
       "schemaMetadata",
       "status",
       "container",
       "deprecation",
-      "usageFeatures",
-      "storageFeatures",
-      "lineageFeatures",
       "testResults",
       "siblings",
       "embed",
       "incidentsSummary",
-      "inferredNeighbors",
-      "inferredMetadata",
-      "schemaFieldsInferredMetadata",
-      "schemaFieldsInferredNeighbors",
-      "assertionsSummary",
       "datasetProperties",
       "editableDatasetProperties",
       "datasetDeprecation",
@@ -42,16 +32,27 @@
       "browsePaths",
       "dataPlatformInstance",
       "browsePathsV2",
-      "anomaliesSummary",
       "access",
       "structuredProperties",
       "forms",
       "partitionsSummary",
+      "versionProperties",
+      "icebergCatalogInfo",
+      "inferredNeighbors",
+      "inferredMetadata",
+      "schemaFieldsInferredMetadata",
+      "schemaFieldsInferredNeighbors",
+      "assertionsSummary",
+      "usageFeatures",
+      "storageFeatures",
+      "lineageFeatures",
+      "proposals",
+      "schemaProposals",
+      "anomaliesSummary",
       "share",
       "origin",
       "documentation",
-      "entityInferenceMetadata",
-      "versionProperties"
+      "entityInferenceMetadata"
     ],
     "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
   },