acryl-datahub-cloud 0.3.8rc8__py3-none-any.whl → 0.3.8rc9__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DatasetProperties.avsc

@@ -0,0 +1,165 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datasetProperties"
+  },
+  "name": "DatasetProperties",
+  "namespace": "com.linkedin.pegasus2avro.dataset",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldNameAliases": [
+          "_entityName"
+        ],
+        "fieldType": "WORD_GRAM"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "name",
+      "default": null,
+      "doc": "Display name of the Dataset"
+    },
+    {
+      "Searchable": {
+        "addToFilters": false,
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldType": "WORD_GRAM"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "qualifiedName",
+      "default": null,
+      "doc": "Fully-qualified name of the Dataset"
+    },
+    {
+      "Searchable": {
+        "fieldType": "TEXT",
+        "hasValuesFieldName": "hasDescription"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "Documentation of the dataset"
+    },
+    {
+      "deprecated": "Use ExternalReference.externalUrl field instead.",
+      "java": {
+        "class": "java.net.URI"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "uri",
+      "default": null,
+      "doc": "The abstracted URI such as hdfs:///data/tracking/PageViewEvent, file:///dir/file_name. Uri should not include any environment specific properties. Some datasets might not have a standardized uri, which makes this field optional (i.e. kafka topic)."
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the event occur"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "actor",
+              "default": null,
+              "doc": "Optional: The actor urn involved in the event.",
+              "Urn": "Urn"
+            }
+          ],
+          "doc": "A standard event timestamp"
+        }
+      ],
+      "name": "created",
+      "default": null,
+      "doc": "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.TimeStamp"
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)"
+    },
+    {
+      "deprecated": "Use GlobalTags aspect instead.",
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "tags",
+      "default": [],
+      "doc": "[Legacy] Unstructured tags for the dataset. Structured tags can be applied via the `GlobalTags` aspect.\nThis is now deprecated."
+    }
+  ],
+  "doc": "Properties associated with a Dataset"
+}

acryl_datahub_cloud/metadata/schemas/DatasetUpstreamLineage.avsc

@@ -0,0 +1,129 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datasetUpstreamLineage"
+  },
+  "deprecated": "use UpstreamLineage.fineGrainedLineages instead",
+  "name": "DatasetUpstreamLineage",
+  "namespace": "com.linkedin.pegasus2avro.dataset",
+  "fields": [
+    {
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "deprecated": "use FineGrainedLineage instead",
+          "name": "DatasetFieldMapping",
+          "namespace": "com.linkedin.pegasus2avro.dataset",
+          "fields": [
+            {
+              "type": {
+                "type": "record",
+                "name": "AuditStamp",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "fields": [
+                  {
+                    "type": "long",
+                    "name": "time",
+                    "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": "string",
+                    "name": "actor",
+                    "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "impersonator",
+                    "default": null,
+                    "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "message",
+                    "default": null,
+                    "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                  }
+                ],
+                "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+              },
+              "name": "created",
+              "doc": "Audit stamp containing who reported the field mapping and when"
+            },
+            {
+              "type": [
+                {
+                  "type": "enum",
+                  "symbolDocs": {
+                    "BLACKBOX": "Field transformation expressed as unknown black box function.",
+                    "IDENTITY": "Field transformation expressed as Identity function."
+                  },
+                  "name": "TransformationType",
+                  "namespace": "com.linkedin.pegasus2avro.common.fieldtransformer",
+                  "symbols": [
+                    "BLACKBOX",
+                    "IDENTITY"
+                  ],
+                  "doc": "Type of the transformation involved in generating destination fields from source fields."
+                },
+                {
+                  "type": "record",
+                  "name": "UDFTransformer",
+                  "namespace": "com.linkedin.pegasus2avro.common.fieldtransformer",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "udf",
+                      "doc": "A UDF mentioning how the source fields got transformed to destination field. This is the FQCN(Fully Qualified Class Name) of the udf."
+                    }
+                  ],
+                  "doc": "Field transformation expressed in UDF"
+                }
+              ],
+              "name": "transformation",
+              "doc": "Transfomration function between the fields involved"
+            },
+            {
+              "type": {
+                "type": "array",
+                "items": [
+                  "string"
+                ]
+              },
+              "name": "sourceFields",
+              "doc": "Source fields from which the fine grained lineage is derived"
+            },
+            {
+              "deprecated": "use SchemaFieldPath and represent as generic Urn instead",
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.DatasetFieldUrn"
+              },
+              "type": "string",
+              "name": "destinationField",
+              "doc": "Destination field which is derived from source fields",
+              "Urn": "DatasetFieldUrn"
+            }
+          ],
+          "doc": "Representation of mapping between fields in source dataset to the field in destination dataset"
+        }
+      },
+      "name": "fieldMappings",
+      "doc": "Upstream to downstream field level lineage mappings"
+    }
+  ],
+  "doc": "Fine Grained upstream lineage for fields in a dataset"
+}

acryl_datahub_cloud/metadata/schemas/DatasetUsageStatistics.avsc

@@ -0,0 +1,247 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datasetUsageStatistics",
+    "type": "timeseries"
+  },
+  "name": "DatasetUsageStatistics",
+  "namespace": "com.linkedin.pegasus2avro.dataset",
+  "fields": [
+    {
+      "type": "long",
+      "name": "timestampMillis",
+      "doc": "The event timestamp field as epoch at UTC in milli seconds."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeWindowSize",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "name": "CalendarInterval",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "SECOND",
+                  "MINUTE",
+                  "HOUR",
+                  "DAY",
+                  "WEEK",
+                  "MONTH",
+                  "QUARTER",
+                  "YEAR"
+                ]
+              },
+              "name": "unit",
+              "doc": "Interval unit such as minute/hour/day etc."
+            },
+            {
+              "type": "int",
+              "name": "multiple",
+              "default": 1,
+              "doc": "How many units. Defaults to 1."
+            }
+          ],
+          "doc": "Defines the size of a time window."
+        }
+      ],
+      "name": "eventGranularity",
+      "default": null,
+      "doc": "Granularity of the event if applicable"
+    },
+    {
+      "type": [
+        {
+          "type": "record",
+          "name": "PartitionSpec",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": "string",
+              "name": "partition",
+              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "TimeWindow",
+                  "namespace": "com.linkedin.pegasus2avro.timeseries",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "startTimeMillis",
+                      "doc": "Start time as epoch at UTC."
+                    },
+                    {
+                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
+                      "name": "length",
+                      "doc": "The length of the window."
+                    }
+                  ]
+                }
+              ],
+              "name": "timePartition",
+              "default": null,
+              "doc": "Time window of the partition, if we are able to extract it from the partition key."
+            },
+            {
+              "deprecated": true,
+              "type": {
+                "type": "enum",
+                "name": "PartitionType",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "FULL_TABLE",
+                  "QUERY",
+                  "PARTITION"
+                ]
+              },
+              "name": "type",
+              "default": "PARTITION",
+              "doc": "Unused!"
+            }
+          ],
+          "doc": "A reference to a specific partition in a dataset."
+        },
+        "null"
+      ],
+      "name": "partitionSpec",
+      "default": {
+        "partition": "FULL_TABLE_SNAPSHOT",
+        "type": "FULL_TABLE",
+        "timePartition": null
+      },
+      "doc": "The optional partition specification."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "messageId",
+      "default": null,
+      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "int"
+      ],
+      "name": "uniqueUserCount",
+      "default": null,
+      "doc": "Unique user count"
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "int"
+      ],
+      "name": "totalSqlQueries",
+      "default": null,
+      "doc": "Total SQL query count"
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "topSqlQueries",
+      "default": null,
+      "doc": "Frequent SQL queries; mostly makes sense for datasets in SQL databases"
+    },
+    {
+      "TimeseriesFieldCollection": {
+        "key": "user"
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "DatasetUserUsageCounts",
+            "namespace": "com.linkedin.pegasus2avro.dataset",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "user",
+                "doc": "The unique id of the user.",
+                "Urn": "Urn"
+              },
+              {
+                "TimeseriesField": {},
+                "type": "int",
+                "name": "count",
+                "doc": "Number of times the dataset has been used by the user."
+              },
+              {
+                "TimeseriesField": {},
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "userEmail",
+                "default": null,
+                "doc": "If user_email is set, we attempt to resolve the user's urn upon ingest"
+              }
+            ],
+            "doc": "Records a single user's usage counts for a given resource"
+          }
+        }
+      ],
+      "name": "userCounts",
+      "default": null,
+      "doc": "Users within this bucket, with frequency counts"
+    },
+    {
+      "TimeseriesFieldCollection": {
+        "key": "fieldPath"
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "DatasetFieldUsageCounts",
+            "namespace": "com.linkedin.pegasus2avro.dataset",
+            "fields": [
+              {
+                "type": "string",
+                "name": "fieldPath",
+                "doc": "The name of the field."
+              },
+              {
+                "TimeseriesField": {},
+                "type": "int",
+                "name": "count",
+                "doc": "Number of times the field has been used."
+              }
+            ],
+            "doc": "Records field-level usage counts for a given dataset"
+          }
+        }
+      ],
+      "name": "fieldCounts",
+      "default": null,
+      "doc": "Field-level usage stats"
+    }
+  ],
+  "doc": "Stats corresponding to dataset's usage."
+}

acryl_datahub_cloud/metadata/schemas/Deprecation.avsc

@@ -0,0 +1,57 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "deprecation"
+  },
+  "name": "Deprecation",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN",
+        "weightsPerFieldValue": {
+          "true": 0.5
+        }
+      },
+      "type": "boolean",
+      "name": "deprecated",
+      "doc": "Whether the entity is deprecated."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "decommissionTime",
+      "default": null,
+      "doc": "The time user plan to decommission this entity."
+    },
+    {
+      "type": "string",
+      "name": "note",
+      "doc": "Additional information about the entity deprecation plan, such as the wiki, doc, RB."
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "actor",
+      "doc": "The user URN which will be credited for modifying this deprecation content.",
+      "Urn": "Urn"
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "replacement",
+      "default": null,
+      "Urn": "Urn"
+    }
+  ],
+  "doc": "Deprecation status of an entity"
+}

acryl_datahub_cloud/metadata/schemas/DisplayProperties.avsc

@@ -0,0 +1,62 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "displayProperties"
+  },
+  "name": "DisplayProperties",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "colorHex",
+      "default": null,
+      "doc": "The color associated with the entity in Hex. For example #FFFFFF."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "IconProperties",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "symbolDocs": {
+                  "MATERIAL": "Material UI"
+                },
+                "name": "IconLibrary",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "symbols": [
+                  "MATERIAL"
+                ],
+                "doc": "Enum of possible icon sources"
+              },
+              "name": "iconLibrary",
+              "doc": "The source of the icon: e.g. Antd, Material, etc"
+            },
+            {
+              "type": "string",
+              "name": "name",
+              "doc": "The name of the icon"
+            },
+            {
+              "type": "string",
+              "name": "style",
+              "doc": "Any modifier for the icon, this will be library-specific, e.g. filled/outlined, etc"
+            }
+          ],
+          "doc": "Properties describing an icon associated with an entity"
+        }
+      ],
+      "name": "icon",
+      "default": null,
+      "doc": "The icon associated with the entity"
+    }
+  ],
+  "doc": "Properties related to how the entity is displayed in the Datahub UI"
+}