acryl-datahub-cloud 0.3.8rc3__py3-none-any.whl → 0.3.8rc6__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/UpstreamLineage.avsc

@@ -1,286 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "upstreamLineage"
-  },
-  "name": "UpstreamLineage",
-  "namespace": "com.linkedin.pegasus2avro.dataset",
-  "fields": [
-    {
-      "type": {
-        "type": "array",
-        "items": {
-          "type": "record",
-          "name": "Upstream",
-          "namespace": "com.linkedin.pegasus2avro.dataset",
-          "fields": [
-            {
-              "type": {
-                "type": "record",
-                "name": "AuditStamp",
-                "namespace": "com.linkedin.pegasus2avro.common",
-                "fields": [
-                  {
-                    "type": "long",
-                    "name": "time",
-                    "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
-                  },
-                  {
-                    "java": {
-                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                    },
-                    "type": "string",
-                    "name": "actor",
-                    "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
-                    "Urn": "Urn"
-                  },
-                  {
-                    "java": {
-                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                    },
-                    "type": [
-                      "null",
-                      "string"
-                    ],
-                    "name": "impersonator",
-                    "default": null,
-                    "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
-                    "Urn": "Urn"
-                  },
-                  {
-                    "type": [
-                      "null",
-                      "string"
-                    ],
-                    "name": "message",
-                    "default": null,
-                    "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
-                  }
-                ],
-                "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
-              },
-              "name": "auditStamp",
-              "default": {
-                "actor": "urn:li:corpuser:unknown",
-                "impersonator": null,
-                "time": 0,
-                "message": null
-              },
-              "doc": "Audit stamp containing who reported the lineage and when."
-            },
-            {
-              "type": [
-                "null",
-                "com.linkedin.pegasus2avro.common.AuditStamp"
-              ],
-              "name": "created",
-              "default": null,
-              "doc": "Audit stamp containing who created the lineage and when."
-            },
-            {
-              "Relationship": {
-                "createdActor": "upstreams/*/created/actor",
-                "createdOn": "upstreams/*/created/time",
-                "entityTypes": [
-                  "dataset"
-                ],
-                "isLineage": true,
-                "name": "DownstreamOf",
-                "properties": "upstreams/*/properties",
-                "updatedActor": "upstreams/*/auditStamp/actor",
-                "updatedOn": "upstreams/*/auditStamp/time",
-                "via": "upstreams/*/query"
-              },
-              "Searchable": {
-                "fieldName": "upstreams",
-                "fieldType": "URN",
-                "queryByDefault": false
-              },
-              "java": {
-                "class": "com.linkedin.pegasus2avro.common.urn.DatasetUrn"
-              },
-              "type": "string",
-              "name": "dataset",
-              "doc": "The upstream dataset the lineage points to",
-              "Urn": "DatasetUrn",
-              "entityTypes": [
-                "dataset"
-              ]
-            },
-            {
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "COPY": "Direct copy without modification",
-                  "TRANSFORMED": "Transformed data with modification (format or content change)",
-                  "VIEW": "Represents a view defined on the sources e.g. Hive view defined on underlying hive tables or a Hive table pointing to a HDFS dataset or DALI view defined on multiple sources"
-                },
-                "name": "DatasetLineageType",
-                "namespace": "com.linkedin.pegasus2avro.dataset",
-                "symbols": [
-                  "COPY",
-                  "TRANSFORMED",
-                  "VIEW"
-                ],
-                "doc": "The various types of supported dataset lineage"
-              },
-              "name": "type",
-              "doc": "The type of the lineage"
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "map",
-                  "values": "string"
-                }
-              ],
-              "name": "properties",
-              "default": null,
-              "doc": "A generic properties bag that allows us to store specific information on this graph edge."
-            },
-            {
-              "java": {
-                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-              },
-              "type": [
-                "null",
-                "string"
-              ],
-              "name": "query",
-              "default": null,
-              "doc": "If the lineage is generated by a query, a reference to the query",
-              "Urn": "Urn"
-            }
-          ],
-          "doc": "Upstream lineage information about a dataset including the source reporting the lineage"
-        }
-      },
-      "name": "upstreams",
-      "doc": "List of upstream dataset lineage information"
-    },
-    {
-      "Relationship": {
-        "/*/upstreams/*": {
-          "entityTypes": [
-            "dataset",
-            "schemaField"
-          ],
-          "name": "DownstreamOf"
-        }
-      },
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": {
-            "type": "record",
-            "name": "FineGrainedLineage",
-            "namespace": "com.linkedin.pegasus2avro.dataset",
-            "fields": [
-              {
-                "type": {
-                  "type": "enum",
-                  "symbolDocs": {
-                    "DATASET": " Indicates that this lineage is originating from upstream dataset(s)",
-                    "FIELD_SET": " Indicates that this lineage is originating from upstream field(s)",
-                    "NONE": " Indicates that there is no upstream lineage i.e. the downstream field is not a derived field"
-                  },
-                  "name": "FineGrainedLineageUpstreamType",
-                  "namespace": "com.linkedin.pegasus2avro.dataset",
-                  "symbols": [
-                    "FIELD_SET",
-                    "DATASET",
-                    "NONE"
-                  ],
-                  "doc": "The type of upstream entity in a fine-grained lineage"
-                },
-                "name": "upstreamType",
-                "doc": "The type of upstream entity"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "array",
-                    "items": "string"
-                  }
-                ],
-                "name": "upstreams",
-                "default": null,
-                "doc": "Upstream entities in the lineage",
-                "Urn": "Urn",
-                "urn_is_array": true
-              },
-              {
-                "type": {
-                  "type": "enum",
-                  "symbolDocs": {
-                    "FIELD": " Indicates that the lineage is for a single, specific, downstream field",
-                    "FIELD_SET": " Indicates that the lineage is for a set of downstream fields"
-                  },
-                  "name": "FineGrainedLineageDownstreamType",
-                  "namespace": "com.linkedin.pegasus2avro.dataset",
-                  "symbols": [
-                    "FIELD",
-                    "FIELD_SET"
-                  ],
-                  "doc": "The type of downstream field(s) in a fine-grained lineage"
-                },
-                "name": "downstreamType",
-                "doc": "The type of downstream field(s)"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "array",
-                    "items": "string"
-                  }
-                ],
-                "name": "downstreams",
-                "default": null,
-                "doc": "Downstream fields in the lineage",
-                "Urn": "Urn",
-                "urn_is_array": true
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "transformOperation",
-                "default": null,
-                "doc": "The transform operation applied to the upstream entities to produce the downstream field(s)"
-              },
-              {
-                "type": "float",
-                "name": "confidenceScore",
-                "default": 1.0,
-                "doc": "The confidence in this lineage between 0 (low confidence) and 1 (high confidence)"
-              },
-              {
-                "java": {
-                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                },
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "query",
-                "default": null,
-                "doc": "The query that was used to generate this lineage. \nPresent only if the lineage was generated from a detected query.",
-                "Urn": "Urn"
-              }
-            ],
-            "doc": "A fine-grained lineage from upstream fields/datasets to downstream field(s)"
-          }
-        }
-      ],
-      "name": "fineGrainedLineages",
-      "default": null,
-      "doc": " List of fine-grained lineage information, including field-level lineage"
-    }
-  ],
-  "doc": "Upstream lineage of a dataset"
-}

acryl_datahub_cloud/metadata/schemas/UsageAggregation.avsc

@@ -1,153 +0,0 @@
-{
-  "type": "record",
-  "deprecated": "Use DatasetUsageStatistics, or other UsageStatistics records, instead",
-  "name": "UsageAggregation",
-  "namespace": "com.linkedin.pegasus2avro.usage",
-  "fields": [
-    {
-      "type": "long",
-      "name": "bucket",
-      "doc": " Bucket start time in milliseconds "
-    },
-    {
-      "type": {
-        "type": "enum",
-        "name": "WindowDuration",
-        "namespace": "com.linkedin.pegasus2avro.common",
-        "symbols": [
-          "YEAR",
-          "MONTH",
-          "WEEK",
-          "DAY",
-          "HOUR"
-        ],
-        "doc": "Enum to define the length of a bucket when doing aggregations"
-      },
-      "name": "duration",
-      "doc": " Bucket duration "
-    },
-    {
-      "java": {
-        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-      },
-      "type": "string",
-      "name": "resource",
-      "doc": " Resource associated with these usage stats ",
-      "Urn": "Urn"
-    },
-    {
-      "type": {
-        "type": "record",
-        "name": "UsageAggregationMetrics",
-        "namespace": "com.linkedin.pegasus2avro.usage",
-        "fields": [
-          {
-            "type": [
-              "null",
-              "int"
-            ],
-            "name": "uniqueUserCount",
-            "default": null,
-            "doc": " Unique user count "
-          },
-          {
-            "type": [
-              "null",
-              {
-                "type": "array",
-                "items": {
-                  "type": "record",
-                  "name": "UserUsageCounts",
-                  "namespace": "com.linkedin.pegasus2avro.usage",
-                  "fields": [
-                    {
-                      "java": {
-                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                      },
-                      "type": [
-                        "null",
-                        "string"
-                      ],
-                      "name": "user",
-                      "default": null,
-                      "Urn": "Urn"
-                    },
-                    {
-                      "type": "int",
-                      "name": "count"
-                    },
-                    {
-                      "type": [
-                        "null",
-                        "string"
-                      ],
-                      "name": "userEmail",
-                      "default": null,
-                      "doc": " If user_email is set, we attempt to resolve the user's urn upon ingest "
-                    }
-                  ],
-                  "doc": " Records a single user's usage counts for a given resource "
-                }
-              }
-            ],
-            "name": "users",
-            "default": null,
-            "doc": " Users within this bucket, with frequency counts "
-          },
-          {
-            "type": [
-              "null",
-              "int"
-            ],
-            "name": "totalSqlQueries",
-            "default": null,
-            "doc": " Total SQL query count "
-          },
-          {
-            "type": [
-              "null",
-              {
-                "type": "array",
-                "items": "string"
-              }
-            ],
-            "name": "topSqlQueries",
-            "default": null,
-            "doc": " Frequent SQL queries; mostly makes sense for datasets in SQL databases "
-          },
-          {
-            "type": [
-              "null",
-              {
-                "type": "array",
-                "items": {
-                  "type": "record",
-                  "name": "FieldUsageCounts",
-                  "namespace": "com.linkedin.pegasus2avro.usage",
-                  "fields": [
-                    {
-                      "type": "string",
-                      "name": "fieldName"
-                    },
-                    {
-                      "type": "int",
-                      "name": "count"
-                    }
-                  ],
-                  "doc": " Records field-level usage counts for a given resource "
-                }
-              }
-            ],
-            "name": "fields",
-            "default": null,
-            "doc": " Field-level usage stats "
-          }
-        ],
-        "doc": "Metrics for usage data for a given resource and bucket. Not all fields\nmake sense for all buckets, so every field is optional."
-      },
-      "name": "metrics",
-      "doc": " Metrics associated with this bucket "
-    }
-  ],
-  "doc": "Usage data for a given resource, rolled up into a bucket."
-}

acryl_datahub_cloud/metadata/schemas/UsageFeatures.avsc

@@ -1,243 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "usageFeatures"
-  },
-  "name": "UsageFeatures",
-  "namespace": "com.linkedin.pegasus2avro.metadata.search.features",
-  "fields": [
-    {
-      "Searchable": {
-        "fieldName": "viewCountLast30DaysFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "viewCountLast30Days",
-      "default": null,
-      "doc": "Usage count in the last 30 days\nThis field is for Dashboard usage"
-    },
-    {
-      "Searchable": {
-        "fieldName": "viewCountFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "viewCountTotal",
-      "default": null,
-      "doc": "View count total \nThis field is for Dashboard usage"
-    },
-    {
-      "Searchable": {
-        "fieldName": "viewCountPercentileLast30Days",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "viewCountPercentileLast30Days",
-      "default": null,
-      "doc": "Percentile rank of the dashboard views based on the the last 30 days view count on a platform"
-    },
-    {
-      "Searchable": {
-        "fieldName": "usageCountLast30DaysFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "usageCountLast30Days",
-      "default": null,
-      "doc": "Deprecated - Use queryCountLast30Days instead. Usage count in the last 30 days"
-    },
-    {
-      "Searchable": {
-        "fieldName": "queryCountLast30DaysFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "queryCountLast30Days",
-      "default": null,
-      "doc": "Dataset Query count in the last 30 days"
-    },
-    {
-      "Searchable": {
-        "fieldName": "uniqueUserCountLast30DaysFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "uniqueUserCountLast30Days",
-      "default": null,
-      "doc": "Unique user count in the last 30 days"
-    },
-    {
-      "Searchable": {
-        "fieldName": "writeCountLast30DaysFeature",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "writeCountLast30Days",
-      "default": null,
-      "doc": "Write (insert/update) count in the last 30 days"
-    },
-    {
-      "Searchable": {
-        "fieldName": "queryCountPercentileLast30Days",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "queryCountPercentileLast30Days",
-      "default": null,
-      "doc": "Percentile rank of the dataset based on the the last 30 days usage count on a platform"
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "queryCountRankLast30Days",
-      "default": null,
-      "doc": "Overall rank of the dataset based on the the last 30 days usage count on a platform\nIt returns one plus the number of rows proceeding or equals to the current row in the ordering. "
-    },
-    {
-      "Searchable": {
-        "fieldName": "uniqueUserPercentileLast30Days",
-        "fieldType": "COUNT"
-      },
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "uniqueUserPercentileLast30Days",
-      "default": null,
-      "doc": "Percentile rank of the dataset based on the the last 30 days of select queries by unique users on a platform"
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "uniqueUserRankLast30Days",
-      "default": null,
-      "doc": "Overall rank of the dataset based on the the last 30 days of select queries by unique users on a platform\nIt returns one plus the number of rows proceeding or equals to the current row in the ordering. "
-    },
-    {
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "writeCountPercentileLast30Days",
-      "default": null,
-      "doc": "Percentile rank of the dataset based on the the last 30 days insert/update operation count on a platform"
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "writeCountRankLast30Days",
-      "default": null,
-      "doc": "Overall rank of the dataset based on the the last 30 days insert/update operation count on a platform\nIt returns one plus the number of rows proceeding or equals to the current row in the ordering. "
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": "string"
-        }
-      ],
-      "name": "topUsersLast30Days",
-      "default": null,
-      "doc": "Ranked list of the top 10 users in past 30 days. \nIt is ranked by highest usage descending.",
-      "Urn": "Urn",
-      "urn_is_array": true
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "sizeInBytesPercentile",
-      "default": null,
-      "doc": "Deprecated - use StorageFeatures.pdl instead. The percentile of size in bytes relative to other assets from the data plaform instance."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "sizeInBytesRank",
-      "default": null,
-      "doc": "Deprecated - use StorageFeatures.pdl instead. The percentile of size in bytes relative to other assets from the data plaform instance."
-    },
-    {
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "rowCountPercentile",
-      "default": null,
-      "doc": "Deprecated - use StorageFeatures.pdl instead. The percentile of row count relative to other assets from the data platform instance."
-    },
-    {
-      "type": [
-        "null",
-        "float"
-      ],
-      "name": "usageSearchScoreMultiplier",
-      "default": null,
-      "doc": "Multiplier for search score coming from usage"
-    },
-    {
-      "type": [
-        "null",
-        "float"
-      ],
-      "name": "usageFreshnessScoreMultiplier",
-      "default": null,
-      "doc": "Multiplier for search score coming from freshness"
-    },
-    {
-      "type": [
-        "null",
-        "float"
-      ],
-      "name": "customDatahubScoreMultiplier",
-      "default": null,
-      "doc": "Multiplier for search score coming from property based adjustment- e.g. URN"
-    },
-    {
-      "Searchable": {
-        "fieldName": "combinedSearchRankingMultiplier"
-      },
-      "type": [
-        "null",
-        "float"
-      ],
-      "name": "combinedSearchRankingMultiplier",
-      "default": null,
-      "doc": "Combined score of all multipliers"
-    }
-  ]
-}