acryl-datahub-cloud 0.3.8rc3__py3-none-any.whl → 0.3.8rc6__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc

@@ -1,135 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "datasetKey",
-    "keyForEntity": "dataset",
-    "entityCategory": "core",
-    "entityAspects": [
-      "viewProperties",
-      "subTypes",
-      "datasetProfile",
-      "datasetUsageStatistics",
-      "operation",
-      "domains",
-      "proposals",
-      "schemaProposals",
-      "schemaMetadata",
-      "status",
-      "container",
-      "deprecation",
-      "usageFeatures",
-      "storageFeatures",
-      "lineageFeatures",
-      "testResults",
-      "siblings",
-      "embed",
-      "incidentsSummary",
-      "inferredNeighbors",
-      "inferredMetadata",
-      "schemaFieldsInferredMetadata",
-      "schemaFieldsInferredNeighbors",
-      "assertionsSummary",
-      "datasetProperties",
-      "editableDatasetProperties",
-      "datasetDeprecation",
-      "datasetUpstreamLineage",
-      "upstreamLineage",
-      "institutionalMemory",
-      "ownership",
-      "editableSchemaMetadata",
-      "globalTags",
-      "glossaryTerms",
-      "browsePaths",
-      "dataPlatformInstance",
-      "browsePathsV2",
-      "anomaliesSummary",
-      "access",
-      "structuredProperties",
-      "forms",
-      "partitionsSummary",
-      "share",
-      "origin",
-      "documentation",
-      "entityInferenceMetadata",
-      "versionProperties"
-    ],
-    "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
-  },
-  "name": "DatasetKey",
-  "namespace": "com.linkedin.pegasus2avro.metadata.key",
-  "fields": [
-    {
-      "Searchable": {
-        "enableAutocomplete": true,
-        "fieldType": "URN"
-      },
-      "java": {
-        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-      },
-      "type": "string",
-      "name": "platform",
-      "doc": "Data platform urn associated with the dataset",
-      "Urn": "Urn"
-    },
-    {
-      "Searchable": {
-        "boostScore": 10.0,
-        "enableAutocomplete": true,
-        "fieldName": "id",
-        "fieldType": "WORD_GRAM"
-      },
-      "type": "string",
-      "name": "name",
-      "doc": "Unique guid for dataset"
-    },
-    {
-      "Searchable": {
-        "addToFilters": true,
-        "fieldType": "TEXT_PARTIAL",
-        "filterNameOverride": "Environment",
-        "queryByDefault": false
-      },
-      "type": {
-        "type": "enum",
-        "symbolDocs": {
-          "CORP": "Designates corporation fabrics",
-          "DEV": "Designates development fabrics",
-          "EI": "Designates early-integration fabrics",
-          "NON_PROD": "Designates non-production fabrics",
-          "PRD": "Alternative Prod spelling",
-          "PRE": "Designates pre-production fabrics",
-          "PROD": "Designates production fabrics",
-          "QA": "Designates quality assurance fabrics",
-          "RVW": "Designates review fabrics",
-          "SANDBOX": "Designates sandbox fabrics",
-          "STG": "Designates staging fabrics",
-          "TEST": "Designates testing fabrics",
-          "TST": "Alternative Test spelling",
-          "UAT": "Designates user acceptance testing fabrics"
-        },
-        "name": "FabricType",
-        "namespace": "com.linkedin.pegasus2avro.common",
-        "symbols": [
-          "DEV",
-          "TEST",
-          "QA",
-          "UAT",
-          "EI",
-          "PRE",
-          "STG",
-          "NON_PROD",
-          "PROD",
-          "CORP",
-          "RVW",
-          "PRD",
-          "TST",
-          "SANDBOX"
-        ],
-        "doc": "Fabric group type"
-      },
-      "name": "origin",
-      "doc": "Fabric type where dataset belongs to or where it was generated."
-    }
-  ],
-  "doc": "Key for a Dataset"
-}

acryl_datahub_cloud/metadata/schemas/DatasetProfile.avsc

@@ -1,539 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "datasetProfile",
-    "type": "timeseries"
-  },
-  "name": "DatasetProfile",
-  "namespace": "com.linkedin.pegasus2avro.dataset",
-  "fields": [
-    {
-      "type": "long",
-      "name": "timestampMillis",
-      "doc": "The event timestamp field as epoch at UTC in milli seconds."
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "record",
-          "name": "TimeWindowSize",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "type": {
-                "type": "enum",
-                "name": "CalendarInterval",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "SECOND",
-                  "MINUTE",
-                  "HOUR",
-                  "DAY",
-                  "WEEK",
-                  "MONTH",
-                  "QUARTER",
-                  "YEAR"
-                ]
-              },
-              "name": "unit",
-              "doc": "Interval unit such as minute/hour/day etc."
-            },
-            {
-              "type": "int",
-              "name": "multiple",
-              "default": 1,
-              "doc": "How many units. Defaults to 1."
-            }
-          ],
-          "doc": "Defines the size of a time window."
-        }
-      ],
-      "name": "eventGranularity",
-      "default": null,
-      "doc": "Granularity of the event if applicable"
-    },
-    {
-      "type": [
-        {
-          "type": "record",
-          "name": "PartitionSpec",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "TimeseriesField": {},
-              "type": "string",
-              "name": "partition",
-              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "record",
-                  "name": "TimeWindow",
-                  "namespace": "com.linkedin.pegasus2avro.timeseries",
-                  "fields": [
-                    {
-                      "type": "long",
-                      "name": "startTimeMillis",
-                      "doc": "Start time as epoch at UTC."
-                    },
-                    {
-                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
-                      "name": "length",
-                      "doc": "The length of the window."
-                    }
-                  ]
-                }
-              ],
-              "name": "timePartition",
-              "default": null,
-              "doc": "Time window of the partition, if we are able to extract it from the partition key."
-            },
-            {
-              "deprecated": true,
-              "type": {
-                "type": "enum",
-                "name": "PartitionType",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "FULL_TABLE",
-                  "QUERY",
-                  "PARTITION"
-                ]
-              },
-              "name": "type",
-              "default": "PARTITION",
-              "doc": "Unused!"
-            }
-          ],
-          "doc": "A reference to a specific partition in a dataset."
-        },
-        "null"
-      ],
-      "name": "partitionSpec",
-      "default": {
-        "partition": "FULL_TABLE_SNAPSHOT",
-        "type": "FULL_TABLE",
-        "timePartition": null
-      },
-      "doc": "The optional partition specification."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "messageId",
-      "default": null,
-      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
-    },
-    {
-      "Searchable": {
-        "fieldType": "COUNT",
-        "hasValuesFieldName": "hasRowCount"
-      },
-      "TimeseriesField": {},
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "rowCount",
-      "default": null,
-      "doc": "The total number of rows"
-    },
-    {
-      "Searchable": {
-        "fieldType": "COUNT",
-        "hasValuesFieldName": "hasColumnCount"
-      },
-      "TimeseriesField": {},
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "columnCount",
-      "default": null,
-      "doc": "The total number of columns (or schema fields)"
-    },
-    {
-      "Searchable": {
-        "fieldType": "COUNT",
-        "hasValuesFieldName": "hasSizeInBytes"
-      },
-      "TimeseriesField": {},
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "sizeInBytes",
-      "default": null,
-      "doc": "Storage size in bytes"
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "array",
-          "items": {
-            "type": "record",
-            "name": "DatasetFieldProfile",
-            "namespace": "com.linkedin.pegasus2avro.dataset",
-            "fields": [
-              {
-                "type": [
-                  "null",
-                  "long"
-                ],
-                "name": "uniqueCount",
-                "default": null,
-                "doc": "The count of unique values for the field across the dataset.\nUseful for understanding the diversity of field values."
-              },
-              {
-                "type": [
-                  "null",
-                  "float"
-                ],
-                "name": "uniqueProportion",
-                "default": null,
-                "doc": "The proportion of unique values over the total number of non-null entries.\nHelps in assessing the uniqueness of the data in the field."
-              },
-              {
-                "type": [
-                  "null",
-                  "long"
-                ],
-                "name": "nullCount",
-                "default": null,
-                "doc": "The count of null or missing entries for the field.\nImportant for data quality assessments."
-              },
-              {
-                "type": [
-                  "null",
-                  "float"
-                ],
-                "name": "nullProportion",
-                "default": null,
-                "doc": "The proportion of null entries over the total number of entries in the field.\nUseful for identifying fields with a high rate of missing values."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "min",
-                "default": null,
-                "doc": "The minimum value in the field for string-representable data types.\nProvides a sense of the lower bound of field values."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "max",
-                "default": null,
-                "doc": "The maximum value in the field for string-representable data types.\nProvides a sense of the upper bound of field values."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "mean",
-                "default": null,
-                "doc": "The mean (average) value of the field, represented as a string.\nUseful for numerical fields to understand the central tendency."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "median",
-                "default": null,
-                "doc": "The median value of the field, represented as a string.\nUseful for understanding the middle value when data is sorted."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "stdev",
-                "default": null,
-                "doc": "The standard deviation of the field values, represented as a string.\nIndicates the amount of variation or dispersion of field values."
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "array",
-                    "items": {
-                      "type": "record",
-                      "name": "Quantile",
-                      "namespace": "com.linkedin.pegasus2avro.dataset",
-                      "fields": [
-                        {
-                          "type": "string",
-                          "name": "quantile"
-                        },
-                        {
-                          "type": "string",
-                          "name": "value"
-                        }
-                      ]
-                    }
-                  }
-                ],
-                "name": "quantiles",
-                "default": null,
-                "doc": "Only for numerical columns. Sorted list of quantile cutoffs for the field, in ascending order.\nHelps in assessing the diversity of the data in the field.\nExpect both the quantile and the value field to be string-ified floats.\nEx: quantile: \"0.25\", value: \"1.0\" means the 25th percentile of values falls at 1.0"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "record",
-                    "name": "Histogram",
-                    "namespace": "com.linkedin.pegasus2avro.dataset",
-                    "fields": [
-                      {
-                        "type": {
-                          "type": "array",
-                          "items": "string"
-                        },
-                        "name": "boundaries"
-                      },
-                      {
-                        "type": {
-                          "type": "array",
-                          "items": "float"
-                        },
-                        "name": "heights"
-                      }
-                    ]
-                  }
-                ],
-                "name": "histogram",
-                "default": null,
-                "doc": "Only for numerical columns. A histogram where a range of values in a numerical column\nare rolled up into bins, with the heights representing the number of values in that bin.\nDivided into k bin boundaries, with k-1 frequencies; first+last frequency values are tail weights\nCurrently unused in the product."
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "array",
-                    "items": {
-                      "type": "record",
-                      "name": "ValueFrequency",
-                      "namespace": "com.linkedin.pegasus2avro.dataset",
-                      "fields": [
-                        {
-                          "type": "string",
-                          "name": "value"
-                        },
-                        {
-                          "type": "long",
-                          "name": "frequency"
-                        }
-                      ]
-                    }
-                  }
-                ],
-                "name": "distinctValueFrequencies",
-                "default": null,
-                "doc": "Volume of each column value for a low-cardinality / categorical field.\nUseful for understanding the distribution of values in the field, and monitoring for changes\nin specific value volumes.\nFor numeric colums, the value will contain a strigified value."
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "array",
-                    "items": "string"
-                  }
-                ],
-                "name": "sampleValues",
-                "default": null,
-                "doc": "N randomly sampled values of the column. Useful for showing rough preview of the column."
-              },
-              {
-                "type": "string",
-                "name": "fieldPath",
-                "doc": "The standard path to the field"
-              }
-            ],
-            "doc": "Stats corresponding to fields in a dataset"
-          }
-        }
-      ],
-      "name": "fieldProfiles",
-      "default": null,
-      "doc": "Profiles for each column (or schema field). Notice that we are moving away from\nusing this model and moving toward DatasetFieldProfiles attached to the dataset\nfield urn directly. This field is kept for backward compatibility."
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "record",
-          "name": "DatasetMetricsSourceDetails",
-          "namespace": "com.linkedin.pegasus2avro.dataset",
-          "fields": [
-            {
-              "TimeseriesField": {},
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "INGESTION_SOURCE": "Operation was produced during a normal ingestion source run.",
-                  "MONITOR": "A running monitor produced the metadata."
-                },
-                "name": "MetadataProducerType",
-                "namespace": "com.linkedin.pegasus2avro.common",
-                "symbols": [
-                  "INGESTION_SOURCE",
-                  "MONITOR"
-                ],
-                "doc": "The type of the producer of the metadata, e.g. reflective of the process that produced the metadata."
-              },
-              "name": "producerType",
-              "doc": "The specific type of producer for the operation."
-            },
-            {
-              "TimeseriesField": {},
-              "java": {
-                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-              },
-              "type": [
-                "null",
-                "string"
-              ],
-              "name": "producerUrn",
-              "default": null,
-              "doc": "The producer urn for the operation. Only provided is producerType == MONITOR currently,\nwhere it will be set to the monitor urn that sampled the operation. Ideally, this also contains\nthe ingestion source urn if applicable.",
-              "Urn": "Urn"
-            },
-            {
-              "TimeseriesField": {},
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "ALL_ROWS_QUERY": "Issue a query to the table to obtain the metric or values across\nall rows.",
-                  "AUDIT_LOG": "Operation was identified by inspecting an audit log API",
-                  "CHANGED_ROWS_QUERY": "Issue a query to the table to obtain the metric values across\nonly the rows which have changed since the previous evaluation.",
-                  "FILE_METADATA": "Operation was identified by inspecting underlying file system.",
-                  "INFORMATION_SCHEMA": "Operation was identified by inspecting an information schema table, or other system metadata table."
-                },
-                "name": "DatasetMetricsOriginType",
-                "namespace": "com.linkedin.pegasus2avro.dataset",
-                "symbols": [
-                  "ALL_ROWS_QUERY",
-                  "CHANGED_ROWS_QUERY",
-                  "INFORMATION_SCHEMA",
-                  "AUDIT_LOG",
-                  "FILE_METADATA"
-                ]
-              },
-              "name": "originType",
-              "doc": "The specific origin of the metric with regards to the source platform."
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "record",
-                  "name": "FreshnessFieldSpec",
-                  "namespace": "com.linkedin.pegasus2avro.assertion",
-                  "fields": [
-                    {
-                      "type": "string",
-                      "name": "path",
-                      "doc": "The field path"
-                    },
-                    {
-                      "type": "string",
-                      "name": "type",
-                      "doc": "The DataHub standard schema field type."
-                    },
-                    {
-                      "type": "string",
-                      "name": "nativeType",
-                      "doc": "The native field type"
-                    },
-                    {
-                      "type": [
-                        "null",
-                        {
-                          "type": "enum",
-                          "symbolDocs": {
-                            "HIGH_WATERMARK": "Determine that a change has occurred by inspecting a field which should be tracked as the\n\"high watermark\" for the table. This should be an ascending number or date field.\n\nIf rows with this column have not been added since the previous check\nthen the Freshness Assertion will fail.",
-                            "LAST_MODIFIED": "Determine that a change has occurred by inspecting an last modified field which\nrepresents the last time at which a row was changed."
-                          },
-                          "name": "FreshnessFieldKind",
-                          "namespace": "com.linkedin.pegasus2avro.assertion",
-                          "symbols": [
-                            "LAST_MODIFIED",
-                            "HIGH_WATERMARK"
-                          ]
-                        }
-                      ],
-                      "name": "kind",
-                      "default": null,
-                      "doc": "The type of the field being used to verify the Freshness of the asset."
-                    }
-                  ],
-                  "doc": "Lightweight spec used for referencing a particular schema field that is used to compute\na freshness signal or operation.\nTODO: Since this is now leveraged across assertions & metrics / operations, we should consider moving this to a common package."
-                }
-              ],
-              "name": "changedRowsField",
-              "default": null,
-              "doc": "A field that can be used to filter for the fresh rows since the previous assertion\nevaluation. This should be present when sourceType = CHANGED_ROWS_QUERY.\n\nIf not provided, or the metricSourceType is not QUERY, then the assertion will be evaluated against the\nentire dataset."
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "record",
-                  "name": "DatasetFilter",
-                  "namespace": "com.linkedin.pegasus2avro.dataset",
-                  "fields": [
-                    {
-                      "type": {
-                        "type": "enum",
-                        "symbolDocs": {
-                          "SQL": "The partition is represented as a an opaque, raw SQL\nclause."
-                        },
-                        "name": "DatasetFilterType",
-                        "namespace": "com.linkedin.pegasus2avro.dataset",
-                        "symbols": [
-                          "SQL"
-                        ]
-                      },
-                      "name": "type",
-                      "doc": "How the partition will be represented in this model.\n\nIn the future, we'll likely add support for more structured\npredicates."
-                    },
-                    {
-                      "type": [
-                        "null",
-                        "string"
-                      ],
-                      "name": "sql",
-                      "default": null,
-                      "doc": "The raw where clause string which will be used for monitoring.\nRequired if the type is SQL."
-                    }
-                  ],
-                  "doc": "A definition of filters that should be used when\nquerying an external Dataset or Table.\n\nNote that this models should NOT be used for working with\nsearch / filter on DataHub Platform itself."
-                }
-              ],
-              "name": "filter",
-              "default": null,
-              "doc": "Any additional filters that were applied in a query to retrieve the metric.\nOnly possible to be present if originType is ALL_ROWS_QUERY or CHANGED_ROWS_QUERY."
-            }
-          ],
-          "doc": "Additional details about the origins of a captured source-platform Metric or Metric Set."
-        }
-      ],
-      "name": "source",
-      "default": null,
-      "doc": "Rich details about the source of the metrics provided in this profile."
-    }
-  ],
-  "doc": "Metrics corresponding to datasets.\n\nNotice that this aspect may be partially completed so the user\nshould filter for existence of the stats they need when querying."
-}