acryl-datahub-cloud 0.3.8rc8__py3-none-any.whl → 0.3.8rc10__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/Operation.avsc

@@ -0,0 +1,381 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "operation",
+    "type": "timeseries"
+  },
+  "name": "Operation",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": "long",
+      "name": "timestampMillis",
+      "doc": "The event timestamp field as epoch at UTC in milli seconds."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeWindowSize",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "name": "CalendarInterval",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "SECOND",
+                  "MINUTE",
+                  "HOUR",
+                  "DAY",
+                  "WEEK",
+                  "MONTH",
+                  "QUARTER",
+                  "YEAR"
+                ]
+              },
+              "name": "unit",
+              "doc": "Interval unit such as minute/hour/day etc."
+            },
+            {
+              "type": "int",
+              "name": "multiple",
+              "default": 1,
+              "doc": "How many units. Defaults to 1."
+            }
+          ],
+          "doc": "Defines the size of a time window."
+        }
+      ],
+      "name": "eventGranularity",
+      "default": null,
+      "doc": "Granularity of the event if applicable"
+    },
+    {
+      "type": [
+        {
+          "type": "record",
+          "name": "PartitionSpec",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": "string",
+              "name": "partition",
+              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "TimeWindow",
+                  "namespace": "com.linkedin.pegasus2avro.timeseries",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "startTimeMillis",
+                      "doc": "Start time as epoch at UTC."
+                    },
+                    {
+                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
+                      "name": "length",
+                      "doc": "The length of the window."
+                    }
+                  ]
+                }
+              ],
+              "name": "timePartition",
+              "default": null,
+              "doc": "Time window of the partition, if we are able to extract it from the partition key."
+            },
+            {
+              "deprecated": true,
+              "type": {
+                "type": "enum",
+                "name": "PartitionType",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "FULL_TABLE",
+                  "QUERY",
+                  "PARTITION"
+                ]
+              },
+              "name": "type",
+              "default": "PARTITION",
+              "doc": "Unused!"
+            }
+          ],
+          "doc": "A reference to a specific partition in a dataset."
+        },
+        "null"
+      ],
+      "name": "partitionSpec",
+      "default": {
+        "partition": "FULL_TABLE_SNAPSHOT",
+        "type": "FULL_TABLE",
+        "timePartition": null
+      },
+      "doc": "The optional partition specification."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "messageId",
+      "default": null,
+      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
+    },
+    {
+      "TimeseriesField": {},
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "actor",
+      "default": null,
+      "doc": "Actor who issued this operation.",
+      "Urn": "Urn"
+    },
+    {
+      "TimeseriesField": {},
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "ALTER": "Asset was altered",
+          "CREATE": "Asset was created",
+          "CUSTOM": "Custom asset operation",
+          "DELETE": "Rows were deleted",
+          "DROP": "Asset was dropped",
+          "INSERT": "Rows were inserted",
+          "UPDATE": "Rows were updated"
+        },
+        "name": "OperationType",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "symbols": [
+          "INSERT",
+          "UPDATE",
+          "DELETE",
+          "CREATE",
+          "ALTER",
+          "DROP",
+          "CUSTOM",
+          "UNKNOWN"
+        ],
+        "doc": "Enum to define the operation type when an entity changes."
+      },
+      "name": "operationType",
+      "doc": "Operation type of change."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "customOperationType",
+      "default": null,
+      "doc": "A custom type of operation. Required if operationType is CUSTOM."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numAffectedRows",
+      "default": null,
+      "doc": "How many rows were affected by this operation."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "affectedDatasets",
+      "default": null,
+      "doc": "Which other datasets were affected by this operation.",
+      "Urn": "Urn",
+      "urn_is_array": true
+    },
+    {
+      "TimeseriesField": {},
+      "deprecated": true,
+      "type": [
+        "null",
+        {
+          "type": "enum",
+          "symbolDocs": {
+            "DATA_PLATFORM": "Update was provided by consulting the data platform  itself.",
+            "DATA_PROCESS": "Provided by a Data Process"
+          },
+          "name": "OperationSourceType",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "symbols": [
+            "DATA_PROCESS",
+            "DATA_PLATFORM"
+          ],
+          "doc": "The source of an operation. Note that this may be superceded by sourceDetails > producerType."
+        }
+      ],
+      "name": "sourceType",
+      "default": null,
+      "doc": "Source Type - E.g. which process produced this operation. Note that this field will be superseded by the richer information\nprovided inside the \"source\" field (e.g source > producerType)"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "OperationSourceDetails",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": {
+                "type": "enum",
+                "symbolDocs": {
+                  "INGESTION_SOURCE": "Operation was produced during a normal ingestion source run.",
+                  "MONITOR": "A running monitor produced the metadata."
+                },
+                "name": "MetadataProducerType",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "symbols": [
+                  "INGESTION_SOURCE",
+                  "MONITOR"
+                ],
+                "doc": "The type of the producer of the metadata, e.g. reflective of the process that produced the metadata."
+              },
+              "name": "producerType",
+              "doc": "The specific type of producer for the operation."
+            },
+            {
+              "TimeseriesField": {},
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "producerUrn",
+              "default": null,
+              "doc": "The producer urn for the operation. Only provided is producerType == MONITOR currently,\nwhere it will be set to the monitor urn that sampled the operation. Ideally, this also contains\nthe ingestion source urn if applicable.",
+              "Urn": "Urn"
+            },
+            {
+              "TimeseriesField": {},
+              "type": {
+                "type": "enum",
+                "symbolDocs": {
+                  "AUDIT_LOG": "Operation was identified by inspecting an audit log API",
+                  "FIELD_VALUE": "Operation was identified by querying a particular field / column value.",
+                  "FILE_METADATA": "Operation was identified by inspecting underlying file system.",
+                  "INFORMATION_SCHEMA": "Operation was identified by inspecting an information schema table, or other system metadata table."
+                },
+                "name": "OperationSourceOriginType",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "symbols": [
+                  "FIELD_VALUE",
+                  "INFORMATION_SCHEMA",
+                  "AUDIT_LOG",
+                  "FILE_METADATA"
+                ]
+              },
+              "name": "originType",
+              "doc": "The specific origin of the operation with regards to the source platform."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "FreshnessFieldSpec",
+                  "namespace": "com.linkedin.pegasus2avro.assertion",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "path",
+                      "doc": "The field path"
+                    },
+                    {
+                      "type": "string",
+                      "name": "type",
+                      "doc": "The DataHub standard schema field type."
+                    },
+                    {
+                      "type": "string",
+                      "name": "nativeType",
+                      "doc": "The native field type"
+                    },
+                    {
+                      "type": [
+                        "null",
+                        {
+                          "type": "enum",
+                          "symbolDocs": {
+                            "HIGH_WATERMARK": "Determine that a change has occurred by inspecting a field which should be tracked as the\n\"high watermark\" for the table. This should be an ascending number or date field.\n\nIf rows with this column have not been added since the previous check\nthen the Freshness Assertion will fail.",
+                            "LAST_MODIFIED": "Determine that a change has occurred by inspecting an last modified field which\nrepresents the last time at which a row was changed."
+                          },
+                          "name": "FreshnessFieldKind",
+                          "namespace": "com.linkedin.pegasus2avro.assertion",
+                          "symbols": [
+                            "LAST_MODIFIED",
+                            "HIGH_WATERMARK"
+                          ]
+                        }
+                      ],
+                      "name": "kind",
+                      "default": null,
+                      "doc": "The type of the field being used to verify the Freshness of the asset."
+                    }
+                  ],
+                  "doc": "Lightweight spec used for referencing a particular schema field that is used to compute\na freshness signal or operation.\nTODO: Since this is now leveraged across assertions & metrics / operations, we should consider moving this to a common package."
+                }
+              ],
+              "name": "field",
+              "default": null,
+              "doc": "Information about the field / column. May be present when originType is FIELD_VALUE"
+            }
+          ],
+          "doc": "Additional details about the origins of a captured source-platform Operation."
+        }
+      ],
+      "name": "source",
+      "default": null,
+      "doc": "More detailed information about the source of the operation."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": "string"
+        }
+      ],
+      "name": "customProperties",
+      "default": null,
+      "doc": "Custom properties"
+    },
+    {
+      "Searchable": {
+        "fieldName": "lastOperationTime",
+        "fieldType": "DATETIME"
+      },
+      "TimeseriesField": {},
+      "type": "long",
+      "name": "lastUpdatedTimestamp",
+      "doc": "The time at which the operation occurred. Would be better named 'operationTime'"
+    }
+  ],
+  "doc": "Operational info for an entity."
+}

acryl_datahub_cloud/metadata/schemas/Origin.avsc

@@ -0,0 +1,157 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "origin"
+  },
+  "name": "Origin",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "EXTERNAL": "The entity is external to DataHub.",
+          "NATIVE": "The entity is native to DataHub."
+        },
+        "name": "OriginType",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "symbols": [
+          "NATIVE",
+          "EXTERNAL"
+        ],
+        "doc": "Enum to define where an entity originated from."
+      },
+      "name": "type",
+      "doc": "Where an entity originated from. Either NATIVE or EXTERNAL."
+    },
+    {
+      "deprecated": "Use sourceDetails instead",
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalType",
+      "default": null,
+      "doc": "Only populated if type is EXTERNAL. The externalType of the entity, such as the name of the identity provider."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "SourceDetails",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "source",
+                "doc": "Urn of the source of this asset. Could be an ingestion source (e.g. urn:li:dataHubIngestionSource:1234), an acryl\n    platform instance, etc.",
+                "Urn": "Urn"
+              },
+              {
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "platform",
+                "doc": "Urn of the platform this asset originated from ie. urn:li:dataPlatform:snowflake for snowflake ingestion\nor urn:li:dataPlatform:acryl for assets synced from another catalog instance.",
+                "Urn": "Urn"
+              },
+              {
+                "type": {
+                  "type": "record",
+                  "name": "AuditStamp",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "impersonator",
+                      "default": null,
+                      "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "message",
+                      "default": null,
+                      "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                    }
+                  ],
+                  "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                },
+                "name": "lastModified",
+                "doc": "Audit stamp for when this asset was synced"
+              },
+              {
+                "type": {
+                  "type": "enum",
+                  "symbolDocs": {
+                    "API": "Synced via the API (pushed from clients using the API directly)",
+                    "INGEST": "Synced via ingestion recipes",
+                    "OTHER": "Synced via some other mechanism (used as a placeholder for incubating\n      future mechanisms)",
+                    "SHARE": "Synced via metadata sharing with another Acryl instance"
+                  },
+                  "name": "SyncMechanism",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "symbols": [
+                    "INGEST",
+                    "SHARE",
+                    "API",
+                    "OTHER"
+                  ]
+                },
+                "name": "mechanism",
+                "doc": "The means by which this asset got synchronized"
+              },
+              {
+                "type": [
+                  {
+                    "type": "map",
+                    "values": "string"
+                  },
+                  "null"
+                ],
+                "name": "properties",
+                "default": {},
+                "doc": "Additional information related to the origin of this metadata"
+              }
+            ],
+            "doc": "Info about how this asset was brought into DataHub"
+          }
+        }
+      ],
+      "name": "sourceDetails",
+      "default": null,
+      "doc": "Details about the source(s) by which this asset was originated"
+    }
+  ],
+  "doc": "Carries information about where an entity originated from."
+}