acryl-datahub-cloud 0.3.8rc7__py3-none-any.whl → 0.3.8rc9__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DatahubIngestionCheckpoint.avsc

@@ -0,0 +1,188 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datahubIngestionCheckpoint",
+    "type": "timeseries"
+  },
+  "name": "DatahubIngestionCheckpoint",
+  "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
+  "fields": [
+    {
+      "type": "long",
+      "name": "timestampMillis",
+      "doc": "The event timestamp field as epoch at UTC in milli seconds."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeWindowSize",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "name": "CalendarInterval",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "SECOND",
+                  "MINUTE",
+                  "HOUR",
+                  "DAY",
+                  "WEEK",
+                  "MONTH",
+                  "QUARTER",
+                  "YEAR"
+                ]
+              },
+              "name": "unit",
+              "doc": "Interval unit such as minute/hour/day etc."
+            },
+            {
+              "type": "int",
+              "name": "multiple",
+              "default": 1,
+              "doc": "How many units. Defaults to 1."
+            }
+          ],
+          "doc": "Defines the size of a time window."
+        }
+      ],
+      "name": "eventGranularity",
+      "default": null,
+      "doc": "Granularity of the event if applicable"
+    },
+    {
+      "type": [
+        {
+          "type": "record",
+          "name": "PartitionSpec",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": "string",
+              "name": "partition",
+              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "TimeWindow",
+                  "namespace": "com.linkedin.pegasus2avro.timeseries",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "startTimeMillis",
+                      "doc": "Start time as epoch at UTC."
+                    },
+                    {
+                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
+                      "name": "length",
+                      "doc": "The length of the window."
+                    }
+                  ]
+                }
+              ],
+              "name": "timePartition",
+              "default": null,
+              "doc": "Time window of the partition, if we are able to extract it from the partition key."
+            },
+            {
+              "deprecated": true,
+              "type": {
+                "type": "enum",
+                "name": "PartitionType",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "FULL_TABLE",
+                  "QUERY",
+                  "PARTITION"
+                ]
+              },
+              "name": "type",
+              "default": "PARTITION",
+              "doc": "Unused!"
+            }
+          ],
+          "doc": "A reference to a specific partition in a dataset."
+        },
+        "null"
+      ],
+      "name": "partitionSpec",
+      "default": {
+        "partition": "FULL_TABLE_SNAPSHOT",
+        "type": "FULL_TABLE",
+        "timePartition": null
+      },
+      "doc": "The optional partition specification."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "messageId",
+      "default": null,
+      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "pipelineName",
+      "doc": "The name of the pipeline that ran ingestion, a stable unique user provided identifier.\n e.g. my_snowflake1-to-datahub."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "platformInstanceId",
+      "doc": "The id of the instance against which the ingestion pipeline ran.\ne.g.: Bigquery project ids, MySQL hostnames etc."
+    },
+    {
+      "type": "string",
+      "name": "config",
+      "doc": "Json-encoded string representation of the non-secret members of the config ."
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "IngestionCheckpointState",
+        "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
+        "fields": [
+          {
+            "type": "string",
+            "name": "formatVersion",
+            "doc": "The version of the state format."
+          },
+          {
+            "type": "string",
+            "name": "serde",
+            "doc": "The serialization/deserialization protocol."
+          },
+          {
+            "type": [
+              "null",
+              "bytes"
+            ],
+            "name": "payload",
+            "default": null,
+            "doc": "Opaque blob of the state representation."
+          }
+        ],
+        "doc": "The checkpoint state object of a datahub ingestion run for a given job."
+      },
+      "name": "state",
+      "doc": "Opaque blob of the state representation."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "runId",
+      "doc": "The run identifier of this job."
+    }
+  ],
+  "doc": "Checkpoint of a datahub ingestion run for a given job."
+}

acryl_datahub_cloud/metadata/schemas/DatahubIngestionRunSummary.avsc

@@ -0,0 +1,365 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datahubIngestionRunSummary",
+    "type": "timeseries"
+  },
+  "name": "DatahubIngestionRunSummary",
+  "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
+  "fields": [
+    {
+      "type": "long",
+      "name": "timestampMillis",
+      "doc": "The event timestamp field as epoch at UTC in milli seconds."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeWindowSize",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "name": "CalendarInterval",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "SECOND",
+                  "MINUTE",
+                  "HOUR",
+                  "DAY",
+                  "WEEK",
+                  "MONTH",
+                  "QUARTER",
+                  "YEAR"
+                ]
+              },
+              "name": "unit",
+              "doc": "Interval unit such as minute/hour/day etc."
+            },
+            {
+              "type": "int",
+              "name": "multiple",
+              "default": 1,
+              "doc": "How many units. Defaults to 1."
+            }
+          ],
+          "doc": "Defines the size of a time window."
+        }
+      ],
+      "name": "eventGranularity",
+      "default": null,
+      "doc": "Granularity of the event if applicable"
+    },
+    {
+      "type": [
+        {
+          "type": "record",
+          "name": "PartitionSpec",
+          "namespace": "com.linkedin.pegasus2avro.timeseries",
+          "fields": [
+            {
+              "TimeseriesField": {},
+              "type": "string",
+              "name": "partition",
+              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
+            },
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "TimeWindow",
+                  "namespace": "com.linkedin.pegasus2avro.timeseries",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "startTimeMillis",
+                      "doc": "Start time as epoch at UTC."
+                    },
+                    {
+                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
+                      "name": "length",
+                      "doc": "The length of the window."
+                    }
+                  ]
+                }
+              ],
+              "name": "timePartition",
+              "default": null,
+              "doc": "Time window of the partition, if we are able to extract it from the partition key."
+            },
+            {
+              "deprecated": true,
+              "type": {
+                "type": "enum",
+                "name": "PartitionType",
+                "namespace": "com.linkedin.pegasus2avro.timeseries",
+                "symbols": [
+                  "FULL_TABLE",
+                  "QUERY",
+                  "PARTITION"
+                ]
+              },
+              "name": "type",
+              "default": "PARTITION",
+              "doc": "Unused!"
+            }
+          ],
+          "doc": "A reference to a specific partition in a dataset."
+        },
+        "null"
+      ],
+      "name": "partitionSpec",
+      "default": {
+        "partition": "FULL_TABLE_SNAPSHOT",
+        "type": "FULL_TABLE",
+        "timePartition": null
+      },
+      "doc": "The optional partition specification."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "messageId",
+      "default": null,
+      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "pipelineName",
+      "doc": "The name of the pipeline that ran ingestion, a stable unique user provided identifier.\n e.g. my_snowflake1-to-datahub."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "platformInstanceId",
+      "doc": "The id of the instance against which the ingestion pipeline ran.\ne.g.: Bigquery project ids, MySQL hostnames etc."
+    },
+    {
+      "TimeseriesField": {},
+      "type": "string",
+      "name": "runId",
+      "doc": "The runId for this pipeline instance."
+    },
+    {
+      "TimeseriesField": {},
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "COMPLETED": "Jobs with successful completion.",
+          "FAILED": "Jobs that have failed.",
+          "IN_PROGRESS": "Jobs currently running.",
+          "SKIPPED": "Jobs that have been skipped.",
+          "STARTING": "Jobs being initialized.",
+          "STOPPED": "Jobs that have stopped.",
+          "STOPPING": "Jobs being stopped.",
+          "UNKNOWN": "Jobs with unknown status (either unmappable or unavailable)"
+        },
+        "name": "JobStatus",
+        "namespace": "com.linkedin.pegasus2avro.datajob",
+        "symbols": [
+          "STARTING",
+          "IN_PROGRESS",
+          "STOPPING",
+          "STOPPED",
+          "COMPLETED",
+          "FAILED",
+          "UNKNOWN",
+          "SKIPPED"
+        ],
+        "doc": "Job statuses"
+      },
+      "name": "runStatus",
+      "doc": "Run Status - Succeeded/Skipped/Failed etc."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numWorkUnitsCommitted",
+      "default": null,
+      "doc": "The number of workunits written to sink."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numWorkUnitsCreated",
+      "default": null,
+      "doc": "The number of workunits that are produced."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numEvents",
+      "default": null,
+      "doc": "The number of events produced (MCE + MCP)."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numEntities",
+      "default": null,
+      "doc": "The total number of entities produced (unique entity urns)."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numAspects",
+      "default": null,
+      "doc": "The total number of aspects produced across all entities."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numSourceAPICalls",
+      "default": null,
+      "doc": "Total number of source API calls."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "totalLatencySourceAPICalls",
+      "default": null,
+      "doc": "Total latency across all source API calls."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numSinkAPICalls",
+      "default": null,
+      "doc": "Total number of sink API calls."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "totalLatencySinkAPICalls",
+      "default": null,
+      "doc": "Total latency across all sink API calls."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numWarnings",
+      "default": null,
+      "doc": "Number of warnings generated."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numErrors",
+      "default": null,
+      "doc": "Number of errors generated."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "numEntitiesSkipped",
+      "default": null,
+      "doc": "Number of entities skipped."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "config",
+      "default": null,
+      "doc": "The non-sensitive key-value pairs of the yaml config used as json string."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "custom_summary",
+      "default": null,
+      "doc": "Custom value."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "softwareVersion",
+      "default": null,
+      "doc": "The software version of this ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "systemHostName",
+      "default": null,
+      "doc": "The hostname the ingestion pipeline ran on."
+    },
+    {
+      "TimeseriesField": {},
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "operatingSystemName",
+      "default": null,
+      "doc": "The os the ingestion pipeline ran on."
+    },
+    {
+      "type": [
+        "null",
+        "int"
+      ],
+      "name": "numProcessors",
+      "default": null,
+      "doc": "The number of processors on the host the ingestion pipeline ran on."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "totalMemory",
+      "default": null,
+      "doc": "The total amount of memory on the host the ingestion pipeline ran on."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "availableMemory",
+      "default": null,
+      "doc": "The available memory on the host the ingestion pipeline ran on."
+    }
+  ],
+  "doc": "Summary of a datahub ingestion run for a given platform."
+}

acryl_datahub_cloud/metadata/schemas/DatasetDeprecation.avsc

@@ -0,0 +1,50 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "datasetDeprecation"
+  },
+  "Deprecated": true,
+  "name": "DatasetDeprecation",
+  "namespace": "com.linkedin.pegasus2avro.dataset",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN",
+        "weightsPerFieldValue": {
+          "true": 0.5
+        }
+      },
+      "type": "boolean",
+      "name": "deprecated",
+      "doc": "Whether the dataset is deprecated by owner."
+    },
+    {
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "decommissionTime",
+      "default": null,
+      "doc": "The time user plan to decommission this dataset."
+    },
+    {
+      "type": "string",
+      "name": "note",
+      "doc": "Additional information about the dataset deprecation plan, such as the wiki, doc, RB."
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "actor",
+      "default": null,
+      "doc": "The corpuser URN which will be credited for modifying this deprecation content.",
+      "Urn": "Urn"
+    }
+  ],
+  "doc": "Dataset deprecation status\nDeprecated! This aspect is deprecated in favor of the more-general-purpose 'Deprecation' aspect."
+}