acryl-datahub-cloud 0.3.8rc3__py3-none-any.whl → 0.3.8rc6__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DatahubIngestionCheckpoint.avsc

@@ -1,188 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "datahubIngestionCheckpoint",
-    "type": "timeseries"
-  },
-  "name": "DatahubIngestionCheckpoint",
-  "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
-  "fields": [
-    {
-      "type": "long",
-      "name": "timestampMillis",
-      "doc": "The event timestamp field as epoch at UTC in milli seconds."
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "record",
-          "name": "TimeWindowSize",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "type": {
-                "type": "enum",
-                "name": "CalendarInterval",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "SECOND",
-                  "MINUTE",
-                  "HOUR",
-                  "DAY",
-                  "WEEK",
-                  "MONTH",
-                  "QUARTER",
-                  "YEAR"
-                ]
-              },
-              "name": "unit",
-              "doc": "Interval unit such as minute/hour/day etc."
-            },
-            {
-              "type": "int",
-              "name": "multiple",
-              "default": 1,
-              "doc": "How many units. Defaults to 1."
-            }
-          ],
-          "doc": "Defines the size of a time window."
-        }
-      ],
-      "name": "eventGranularity",
-      "default": null,
-      "doc": "Granularity of the event if applicable"
-    },
-    {
-      "type": [
-        {
-          "type": "record",
-          "name": "PartitionSpec",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "TimeseriesField": {},
-              "type": "string",
-              "name": "partition",
-              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "record",
-                  "name": "TimeWindow",
-                  "namespace": "com.linkedin.pegasus2avro.timeseries",
-                  "fields": [
-                    {
-                      "type": "long",
-                      "name": "startTimeMillis",
-                      "doc": "Start time as epoch at UTC."
-                    },
-                    {
-                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
-                      "name": "length",
-                      "doc": "The length of the window."
-                    }
-                  ]
-                }
-              ],
-              "name": "timePartition",
-              "default": null,
-              "doc": "Time window of the partition, if we are able to extract it from the partition key."
-            },
-            {
-              "deprecated": true,
-              "type": {
-                "type": "enum",
-                "name": "PartitionType",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "FULL_TABLE",
-                  "QUERY",
-                  "PARTITION"
-                ]
-              },
-              "name": "type",
-              "default": "PARTITION",
-              "doc": "Unused!"
-            }
-          ],
-          "doc": "A reference to a specific partition in a dataset."
-        },
-        "null"
-      ],
-      "name": "partitionSpec",
-      "default": {
-        "partition": "FULL_TABLE_SNAPSHOT",
-        "type": "FULL_TABLE",
-        "timePartition": null
-      },
-      "doc": "The optional partition specification."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "messageId",
-      "default": null,
-      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "pipelineName",
-      "doc": "The name of the pipeline that ran ingestion, a stable unique user provided identifier.\n e.g. my_snowflake1-to-datahub."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "platformInstanceId",
-      "doc": "The id of the instance against which the ingestion pipeline ran.\ne.g.: Bigquery project ids, MySQL hostnames etc."
-    },
-    {
-      "type": "string",
-      "name": "config",
-      "doc": "Json-encoded string representation of the non-secret members of the config ."
-    },
-    {
-      "type": {
-        "type": "record",
-        "name": "IngestionCheckpointState",
-        "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
-        "fields": [
-          {
-            "type": "string",
-            "name": "formatVersion",
-            "doc": "The version of the state format."
-          },
-          {
-            "type": "string",
-            "name": "serde",
-            "doc": "The serialization/deserialization protocol."
-          },
-          {
-            "type": [
-              "null",
-              "bytes"
-            ],
-            "name": "payload",
-            "default": null,
-            "doc": "Opaque blob of the state representation."
-          }
-        ],
-        "doc": "The checkpoint state object of a datahub ingestion run for a given job."
-      },
-      "name": "state",
-      "doc": "Opaque blob of the state representation."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "runId",
-      "doc": "The run identifier of this job."
-    }
-  ],
-  "doc": "Checkpoint of a datahub ingestion run for a given job."
-}

acryl_datahub_cloud/metadata/schemas/DatahubIngestionRunSummary.avsc

@@ -1,365 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "datahubIngestionRunSummary",
-    "type": "timeseries"
-  },
-  "name": "DatahubIngestionRunSummary",
-  "namespace": "com.linkedin.pegasus2avro.datajob.datahub",
-  "fields": [
-    {
-      "type": "long",
-      "name": "timestampMillis",
-      "doc": "The event timestamp field as epoch at UTC in milli seconds."
-    },
-    {
-      "type": [
-        "null",
-        {
-          "type": "record",
-          "name": "TimeWindowSize",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "type": {
-                "type": "enum",
-                "name": "CalendarInterval",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "SECOND",
-                  "MINUTE",
-                  "HOUR",
-                  "DAY",
-                  "WEEK",
-                  "MONTH",
-                  "QUARTER",
-                  "YEAR"
-                ]
-              },
-              "name": "unit",
-              "doc": "Interval unit such as minute/hour/day etc."
-            },
-            {
-              "type": "int",
-              "name": "multiple",
-              "default": 1,
-              "doc": "How many units. Defaults to 1."
-            }
-          ],
-          "doc": "Defines the size of a time window."
-        }
-      ],
-      "name": "eventGranularity",
-      "default": null,
-      "doc": "Granularity of the event if applicable"
-    },
-    {
-      "type": [
-        {
-          "type": "record",
-          "name": "PartitionSpec",
-          "namespace": "com.linkedin.pegasus2avro.timeseries",
-          "fields": [
-            {
-              "TimeseriesField": {},
-              "type": "string",
-              "name": "partition",
-              "doc": "A unique id / value for the partition for which statistics were collected,\ngenerated by applying the key definition to a given row."
-            },
-            {
-              "type": [
-                "null",
-                {
-                  "type": "record",
-                  "name": "TimeWindow",
-                  "namespace": "com.linkedin.pegasus2avro.timeseries",
-                  "fields": [
-                    {
-                      "type": "long",
-                      "name": "startTimeMillis",
-                      "doc": "Start time as epoch at UTC."
-                    },
-                    {
-                      "type": "com.linkedin.pegasus2avro.timeseries.TimeWindowSize",
-                      "name": "length",
-                      "doc": "The length of the window."
-                    }
-                  ]
-                }
-              ],
-              "name": "timePartition",
-              "default": null,
-              "doc": "Time window of the partition, if we are able to extract it from the partition key."
-            },
-            {
-              "deprecated": true,
-              "type": {
-                "type": "enum",
-                "name": "PartitionType",
-                "namespace": "com.linkedin.pegasus2avro.timeseries",
-                "symbols": [
-                  "FULL_TABLE",
-                  "QUERY",
-                  "PARTITION"
-                ]
-              },
-              "name": "type",
-              "default": "PARTITION",
-              "doc": "Unused!"
-            }
-          ],
-          "doc": "A reference to a specific partition in a dataset."
-        },
-        "null"
-      ],
-      "name": "partitionSpec",
-      "default": {
-        "partition": "FULL_TABLE_SNAPSHOT",
-        "type": "FULL_TABLE",
-        "timePartition": null
-      },
-      "doc": "The optional partition specification."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "messageId",
-      "default": null,
-      "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "pipelineName",
-      "doc": "The name of the pipeline that ran ingestion, a stable unique user provided identifier.\n e.g. my_snowflake1-to-datahub."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "platformInstanceId",
-      "doc": "The id of the instance against which the ingestion pipeline ran.\ne.g.: Bigquery project ids, MySQL hostnames etc."
-    },
-    {
-      "TimeseriesField": {},
-      "type": "string",
-      "name": "runId",
-      "doc": "The runId for this pipeline instance."
-    },
-    {
-      "TimeseriesField": {},
-      "type": {
-        "type": "enum",
-        "symbolDocs": {
-          "COMPLETED": "Jobs with successful completion.",
-          "FAILED": "Jobs that have failed.",
-          "IN_PROGRESS": "Jobs currently running.",
-          "SKIPPED": "Jobs that have been skipped.",
-          "STARTING": "Jobs being initialized.",
-          "STOPPED": "Jobs that have stopped.",
-          "STOPPING": "Jobs being stopped.",
-          "UNKNOWN": "Jobs with unknown status (either unmappable or unavailable)"
-        },
-        "name": "JobStatus",
-        "namespace": "com.linkedin.pegasus2avro.datajob",
-        "symbols": [
-          "STARTING",
-          "IN_PROGRESS",
-          "STOPPING",
-          "STOPPED",
-          "COMPLETED",
-          "FAILED",
-          "UNKNOWN",
-          "SKIPPED"
-        ],
-        "doc": "Job statuses"
-      },
-      "name": "runStatus",
-      "doc": "Run Status - Succeeded/Skipped/Failed etc."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numWorkUnitsCommitted",
-      "default": null,
-      "doc": "The number of workunits written to sink."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numWorkUnitsCreated",
-      "default": null,
-      "doc": "The number of workunits that are produced."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numEvents",
-      "default": null,
-      "doc": "The number of events produced (MCE + MCP)."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numEntities",
-      "default": null,
-      "doc": "The total number of entities produced (unique entity urns)."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numAspects",
-      "default": null,
-      "doc": "The total number of aspects produced across all entities."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numSourceAPICalls",
-      "default": null,
-      "doc": "Total number of source API calls."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "totalLatencySourceAPICalls",
-      "default": null,
-      "doc": "Total latency across all source API calls."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numSinkAPICalls",
-      "default": null,
-      "doc": "Total number of sink API calls."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "totalLatencySinkAPICalls",
-      "default": null,
-      "doc": "Total latency across all sink API calls."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numWarnings",
-      "default": null,
-      "doc": "Number of warnings generated."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numErrors",
-      "default": null,
-      "doc": "Number of errors generated."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "numEntitiesSkipped",
-      "default": null,
-      "doc": "Number of entities skipped."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "config",
-      "default": null,
-      "doc": "The non-sensitive key-value pairs of the yaml config used as json string."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "custom_summary",
-      "default": null,
-      "doc": "Custom value."
-    },
-    {
-      "TimeseriesField": {},
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "softwareVersion",
-      "default": null,
-      "doc": "The software version of this ingestion."
-    },
-    {
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "systemHostName",
-      "default": null,
-      "doc": "The hostname the ingestion pipeline ran on."
-    },
-    {
-      "TimeseriesField": {},
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "operatingSystemName",
-      "default": null,
-      "doc": "The os the ingestion pipeline ran on."
-    },
-    {
-      "type": [
-        "null",
-        "int"
-      ],
-      "name": "numProcessors",
-      "default": null,
-      "doc": "The number of processors on the host the ingestion pipeline ran on."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "totalMemory",
-      "default": null,
-      "doc": "The total amount of memory on the host the ingestion pipeline ran on."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "availableMemory",
-      "default": null,
-      "doc": "The available memory on the host the ingestion pipeline ran on."
-    }
-  ],
-  "doc": "Summary of a datahub ingestion run for a given platform."
-}

acryl_datahub_cloud/metadata/schemas/DatasetDeprecation.avsc

@@ -1,50 +0,0 @@
-{
-  "type": "record",
-  "Aspect": {
-    "name": "datasetDeprecation"
-  },
-  "Deprecated": true,
-  "name": "DatasetDeprecation",
-  "namespace": "com.linkedin.pegasus2avro.dataset",
-  "fields": [
-    {
-      "Searchable": {
-        "fieldType": "BOOLEAN",
-        "weightsPerFieldValue": {
-          "true": 0.5
-        }
-      },
-      "type": "boolean",
-      "name": "deprecated",
-      "doc": "Whether the dataset is deprecated by owner."
-    },
-    {
-      "type": [
-        "null",
-        "long"
-      ],
-      "name": "decommissionTime",
-      "default": null,
-      "doc": "The time user plan to decommission this dataset."
-    },
-    {
-      "type": "string",
-      "name": "note",
-      "doc": "Additional information about the dataset deprecation plan, such as the wiki, doc, RB."
-    },
-    {
-      "java": {
-        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-      },
-      "type": [
-        "null",
-        "string"
-      ],
-      "name": "actor",
-      "default": null,
-      "doc": "The corpuser URN which will be credited for modifying this deprecation content.",
-      "Urn": "Urn"
-    }
-  ],
-  "doc": "Dataset deprecation status\nDeprecated! This aspect is deprecated in favor of the more-general-purpose 'Deprecation' aspect."
-}