acryl-datahub-cloud 0.3.8rc7__py3-none-any.whl → 0.3.8rc9__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceKey.avsc

@@ -0,0 +1,35 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataPlatformInstanceKey",
+    "keyForEntity": "dataPlatformInstance",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "dataPlatformInstanceProperties",
+      "ownership",
+      "globalTags",
+      "institutionalMemory",
+      "deprecation",
+      "status"
+    ]
+  },
+  "name": "DataPlatformInstanceKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "platform",
+      "doc": "Data platform urn associated with the instance",
+      "Urn": "Urn"
+    },
+    {
+      "type": "string",
+      "name": "instance",
+      "doc": "Unique instance id"
+    }
+  ],
+  "doc": "Key for a Dataset"
+}

acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceProperties.avsc

@@ -0,0 +1,72 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataPlatformInstanceProperties"
+  },
+  "name": "DataPlatformInstanceProperties",
+  "namespace": "com.linkedin.pegasus2avro.dataplatforminstance",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldNameAliases": [
+          "_entityName"
+        ],
+        "fieldType": "WORD_GRAM"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "name",
+      "default": null,
+      "doc": "Display name of the Data Platform Instance"
+    },
+    {
+      "Searchable": {
+        "fieldType": "TEXT",
+        "hasValuesFieldName": "hasDescription"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "Documentation of the Data Platform Instance"
+    }
+  ],
+  "doc": "Properties associated with a Data Platform Instance"
+}

acryl_datahub_cloud/metadata/schemas/DataPlatformKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataPlatformKey",
+    "keyForEntity": "dataPlatform",
+    "entityCategory": "core",
+    "entityAspects": [
+      "dataPlatformInfo"
+    ]
+  },
+  "name": "DataPlatformKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "platformName",
+      "doc": "Data platform name i.e. hdfs, oracle, espresso"
+    }
+  ],
+  "doc": "Key for a Data Platform"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInfo.avsc

@@ -0,0 +1,73 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInfo"
+  },
+  "name": "DataProcessInfo",
+  "namespace": "com.linkedin.pegasus2avro.dataprocess",
+  "fields": [
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataset"
+          ],
+          "isLineage": true,
+          "name": "Consumes"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "fieldName": "inputs",
+          "fieldType": "URN",
+          "numValuesFieldName": "numInputDatasets",
+          "queryByDefault": false
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "inputs",
+      "default": null,
+      "doc": "the inputs of the data process",
+      "Urn": "DatasetUrn",
+      "urn_is_array": true
+    },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataset"
+          ],
+          "isLineage": true,
+          "name": "Consumes"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "fieldName": "outputs",
+          "fieldType": "URN",
+          "numValuesFieldName": "numOutputDatasets",
+          "queryByDefault": false
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "outputs",
+      "default": null,
+      "doc": "the outputs of the data process",
+      "Urn": "DatasetUrn",
+      "urn_is_array": true
+    }
+  ],
+  "doc": "The inputs and outputs of this data process"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceInput.avsc

@@ -0,0 +1,38 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInstanceInput"
+  },
+  "name": "DataProcessInstanceInput",
+  "namespace": "com.linkedin.pegasus2avro.dataprocess",
+  "fields": [
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataset"
+          ],
+          "name": "Consumes"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "addToFilters": true,
+          "fieldName": "inputs",
+          "fieldType": "URN",
+          "numValuesFieldName": "numInputs",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "inputs",
+      "doc": "Input datasets to be consumed",
+      "Urn": "Urn",
+      "urn_is_array": true
+    }
+  ],
+  "doc": "Information about the inputs datasets of a Data process"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceKey.avsc

@@ -0,0 +1,29 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInstanceKey",
+    "keyForEntity": "dataProcessInstance",
+    "entityCategory": "_unset_",
+    "entityAspects": [
+      "dataProcessInstanceInput",
+      "dataProcessInstanceOutput",
+      "dataProcessInstanceProperties",
+      "dataProcessInstanceRelationships",
+      "dataProcessInstanceRunEvent",
+      "status",
+      "testResults",
+      "lineageFeatures"
+    ],
+    "entityDoc": "DataProcessInstance represents an instance of a datajob/jobflow run"
+  },
+  "name": "DataProcessInstanceKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the DataProcessInstance . Should be separate from the name used for displaying a DataProcessInstance."
+    }
+  ],
+  "doc": "Key for an Asset DataProcessInstance"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc

@@ -0,0 +1,38 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInstanceOutput"
+  },
+  "name": "DataProcessInstanceOutput",
+  "namespace": "com.linkedin.pegasus2avro.dataprocess",
+  "fields": [
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataset"
+          ],
+          "name": "Produces"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "addToFilters": true,
+          "fieldName": "outputs",
+          "fieldType": "URN",
+          "numValuesFieldName": "numOutputs",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "outputs",
+      "doc": "Output datasets to be produced",
+      "Urn": "Urn",
+      "urn_is_array": true
+    }
+  ],
+  "doc": "Information about the outputs of a Data process"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceProperties.avsc

@@ -0,0 +1,131 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInstanceProperties"
+  },
+  "name": "DataProcessInstanceProperties",
+  "namespace": "com.linkedin.pegasus2avro.dataprocess",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldType": "WORD_GRAM"
+      },
+      "type": "string",
+      "name": "name",
+      "doc": "Process name"
+    },
+    {
+      "Searchable": {
+        "addToFilters": true,
+        "fieldName": "processType",
+        "fieldType": "KEYWORD",
+        "filterNameOverride": "Process Type"
+      },
+      "type": [
+        "null",
+        {
+          "type": "enum",
+          "name": "DataProcessType",
+          "namespace": "com.linkedin.pegasus2avro.dataprocess",
+          "symbols": [
+            "BATCH_SCHEDULED",
+            "BATCH_AD_HOC",
+            "STREAMING"
+          ]
+        }
+      ],
+      "name": "type",
+      "default": null,
+      "doc": "Process type"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "created",
+          "fieldType": "COUNT",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "AuditStamp",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": "long",
+            "name": "time",
+            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "actor",
+            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "impersonator",
+            "default": null,
+            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "message",
+            "default": null,
+            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+          }
+        ],
+        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+      },
+      "name": "created",
+      "doc": "Audit stamp containing who reported the lineage and when"
+    }
+  ],
+  "doc": "The inputs and outputs of this data process"
+}

acryl_datahub_cloud/metadata/schemas/DataProcessInstanceRelationships.avsc

@@ -0,0 +1,99 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataProcessInstanceRelationships"
+  },
+  "name": "DataProcessInstanceRelationships",
+  "namespace": "com.linkedin.pegasus2avro.dataprocess",
+  "fields": [
+    {
+      "Relationship": {
+        "entityTypes": [
+          "dataJob",
+          "dataFlow",
+          "dataset"
+        ],
+        "name": "InstanceOf"
+      },
+      "Searchable": {
+        "/*": {
+          "fieldName": "parentTemplate",
+          "fieldType": "URN",
+          "queryByDefault": false
+        }
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "parentTemplate",
+      "default": null,
+      "doc": "The parent entity whose run instance it is",
+      "Urn": "Urn",
+      "entityTypes": [
+        "dataJob",
+        "dataFlow",
+        "dataset"
+      ]
+    },
+    {
+      "Relationship": {
+        "entityTypes": [
+          "dataProcessInstance"
+        ],
+        "name": "ChildOf"
+      },
+      "Searchable": {
+        "/*": {
+          "fieldName": "parentInstance",
+          "fieldType": "URN",
+          "queryByDefault": false
+        }
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "parentInstance",
+      "default": null,
+      "doc": "The parent DataProcessInstance where it belongs to.\nIf it is a Airflow Task then it should belong to an Airflow Dag run as well\nwhich will be another DataProcessInstance",
+      "Urn": "Urn",
+      "entityTypes": [
+        "dataProcessInstance"
+      ]
+    },
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "dataProcessInstance"
+          ],
+          "name": "UpstreamOf"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "fieldName": "upstream",
+          "fieldType": "URN",
+          "numValuesFieldName": "numUpstreams",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "upstreamInstances",
+      "doc": "Input DataProcessInstance which triggered this dataprocess instance",
+      "Urn": "Urn",
+      "urn_is_array": true
+    }
+  ],
+  "doc": "Information about Data process relationships"
+}