acryl-datahub-cloud 0.3.8rc7__py3-none-any.whl → 0.3.8rc9__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DataHubViewInfo.avsc

@@ -0,0 +1,265 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubViewInfo"
+  },
+  "name": "DataHubViewInfo",
+  "namespace": "com.linkedin.pegasus2avro.view",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "TEXT_PARTIAL"
+      },
+      "type": "string",
+      "name": "name",
+      "doc": "The name of the View"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "Description of the view"
+    },
+    {
+      "Searchable": {},
+      "type": {
+        "type": "enum",
+        "symbolDocs": {
+          "GLOBAL": "A global view, which all users can see and use.",
+          "PERSONAL": "A view private for a specific person."
+        },
+        "name": "DataHubViewType",
+        "namespace": "com.linkedin.pegasus2avro.view",
+        "symbols": [
+          "PERSONAL",
+          "GLOBAL"
+        ]
+      },
+      "name": "type",
+      "doc": "The type of View"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DataHubViewDefinition",
+        "namespace": "com.linkedin.pegasus2avro.view",
+        "fields": [
+          {
+            "type": {
+              "type": "array",
+              "items": "string"
+            },
+            "name": "entityTypes",
+            "doc": "The Entity Types in the scope of the View."
+          },
+          {
+            "type": {
+              "type": "record",
+              "name": "Filter",
+              "namespace": "com.linkedin.pegasus2avro.metadata.query.filter",
+              "fields": [
+                {
+                  "type": [
+                    "null",
+                    {
+                      "type": "array",
+                      "items": {
+                        "type": "record",
+                        "name": "ConjunctiveCriterion",
+                        "namespace": "com.linkedin.pegasus2avro.metadata.query.filter",
+                        "fields": [
+                          {
+                            "type": {
+                              "type": "array",
+                              "items": {
+                                "type": "record",
+                                "name": "Criterion",
+                                "namespace": "com.linkedin.pegasus2avro.metadata.query.filter",
+                                "fields": [
+                                  {
+                                    "type": "string",
+                                    "name": "field",
+                                    "doc": "The name of the field that the criterion refers to"
+                                  },
+                                  {
+                                    "type": "string",
+                                    "name": "value",
+                                    "doc": "The value of the intended field"
+                                  },
+                                  {
+                                    "type": {
+                                      "type": "array",
+                                      "items": "string"
+                                    },
+                                    "name": "values",
+                                    "default": [],
+                                    "doc": "Values. one of which the intended field should match\nNote, if values is set, the above \"value\" field will be ignored"
+                                  },
+                                  {
+                                    "type": {
+                                      "type": "enum",
+                                      "symbolDocs": {
+                                        "ANCESTORS_INCL": "Represent the relation: URN field matches any nested parent in addition to the given URN",
+                                        "BETWEEN": "Represent the relation within an inclusive range, e.g. 3 <= ownerCount <= 5. Note that the values in Criterion must have two entries for a lower and upper bound.",
+                                        "CONTAIN": "Represent the relation: String field contains value, e.g. name contains Profile",
+                                        "DESCENDANTS_INCL": "Represent the relation: URN field any nested children in addition to the given URN",
+                                        "END_WITH": "Represent the relation: String field ends with value, e.g. name ends with Event",
+                                        "EQUAL": "Represent the relation: field = value, e.g. platform = hdfs",
+                                        "EXISTS": "Represents the relation: field exists and is non-empty, e.g. owners is not null and != [] (empty)",
+                                        "GREATER_THAN": "Represent the relation greater than, e.g. ownerCount > 5",
+                                        "GREATER_THAN_OR_EQUAL_TO": "Represent the relation greater than or equal to, e.g. ownerCount >= 5",
+                                        "IEQUAL": "Represent the relation: field = value and support case insensitive values, e.g. platform = hdfs",
+                                        "IN": "Represent the relation: String field is one of the array values to, e.g. name in [\"Profile\", \"Event\"]",
+                                        "IS_NULL": "Represent the relation: field is null, e.g. platform is null",
+                                        "LESS_THAN": "Represent the relation less than, e.g. ownerCount < 3",
+                                        "LESS_THAN_OR_EQUAL_TO": "Represent the relation less than or equal to, e.g. ownerCount <= 3",
+                                        "RELATED_INCL": "Represent the relation: URN field matches any nested child or parent in addition to the given URN",
+                                        "START_WITH": "Represent the relation: String field starts with value, e.g. name starts with PageView"
+                                      },
+                                      "name": "Condition",
+                                      "namespace": "com.linkedin.pegasus2avro.metadata.query.filter",
+                                      "symbols": [
+                                        "CONTAIN",
+                                        "END_WITH",
+                                        "EQUAL",
+                                        "IEQUAL",
+                                        "IS_NULL",
+                                        "EXISTS",
+                                        "GREATER_THAN",
+                                        "GREATER_THAN_OR_EQUAL_TO",
+                                        "IN",
+                                        "LESS_THAN",
+                                        "LESS_THAN_OR_EQUAL_TO",
+                                        "BETWEEN",
+                                        "START_WITH",
+                                        "DESCENDANTS_INCL",
+                                        "ANCESTORS_INCL",
+                                        "RELATED_INCL"
+                                      ],
+                                      "doc": "The matching condition in a filter criterion"
+                                    },
+                                    "name": "condition",
+                                    "default": "EQUAL",
+                                    "doc": "The condition for the criterion, e.g. EQUAL, START_WITH"
+                                  },
+                                  {
+                                    "type": "boolean",
+                                    "name": "negated",
+                                    "default": false,
+                                    "doc": "Whether the condition should be negated"
+                                  }
+                                ],
+                                "doc": "A criterion for matching a field with given value"
+                              }
+                            },
+                            "name": "and",
+                            "doc": "A list of and criteria the filter applies to the query"
+                          }
+                        ],
+                        "doc": "A list of criterion and'd together."
+                      }
+                    }
+                  ],
+                  "name": "or",
+                  "default": null,
+                  "doc": "A list of disjunctive criterion for the filter. (or operation to combine filters)"
+                },
+                {
+                  "type": [
+                    "null",
+                    {
+                      "type": "array",
+                      "items": "com.linkedin.pegasus2avro.metadata.query.filter.Criterion"
+                    }
+                  ],
+                  "name": "criteria",
+                  "default": null,
+                  "doc": "Deprecated! A list of conjunctive criterion for the filter. If \"or\" field is provided, then this field is ignored."
+                }
+              ],
+              "doc": "The filter for finding a record or a collection of records"
+            },
+            "name": "filter",
+            "doc": "The filter criteria, which represents the view itself"
+          }
+        ],
+        "doc": "A View definition."
+      },
+      "name": "definition",
+      "doc": "The view itself"
+    },
+    {
+      "Searchable": {
+        "/actor": {
+          "fieldName": "createdBy",
+          "fieldType": "URN"
+        },
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "AuditStamp",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": "long",
+            "name": "time",
+            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "actor",
+            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "impersonator",
+            "default": null,
+            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "message",
+            "default": null,
+            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+          }
+        ],
+        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+      },
+      "name": "created",
+      "doc": "Audit stamp capturing the time and actor who created the View."
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": "com.linkedin.pegasus2avro.common.AuditStamp",
+      "name": "lastModified",
+      "doc": "Audit stamp capturing the time and actor who last modified the View."
+    }
+  ],
+  "doc": "Information about a DataHub View. -- TODO: Understand whether an entity type filter is required."
+}

acryl_datahub_cloud/metadata/schemas/DataHubViewKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubViewKey",
+    "keyForEntity": "dataHubView",
+    "entityCategory": "core",
+    "entityAspects": [
+      "dataHubViewInfo"
+    ]
+  },
+  "name": "DataHubViewKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the View"
+    }
+  ],
+  "doc": "Key for a DataHub View"
+}

acryl_datahub_cloud/metadata/schemas/DataJobInfo.avsc

@@ -0,0 +1,254 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataJobInfo"
+  },
+  "name": "DataJobInfo",
+  "namespace": "com.linkedin.pegasus2avro.datajob",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldNameAliases": [
+          "_entityName"
+        ],
+        "fieldType": "WORD_GRAM"
+      },
+      "type": "string",
+      "name": "name",
+      "doc": "Job name"
+    },
+    {
+      "Searchable": {
+        "fieldType": "TEXT",
+        "hasValuesFieldName": "hasDescription"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "Job description"
+    },
+    {
+      "type": [
+        {
+          "type": "enum",
+          "symbolDocs": {
+            "COMMAND": "The command job type is one of the basic built-in types. It runs multiple UNIX commands using java processbuilder.\nUpon execution, Azkaban spawns off a process to run the command.",
+            "GLUE": "Glue type is for running AWS Glue job transforms.",
+            "HADOOP_JAVA": "Runs a java program with ability to access Hadoop cluster.\nhttps://azkaban.readthedocs.io/en/latest/jobTypes.html#java-job-type",
+            "HADOOP_SHELL": "In large part, this is the same Command type. The difference is its ability to talk to a Hadoop cluster\nsecurely, via Hadoop tokens.",
+            "HIVE": "Hive type is for running Hive jobs.",
+            "PIG": "Pig type is for running Pig jobs.",
+            "SQL": "SQL is for running Presto, mysql queries etc"
+          },
+          "name": "AzkabanJobType",
+          "namespace": "com.linkedin.pegasus2avro.datajob.azkaban",
+          "symbols": [
+            "COMMAND",
+            "HADOOP_JAVA",
+            "HADOOP_SHELL",
+            "HIVE",
+            "PIG",
+            "SQL",
+            "GLUE"
+          ],
+          "doc": "The various types of support azkaban jobs"
+        },
+        "string"
+      ],
+      "name": "type",
+      "doc": "Datajob type\n*NOTE**: AzkabanJobType is deprecated. Please use strings instead."
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.DataFlowUrn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "flowUrn",
+      "default": null,
+      "doc": "DataFlow urn that this job is part of",
+      "Urn": "DataFlowUrn"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the event occur"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "actor",
+              "default": null,
+              "doc": "Optional: The actor urn involved in the event.",
+              "Urn": "Urn"
+            }
+          ],
+          "doc": "A standard event timestamp"
+        }
+      ],
+      "name": "created",
+      "default": null,
+      "doc": "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)"
+    },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME"
+        }
+      },
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.TimeStamp"
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)"
+    },
+    {
+      "deprecated": "Use Data Process Instance model, instead",
+      "type": [
+        "null",
+        {
+          "type": "enum",
+          "symbolDocs": {
+            "COMPLETED": "Jobs with successful completion.",
+            "FAILED": "Jobs that have failed.",
+            "IN_PROGRESS": "Jobs currently running.",
+            "SKIPPED": "Jobs that have been skipped.",
+            "STARTING": "Jobs being initialized.",
+            "STOPPED": "Jobs that have stopped.",
+            "STOPPING": "Jobs being stopped.",
+            "UNKNOWN": "Jobs with unknown status (either unmappable or unavailable)"
+          },
+          "name": "JobStatus",
+          "namespace": "com.linkedin.pegasus2avro.datajob",
+          "symbols": [
+            "STARTING",
+            "IN_PROGRESS",
+            "STOPPING",
+            "STOPPED",
+            "COMPLETED",
+            "FAILED",
+            "UNKNOWN",
+            "SKIPPED"
+          ],
+          "doc": "Job statuses"
+        }
+      ],
+      "name": "status",
+      "default": null,
+      "doc": "Status of the job - Deprecated for Data Process Instance model."
+    },
+    {
+      "Searchable": {
+        "addToFilters": true,
+        "fieldType": "KEYWORD",
+        "filterNameOverride": "Environment",
+        "queryByDefault": false
+      },
+      "type": [
+        "null",
+        {
+          "type": "enum",
+          "symbolDocs": {
+            "CORP": "Designates corporation fabrics",
+            "DEV": "Designates development fabrics",
+            "EI": "Designates early-integration fabrics",
+            "NON_PROD": "Designates non-production fabrics",
+            "PRD": "Alternative Prod spelling",
+            "PRE": "Designates pre-production fabrics",
+            "PROD": "Designates production fabrics",
+            "QA": "Designates quality assurance fabrics",
+            "RVW": "Designates review fabrics",
+            "SANDBOX": "Designates sandbox fabrics",
+            "STG": "Designates staging fabrics",
+            "TEST": "Designates testing fabrics",
+            "TST": "Alternative Test spelling",
+            "UAT": "Designates user acceptance testing fabrics"
+          },
+          "name": "FabricType",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "symbols": [
+            "DEV",
+            "TEST",
+            "QA",
+            "UAT",
+            "EI",
+            "PRE",
+            "STG",
+            "NON_PROD",
+            "PROD",
+            "CORP",
+            "RVW",
+            "PRD",
+            "TST",
+            "SANDBOX"
+          ],
+          "doc": "Fabric group type"
+        }
+      ],
+      "name": "env",
+      "default": null,
+      "doc": "Environment for this job"
+    }
+  ],
+  "doc": "Information about a Data processing job"
+}