acryl-datahub-cloud 0.3.11rc0__py3-none-any.whl → 0.3.16.1rc0__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc

@@ -8,6 +8,7 @@
       "datahubIngestionRunSummary",
       "datahubIngestionCheckpoint",
       "domains",
+      "applications",
       "deprecation",
       "versionInfo",
       "dataJobInfo",

acryl_datahub_cloud/metadata/schemas/DataPlatformInfo.avsc

@@ -13,7 +13,9 @@
         "fieldNameAliases": [
           "_entityName"
         ],
-        "fieldType": "WORD_GRAM"
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "entityName",
+        "searchTier": 1
       },
       "validate": {
         "strlen": {

acryl_datahub_cloud/metadata/schemas/DataPlatformInstanceProperties.avsc

@@ -44,7 +44,9 @@
         "fieldNameAliases": [
           "_entityName"
         ],
-        "fieldType": "WORD_GRAM"
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "entityName",
+        "searchTier": 1
       },
       "type": [
         "null",
@@ -57,7 +59,8 @@
     {
       "Searchable": {
         "fieldType": "TEXT",
-        "hasValuesFieldName": "hasDescription"
+        "hasValuesFieldName": "hasDescription",
+        "searchTier": 2
       },
       "type": [
         "null",

acryl_datahub_cloud/metadata/schemas/DataProcessKey.avsc

@@ -52,6 +52,8 @@
           "QA": "Designates quality assurance fabrics",
           "RVW": "Designates review fabrics",
           "SANDBOX": "Designates sandbox fabrics",
+          "SBX": "Alternative spelling for sandbox",
+          "SIT": "System Integration Testing",
           "STG": "Designates staging fabrics",
           "TEST": "Designates testing fabrics",
           "TST": "Alternative Test spelling",
@@ -73,6 +75,8 @@
           "RVW",
           "PRD",
           "TST",
+          "SIT",
+          "SBX",
           "SANDBOX"
         ],
         "doc": "Fabric group type"

acryl_datahub_cloud/metadata/schemas/DataProductKey.avsc

@@ -9,6 +9,7 @@
       "glossaryTerms",
       "globalTags",
       "domains",
+      "applications",
       "dataProductProperties",
       "institutionalMemory",
       "status",
@@ -16,6 +17,7 @@
       "forms",
       "testResults",
       "subTypes",
+      "assetSettings",
       "share",
       "origin"
     ]

acryl_datahub_cloud/metadata/schemas/DataProductProperties.avsc

@@ -44,7 +44,9 @@
         "fieldNameAliases": [
           "_entityName"
         ],
-        "fieldType": "WORD_GRAM"
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "entityName",
+        "searchTier": 1
       },
       "type": [
         "null",
@@ -57,7 +59,8 @@
     {
       "Searchable": {
         "fieldType": "TEXT",
-        "hasValuesFieldName": "hasDescription"
+        "hasValuesFieldName": "hasDescription",
+        "searchTier": 2
       },
       "type": [
         "null",
@@ -65,7 +68,7 @@
       ],
       "name": "description",
       "default": null,
-      "doc": "Documentation of the dataset"
+      "doc": "Documentation of the data product"
     },
     {
       "Relationship": {

acryl_datahub_cloud/metadata/schemas/DataTypeInfo.avsc

@@ -7,6 +7,11 @@
   "namespace": "com.linkedin.pegasus2avro.datatype",
   "fields": [
     {
+      "Searchable": {
+        "fieldType": "KEYWORD",
+        "searchLabel": "qualifiedName",
+        "searchTier": 1
+      },
       "type": "string",
       "name": "qualifiedName",
       "doc": "The qualified name for the data type. Usually a unique namespace + name, e.g. datahub.string"

acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc

@@ -11,6 +11,7 @@
       "datasetUsageStatistics",
       "operation",
       "domains",
+      "applications",
       "schemaMetadata",
       "status",
       "container",
@@ -38,6 +39,7 @@
       "partitionsSummary",
       "versionProperties",
       "icebergCatalogInfo",
+      "logicalParent",
       "inferredNeighbors",
       "inferredMetadata",
       "schemaFieldsInferredMetadata",
@@ -52,7 +54,8 @@
       "share",
       "origin",
       "documentation",
-      "entityInferenceMetadata"
+      "entityInferenceMetadata",
+      "assetSettings"
     ],
     "entityDoc": "Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets."
   },
@@ -88,7 +91,8 @@
         "addToFilters": true,
         "fieldType": "TEXT_PARTIAL",
         "filterNameOverride": "Environment",
-        "queryByDefault": false
+        "queryByDefault": false,
+        "searchLabel": "origin"
       },
       "type": {
         "type": "enum",
@@ -103,6 +107,8 @@
           "QA": "Designates quality assurance fabrics",
           "RVW": "Designates review fabrics",
           "SANDBOX": "Designates sandbox fabrics",
+          "SBX": "Alternative spelling for sandbox",
+          "SIT": "System Integration Testing",
           "STG": "Designates staging fabrics",
           "TEST": "Designates testing fabrics",
           "TST": "Alternative Test spelling",
@@ -124,6 +130,8 @@
           "RVW",
           "PRD",
           "TST",
+          "SIT",
+          "SBX",
           "SANDBOX"
         ],
         "doc": "Fabric group type"

acryl_datahub_cloud/metadata/schemas/DatasetProperties.avsc

@@ -44,7 +44,9 @@
         "fieldNameAliases": [
           "_entityName"
         ],
-        "fieldType": "WORD_GRAM"
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "entityName",
+        "searchTier": 1
       },
       "type": [
         "null",
@@ -59,7 +61,9 @@
         "addToFilters": false,
         "boostScore": 10.0,
         "enableAutocomplete": true,
-        "fieldType": "WORD_GRAM"
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "qualifiedName",
+        "searchTier": 1
       },
       "type": [
         "null",
@@ -72,7 +76,8 @@
     {
       "Searchable": {
         "fieldType": "TEXT",
-        "hasValuesFieldName": "hasDescription"
+        "hasValuesFieldName": "hasDescription",
+        "searchTier": 2
       },
       "type": [
         "null",
@@ -99,7 +104,8 @@
       "Searchable": {
         "/time": {
           "fieldName": "createdAt",
-          "fieldType": "DATETIME"
+          "fieldType": "DATETIME",
+          "searchLabel": "createdAt"
         }
       },
       "type": [
@@ -139,7 +145,8 @@
       "Searchable": {
         "/time": {
           "fieldName": "lastModifiedAt",
-          "fieldType": "DATETIME"
+          "fieldType": "DATETIME",
+          "searchLabel": "lastModifiedAt"
         }
       },
       "type": [

acryl_datahub_cloud/metadata/schemas/DatasetUsageStatistics.avsc

@@ -130,6 +130,10 @@
       "doc": "The optional messageId, if provided serves as a custom user-defined unique identifier for an aspect value."
     },
     {
+      "Searchable": {
+        "fieldType": "COUNT",
+        "hasValuesFieldName": "hasUniqueUserCount"
+      },
       "TimeseriesField": {},
       "type": [
         "null",
@@ -140,6 +144,10 @@
       "doc": "Unique user count"
     },
     {
+      "Searchable": {
+        "fieldType": "COUNT",
+        "hasValuesFieldName": "hasTotalSqlQueriesCount"
+      },
       "TimeseriesField": {},
       "type": [
         "null",

acryl_datahub_cloud/metadata/schemas/DocumentInfo.avsc

@@ -0,0 +1,407 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "documentInfo"
+  },
+  "name": "DocumentInfo",
+  "namespace": "com.linkedin.pegasus2avro.knowledge",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldNameAliases": [
+          "_entityName"
+        ],
+        "fieldType": "WORD_GRAM",
+        "searchLabel": "entityName",
+        "searchTier": 1
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "title",
+      "default": null,
+      "doc": "Optional title for the document. "
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "DocumentSource",
+          "namespace": "com.linkedin.pegasus2avro.knowledge",
+          "fields": [
+            {
+              "Searchable": {
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
+              "type": {
+                "type": "enum",
+                "symbolDocs": {
+                  "EXTERNAL": "External - The document was ingested from an external source. ",
+                  "NATIVE": "Created via the DataHub UI or API"
+                },
+                "name": "DocumentSourceType",
+                "namespace": "com.linkedin.pegasus2avro.knowledge",
+                "symbols": [
+                  "NATIVE",
+                  "EXTERNAL"
+                ]
+              },
+              "name": "sourceType",
+              "doc": "The type of the source (e.g., \"Confluence\", \"Notion\", \"Google Docs\", \"SharePoint\", \"Slack\")"
+            },
+            {
+              "Searchable": {
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "externalUrl",
+              "default": null,
+              "doc": "URL to the external source where this document originated"
+            },
+            {
+              "Searchable": {
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "externalId",
+              "default": null,
+              "doc": "Unique identifier in the external system. Searchable in case we need to find ingested docs via filtering."
+            }
+          ],
+          "doc": "Information about the source of a document, especially for externally sourced documents.\nThis record is embedded within DocumentInfo to track whether a document is first-party \n(created in DataHub) or third-party (ingested from external sources like Slack, Notion, etc.)"
+        }
+      ],
+      "name": "source",
+      "default": null,
+      "doc": "Information about the external source of this document. \nOnly populated for third-party documents ingested from external systems."
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DocumentStatus",
+        "namespace": "com.linkedin.pegasus2avro.knowledge",
+        "fields": [
+          {
+            "Searchable": {
+              "fieldType": "KEYWORD",
+              "queryByDefault": false
+            },
+            "type": {
+              "type": "enum",
+              "symbolDocs": {
+                "PUBLISHED": "Document is published and visible to others.",
+                "UNPUBLISHED": "Document is not published to be consumed by others."
+              },
+              "name": "DocumentState",
+              "namespace": "com.linkedin.pegasus2avro.knowledge",
+              "symbols": [
+                "PUBLISHED",
+                "UNPUBLISHED"
+              ],
+              "doc": "The state of a document"
+            },
+            "name": "state",
+            "doc": "The current visibility state of the document"
+          }
+        ],
+        "doc": "Visibility status information for a document"
+      },
+      "name": "status",
+      "doc": "Status of the document (published, unpublished.)"
+    },
+    {
+      "type": {
+        "type": "record",
+        "name": "DocumentContents",
+        "namespace": "com.linkedin.pegasus2avro.knowledge",
+        "fields": [
+          {
+            "Searchable": {
+              "fieldType": "TEXT"
+            },
+            "type": "string",
+            "name": "text",
+            "doc": "The text contents of the document.\nThis needs to be added to semantic search! "
+          }
+        ],
+        "doc": "The contents of a document"
+      },
+      "name": "contents",
+      "doc": "Content of the document"
+    },
+    {
+      "Searchable": {
+        "/actor": {
+          "fieldName": "creator",
+          "fieldType": "URN",
+          "queryByDefault": false
+        },
+        "/time": {
+          "fieldName": "createdAt",
+          "fieldType": "DATETIME",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "AuditStamp",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": "long",
+            "name": "time",
+            "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": "string",
+            "name": "actor",
+            "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+            "Urn": "Urn"
+          },
+          {
+            "java": {
+              "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+            },
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "impersonator",
+            "default": null,
+            "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+            "Urn": "Urn"
+          },
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "message",
+            "default": null,
+            "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+          }
+        ],
+        "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+      },
+      "name": "created",
+      "doc": "The time and actor who created the document"
+    },
+    {
+      "Searchable": {
+        "/actor": {
+          "fieldName": "lastModifiedBy",
+          "fieldType": "URN",
+          "queryByDefault": false
+        },
+        "/time": {
+          "addToFilters": false,
+          "fieldName": "lastModifiedAt",
+          "fieldType": "DATETIME",
+          "queryByDefault": false
+        }
+      },
+      "type": "com.linkedin.pegasus2avro.common.AuditStamp",
+      "name": "lastModified",
+      "doc": "The time and actor who last modified the document (any field)"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "RelatedAsset",
+            "namespace": "com.linkedin.pegasus2avro.knowledge",
+            "fields": [
+              {
+                "Relationship": {
+                  "entityTypes": [
+                    "container",
+                    "dataset",
+                    "dataJob",
+                    "dataFlow",
+                    "dashboard",
+                    "chart",
+                    "application",
+                    "mlModel",
+                    "mlModelGroup",
+                    "mlPrimaryKey",
+                    "mlFeature",
+                    "mlFeatureTable",
+                    "dataProduct",
+                    "domain",
+                    "glossaryTerm",
+                    "glossaryNode",
+                    "tag",
+                    "structuredProperty",
+                    "mlModel",
+                    "mlModelGroup",
+                    "mlPrimaryKey",
+                    "mlFeature",
+                    "mlFeatureTable"
+                  ],
+                  "name": "RelatedAsset"
+                },
+                "Searchable": {
+                  "fieldName": "relatedAssets",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "asset",
+                "doc": "The asset referenced by or related to the document. ",
+                "Urn": "Urn",
+                "entityTypes": [
+                  "container",
+                  "dataset",
+                  "dataJob",
+                  "dataFlow",
+                  "dashboard",
+                  "chart",
+                  "application",
+                  "mlModel",
+                  "mlModelGroup",
+                  "mlPrimaryKey",
+                  "mlFeature",
+                  "mlFeatureTable",
+                  "dataProduct",
+                  "domain",
+                  "glossaryTerm",
+                  "glossaryNode",
+                  "tag",
+                  "structuredProperty",
+                  "mlModel",
+                  "mlModelGroup",
+                  "mlPrimaryKey",
+                  "mlFeature",
+                  "mlFeatureTable"
+                ]
+              }
+            ],
+            "doc": "A data asset referenced by a document."
+          }
+        }
+      ],
+      "name": "relatedAssets",
+      "default": null,
+      "doc": "Assets referenced by or related to this document. "
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "RelatedDocument",
+            "namespace": "com.linkedin.pegasus2avro.knowledge",
+            "fields": [
+              {
+                "Relationship": {
+                  "entityTypes": [
+                    "document"
+                  ],
+                  "name": "RelatedDocument"
+                },
+                "Searchable": {
+                  "fieldName": "relatedDocuments",
+                  "fieldType": "URN",
+                  "queryByDefault": false
+                },
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "document",
+                "doc": "The document referenced by or related to the document. ",
+                "Urn": "Urn",
+                "entityTypes": [
+                  "document"
+                ]
+              }
+            ],
+            "doc": "An document referenced by or related to another document\nNote that this does NOT include child documents. "
+          }
+        }
+      ],
+      "name": "relatedDocuments",
+      "default": null,
+      "doc": "Documents referenced by or related to this document. "
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "ParentDocument",
+          "namespace": "com.linkedin.pegasus2avro.knowledge",
+          "fields": [
+            {
+              "Relationship": {
+                "entityTypes": [
+                  "document"
+                ],
+                "name": "IsChildOf"
+              },
+              "Searchable": {
+                "fieldName": "parentDocument",
+                "fieldType": "URN",
+                "queryByDefault": false
+              },
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "document",
+              "doc": "The hierarchical parent document for this document. ",
+              "Urn": "Urn",
+              "entityTypes": [
+                "document"
+              ]
+            }
+          ],
+          "doc": "The parent document of the document. "
+        }
+      ],
+      "name": "parentDocument",
+      "default": null,
+      "doc": "Parent article for this asset. "
+    }
+  ],
+  "doc": "Information about a document"
+}

acryl_datahub_cloud/metadata/schemas/DocumentKey.avsc

@@ -0,0 +1,35 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "documentKey",
+    "keyForEntity": "document",
+    "entityCategory": "core",
+    "entityAspects": [
+      "documentInfo",
+      "documentSettings",
+      "status",
+      "ownership",
+      "domains",
+      "structuredProperties",
+      "subTypes",
+      "dataPlatformInstance",
+      "browsePathsV2",
+      "globalTags",
+      "glossaryTerms",
+      "semanticContent",
+      "institutionalMemory",
+      "documentation"
+    ]
+  },
+  "name": "DocumentKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "Searchable": {},
+      "type": "string",
+      "name": "id",
+      "doc": "Unique identifier for the document."
+    }
+  ],
+  "doc": "Key for a Document"
+}