acryl-datahub 1.0.0.2rc4__py3-none-any.whl → 1.0.0.3rc1__py3-none-any.whl

datahub/metadata/schema.avsc

@@ -2343,12 +2343,14 @@
               "type": {
                 "type": "enum",
                 "symbolDocs": {
-                  "SQL": "A SQL Query"
+                  "SQL": "A SQL Query",
+                  "UNKNOWN": "Unknown query language"
                 },
                 "name": "QueryLanguage",
                 "namespace": "com.linkedin.pegasus2avro.query",
                 "symbols": [
-                  "SQL"
+                  "SQL",
+                  "UNKNOWN"
                 ]
               },
               "name": "language",
@@ -2887,6 +2889,11 @@
             "namespace": "com.linkedin.pegasus2avro.form",
             "fields": [
               {
+                "Searchable": {
+                  "fieldName": "promptId",
+                  "fieldType": "KEYWORD",
+                  "queryByDefault": false
+                },
                 "type": "string",
                 "name": "id",
                 "doc": "The unique id for this prompt. This must be GLOBALLY unique."
@@ -4130,6 +4137,92 @@
     ],
     "doc": "Editable information about an Asset Container as defined on the DataHub Platform"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "systemMetadata"
+    },
+    "name": "SystemMetadata",
+    "namespace": "com.linkedin.pegasus2avro.mxe",
+    "fields": [
+      {
+        "type": [
+          "long",
+          "null"
+        ],
+        "name": "lastObserved",
+        "default": 0,
+        "doc": "The timestamp the metadata was observed at"
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "runId",
+        "default": "no-run-id-provided",
+        "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "lastRunId",
+        "default": "no-run-id-provided",
+        "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "pipelineName",
+        "default": null,
+        "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryName",
+        "default": null,
+        "doc": "The model registry name that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryVersion",
+        "default": null,
+        "doc": "The model registry version that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "map",
+            "values": "string"
+          }
+        ],
+        "name": "properties",
+        "default": null,
+        "doc": "Additional properties"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "version",
+        "default": null,
+        "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+      }
+    ],
+    "doc": "Metadata associated with each metadata change that is processed by the system"
+  },
   {
     "type": "record",
     "name": "PlatformEvent",
@@ -4369,89 +4462,7 @@
       {
         "type": [
           "null",
-          {
-            "type": "record",
-            "name": "SystemMetadata",
-            "namespace": "com.linkedin.pegasus2avro.mxe",
-            "fields": [
-              {
-                "type": [
-                  "long",
-                  "null"
-                ],
-                "name": "lastObserved",
-                "default": 0,
-                "doc": "The timestamp the metadata was observed at"
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "runId",
-                "default": "no-run-id-provided",
-                "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "lastRunId",
-                "default": "no-run-id-provided",
-                "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "pipelineName",
-                "default": null,
-                "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryName",
-                "default": null,
-                "doc": "The model registry name that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryVersion",
-                "default": null,
-                "doc": "The model registry version that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "map",
-                    "values": "string"
-                  }
-                ],
-                "name": "properties",
-                "default": null,
-                "doc": "Additional properties"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "version",
-                "default": null,
-                "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
-              }
-            ],
-            "doc": "Metadata associated with each metadata change that is processed by the system"
-          }
+          "com.linkedin.pegasus2avro.mxe.SystemMetadata"
         ],
         "name": "systemMetadata",
         "default": null,
@@ -10156,6 +10167,9 @@
                           "doc": "Version of the MLModelDeployment"
                         },
                         {
+                          "Searchable": {
+                            "fieldName": "deploymentStatus"
+                          },
                           "type": [
                             "null",
                             {
@@ -15705,6 +15719,28 @@
     ],
     "doc": "Key for a External AccessManagement"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubOpenAPISchemaKey",
+      "keyForEntity": "dataHubOpenAPISchema",
+      "entityCategory": "internal",
+      "entityAspects": [
+        "systemMetadata"
+      ],
+      "entityDoc": "Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model."
+    },
+    "name": "DataHubOpenAPISchemaKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "type": "string",
+        "name": "id",
+        "doc": "A unique id for the DataHub OpenAPI schema."
+      }
+    ],
+    "doc": "Key for a Query"
+  },
   {
     "type": "record",
     "Aspect": {

datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc

@@ -0,0 +1,22 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubOpenAPISchemaKey",
+    "keyForEntity": "dataHubOpenAPISchema",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "systemMetadata"
+    ],
+    "entityDoc": "Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model."
+  },
+  "name": "DataHubOpenAPISchemaKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the DataHub OpenAPI schema."
+    }
+  ],
+  "doc": "Key for a Query"
+}

datahub/metadata/schemas/DataTransformLogic.avsc

@@ -31,12 +31,14 @@
                       "type": {
                         "type": "enum",
                         "symbolDocs": {
-                          "SQL": "A SQL Query"
+                          "SQL": "A SQL Query",
+                          "UNKNOWN": "Unknown query language"
                         },
                         "name": "QueryLanguage",
                         "namespace": "com.linkedin.pegasus2avro.query",
                         "symbols": [
-                          "SQL"
+                          "SQL",
+                          "UNKNOWN"
                         ]
                       },
                       "name": "language",

datahub/metadata/schemas/FormInfo.avsc

@@ -53,6 +53,11 @@
           "namespace": "com.linkedin.pegasus2avro.form",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "promptId",
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
               "type": "string",
               "name": "id",
               "doc": "The unique id for this prompt. This must be GLOBALLY unique."

datahub/metadata/schemas/MLModelDeploymentProperties.avsc

@@ -135,6 +135,9 @@
       "doc": "Version of the MLModelDeployment"
     },
     {
+      "Searchable": {
+        "fieldName": "deploymentStatus"
+      },
       "type": [
         "null",
         {

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -6755,6 +6755,9 @@
                         "doc": "Version of the MLModelDeployment"
                       },
                       {
+                        "Searchable": {
+                          "fieldName": "deploymentStatus"
+                        },
                         "type": [
                           "null",
                           {
@@ -8116,6 +8119,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/MetadataChangeLog.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/MetadataChangeProposal.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/QueryProperties.avsc

@@ -36,12 +36,14 @@
             "type": {
               "type": "enum",
               "symbolDocs": {
-                "SQL": "A SQL Query"
+                "SQL": "A SQL Query",
+                "UNKNOWN": "Unknown query language"
               },
               "name": "QueryLanguage",
               "namespace": "com.linkedin.pegasus2avro.query",
               "symbols": [
-                "SQL"
+                "SQL",
+                "UNKNOWN"
               ]
             },
             "name": "language",

datahub/metadata/schemas/SystemMetadata.avsc

@@ -0,0 +1,86 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "systemMetadata"
+  },
+  "name": "SystemMetadata",
+  "namespace": "com.linkedin.pegasus2avro.mxe",
+  "fields": [
+    {
+      "type": [
+        "long",
+        "null"
+      ],
+      "name": "lastObserved",
+      "default": 0,
+      "doc": "The timestamp the metadata was observed at"
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "runId",
+      "default": "no-run-id-provided",
+      "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "lastRunId",
+      "default": "no-run-id-provided",
+      "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "pipelineName",
+      "default": null,
+      "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryName",
+      "default": null,
+      "doc": "The model registry name that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryVersion",
+      "default": null,
+      "doc": "The model registry version that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": "string"
+        }
+      ],
+      "name": "properties",
+      "default": null,
+      "doc": "Additional properties"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "version",
+      "default": null,
+      "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+    }
+  ],
+  "doc": "Metadata associated with each metadata change that is processed by the system"
+}

datahub/sql_parsing/sqlglot_utils.py

@@ -257,7 +257,10 @@ def generate_hash(text: str) -> str:
 
 
 def get_query_fingerprint_debug(
-    expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr, fast: bool = False
+    expression: sqlglot.exp.ExpOrStr,
+    platform: DialectOrStr,
+    fast: bool = False,
+    secondary_id: Optional[str] = None,
 ) -> Tuple[str, Optional[str]]:
     try:
         if not fast:
@@ -272,16 +275,18 @@ def get_query_fingerprint_debug(
         logger.debug("Failed to generalize query for fingerprinting: %s", e)
         expression_sql = None
 
-    fingerprint = generate_hash(
-        expression_sql
-        if expression_sql is not None
-        else _expression_to_string(expression, platform=platform)
-    )
+    text = expression_sql or _expression_to_string(expression, platform=platform)
+    if secondary_id:
+        text = text + " -- " + secondary_id
+    fingerprint = generate_hash(text=text)
     return fingerprint, expression_sql
 
 
 def get_query_fingerprint(
-    expression: sqlglot.exp.ExpOrStr, platform: DialectOrStr, fast: bool = False
+    expression: sqlglot.exp.ExpOrStr,
+    platform: DialectOrStr,
+    fast: bool = False,
+    secondary_id: Optional[str] = None,
 ) -> str:
     """Get a fingerprint for a SQL query.
 
@@ -298,12 +303,15 @@ def get_query_fingerprint(
     Args:
         expression: The SQL query to fingerprint.
         platform: The SQL dialect to use.
+        secondary_id: An optional additional id string to included in the final fingerprint.
 
     Returns:
         The fingerprint for the SQL query.
     """
 
-    return get_query_fingerprint_debug(expression, platform, fast=fast)[0]
+    return get_query_fingerprint_debug(
+        expression=expression, platform=platform, fast=fast, secondary_id=secondary_id
+    )[0]
 
 
 @functools.lru_cache(maxsize=FORMAT_QUERY_CACHE_SIZE)