acryl-datahub 1.0.0.1rc6__py3-none-any.whl → 1.0.0.2__py3-none-any.whl

datahub/metadata/schema.avsc

@@ -2343,12 +2343,14 @@
               "type": {
                 "type": "enum",
                 "symbolDocs": {
-                  "SQL": "A SQL Query"
+                  "SQL": "A SQL Query",
+                  "UNKNOWN": "Unknown query language"
                 },
                 "name": "QueryLanguage",
                 "namespace": "com.linkedin.pegasus2avro.query",
                 "symbols": [
-                  "SQL"
+                  "SQL",
+                  "UNKNOWN"
                 ]
               },
               "name": "language",
@@ -2887,6 +2889,11 @@
             "namespace": "com.linkedin.pegasus2avro.form",
             "fields": [
               {
+                "Searchable": {
+                  "fieldName": "promptId",
+                  "fieldType": "KEYWORD",
+                  "queryByDefault": false
+                },
                 "type": "string",
                 "name": "id",
                 "doc": "The unique id for this prompt. This must be GLOBALLY unique."
@@ -4130,6 +4137,92 @@
     ],
     "doc": "Editable information about an Asset Container as defined on the DataHub Platform"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "systemMetadata"
+    },
+    "name": "SystemMetadata",
+    "namespace": "com.linkedin.pegasus2avro.mxe",
+    "fields": [
+      {
+        "type": [
+          "long",
+          "null"
+        ],
+        "name": "lastObserved",
+        "default": 0,
+        "doc": "The timestamp the metadata was observed at"
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "runId",
+        "default": "no-run-id-provided",
+        "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "lastRunId",
+        "default": "no-run-id-provided",
+        "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "pipelineName",
+        "default": null,
+        "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryName",
+        "default": null,
+        "doc": "The model registry name that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryVersion",
+        "default": null,
+        "doc": "The model registry version that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "map",
+            "values": "string"
+          }
+        ],
+        "name": "properties",
+        "default": null,
+        "doc": "Additional properties"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "version",
+        "default": null,
+        "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+      }
+    ],
+    "doc": "Metadata associated with each metadata change that is processed by the system"
+  },
   {
     "type": "record",
     "name": "PlatformEvent",
@@ -4369,89 +4462,7 @@
       {
         "type": [
           "null",
-          {
-            "type": "record",
-            "name": "SystemMetadata",
-            "namespace": "com.linkedin.pegasus2avro.mxe",
-            "fields": [
-              {
-                "type": [
-                  "long",
-                  "null"
-                ],
-                "name": "lastObserved",
-                "default": 0,
-                "doc": "The timestamp the metadata was observed at"
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "runId",
-                "default": "no-run-id-provided",
-                "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "lastRunId",
-                "default": "no-run-id-provided",
-                "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "pipelineName",
-                "default": null,
-                "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryName",
-                "default": null,
-                "doc": "The model registry name that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryVersion",
-                "default": null,
-                "doc": "The model registry version that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "map",
-                    "values": "string"
-                  }
-                ],
-                "name": "properties",
-                "default": null,
-                "doc": "Additional properties"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "version",
-                "default": null,
-                "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
-              }
-            ],
-            "doc": "Metadata associated with each metadata change that is processed by the system"
-          }
+          "com.linkedin.pegasus2avro.mxe.SystemMetadata"
         ],
         "name": "systemMetadata",
         "default": null,
@@ -10156,6 +10167,9 @@
                           "doc": "Version of the MLModelDeployment"
                         },
                         {
+                          "Searchable": {
+                            "fieldName": "deploymentStatus"
+                          },
                           "type": [
                             "null",
                             {
@@ -15705,6 +15719,28 @@
     ],
     "doc": "Key for a External AccessManagement"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubOpenAPISchemaKey",
+      "keyForEntity": "dataHubOpenAPISchema",
+      "entityCategory": "internal",
+      "entityAspects": [
+        "systemMetadata"
+      ],
+      "entityDoc": "Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model."
+    },
+    "name": "DataHubOpenAPISchemaKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "type": "string",
+        "name": "id",
+        "doc": "A unique id for the DataHub OpenAPI schema."
+      }
+    ],
+    "doc": "Key for a Query"
+  },
   {
     "type": "record",
     "Aspect": {

datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc

@@ -0,0 +1,22 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubOpenAPISchemaKey",
+    "keyForEntity": "dataHubOpenAPISchema",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "systemMetadata"
+    ],
+    "entityDoc": "Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model."
+  },
+  "name": "DataHubOpenAPISchemaKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the DataHub OpenAPI schema."
+    }
+  ],
+  "doc": "Key for a Query"
+}

datahub/metadata/schemas/DataTransformLogic.avsc

@@ -31,12 +31,14 @@
                       "type": {
                         "type": "enum",
                         "symbolDocs": {
-                          "SQL": "A SQL Query"
+                          "SQL": "A SQL Query",
+                          "UNKNOWN": "Unknown query language"
                         },
                         "name": "QueryLanguage",
                         "namespace": "com.linkedin.pegasus2avro.query",
                         "symbols": [
-                          "SQL"
+                          "SQL",
+                          "UNKNOWN"
                         ]
                       },
                       "name": "language",

datahub/metadata/schemas/FormInfo.avsc

@@ -53,6 +53,11 @@
           "namespace": "com.linkedin.pegasus2avro.form",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "promptId",
+                "fieldType": "KEYWORD",
+                "queryByDefault": false
+              },
               "type": "string",
               "name": "id",
               "doc": "The unique id for this prompt. This must be GLOBALLY unique."

datahub/metadata/schemas/MLModelDeploymentProperties.avsc

@@ -135,6 +135,9 @@
       "doc": "Version of the MLModelDeployment"
     },
     {
+      "Searchable": {
+        "fieldName": "deploymentStatus"
+      },
       "type": [
         "null",
         {

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -6755,6 +6755,9 @@
                         "doc": "Version of the MLModelDeployment"
                       },
                       {
+                        "Searchable": {
+                          "fieldName": "deploymentStatus"
+                        },
                         "type": [
                           "null",
                           {
@@ -8116,6 +8119,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/MetadataChangeLog.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/MetadataChangeProposal.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/QueryProperties.avsc

@@ -36,12 +36,14 @@
             "type": {
               "type": "enum",
               "symbolDocs": {
-                "SQL": "A SQL Query"
+                "SQL": "A SQL Query",
+                "UNKNOWN": "Unknown query language"
               },
               "name": "QueryLanguage",
               "namespace": "com.linkedin.pegasus2avro.query",
               "symbols": [
-                "SQL"
+                "SQL",
+                "UNKNOWN"
               ]
             },
             "name": "language",

datahub/metadata/schemas/SystemMetadata.avsc

@@ -0,0 +1,86 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "systemMetadata"
+  },
+  "name": "SystemMetadata",
+  "namespace": "com.linkedin.pegasus2avro.mxe",
+  "fields": [
+    {
+      "type": [
+        "long",
+        "null"
+      ],
+      "name": "lastObserved",
+      "default": 0,
+      "doc": "The timestamp the metadata was observed at"
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "runId",
+      "default": "no-run-id-provided",
+      "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "lastRunId",
+      "default": "no-run-id-provided",
+      "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "pipelineName",
+      "default": null,
+      "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryName",
+      "default": null,
+      "doc": "The model registry name that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryVersion",
+      "default": null,
+      "doc": "The model registry version that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": "string"
+        }
+      ],
+      "name": "properties",
+      "default": null,
+      "doc": "Additional properties"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "version",
+      "default": null,
+      "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+    }
+  ],
+  "doc": "Metadata associated with each metadata change that is processed by the system"
+}

datahub/sdk/search_client.py

@@ -2,36 +2,106 @@ from __future__ import annotations
 
 from typing import (
     TYPE_CHECKING,
-    Dict,
     Iterable,
     List,
     Optional,
+    Tuple,
+    Type,
+    TypeVar,
 )
 
-from datahub.ingestion.graph.filters import RawSearchFilterRule
+from datahub.ingestion.graph.filters import RawSearchFilter, RemovedStatusFilter
 from datahub.metadata.urns import Urn
-from datahub.sdk.search_filters import Filter
+from datahub.sdk.search_filters import (
+    Filter,
+    FilterDsl,
+    _EntityTypeFilter,
+    _OrFilters,
+    _StatusFilter,
+)
 
 if TYPE_CHECKING:
     from datahub.sdk.main_client import DataHubClient
 
 
+_FilterType = TypeVar("_FilterType", bound=Filter)
+
+
+def _typed_dfs(
+    filter: Optional[_FilterType], type: Type[_FilterType]
+) -> Optional[List[_FilterType]]:
+    if filter is None:
+        return None
+
+    found: Optional[List[_FilterType]] = None
+    for f in filter.dfs():
+        if isinstance(f, type):
+            if found is None:
+                found = []
+            found.append(f)
+    return found
+
+
 def compile_filters(
     filter: Optional[Filter],
-) -> Optional[List[Dict[str, List[RawSearchFilterRule]]]]:
+) -> Tuple[Optional[List[str]], RawSearchFilter]:
     # TODO: Not every filter type is supported for every entity type.
     # If we can detect issues with the filters at compile time, we should
     # raise an error.
 
-    if filter is None:
-        return None
+    existing_soft_deleted_filter = _typed_dfs(filter, _StatusFilter)
+    if existing_soft_deleted_filter is None:
+        soft_deleted_filter = FilterDsl.soft_deleted(
+            RemovedStatusFilter.NOT_SOFT_DELETED
+        )
+        if filter is None:
+            filter = soft_deleted_filter
+        else:
+            filter = FilterDsl.and_(filter, soft_deleted_filter)
+
+    # This should be safe - if filter were None coming in, then we would replace it
+    # with the soft-deleted filter.
+    assert filter is not None
 
     initial_filters = filter.compile()
-    return [
+
+    compiled_filters: RawSearchFilter = [
         {"and": [rule.to_raw() for rule in andClause["and"]]}
         for andClause in initial_filters
     ]
 
+    entity_types = compute_entity_types(initial_filters)
+
+    return entity_types, compiled_filters
+
+
+def compute_entity_types(
+    filters: _OrFilters,
+) -> Optional[List[str]]:
+    found_filters = False
+    found_positive_filters = False
+    entity_types: List[str] = []
+    for ands in filters:
+        for clause in ands["and"]:
+            if clause.field == _EntityTypeFilter.ENTITY_TYPE_FIELD:
+                found_filters = True
+                if not clause.negated:
+                    found_positive_filters = True
+
+                entity_types.extend(clause.values)
+
+    if not found_filters:
+        # If we didn't find any filters, use None so we use the default set.
+        return None
+
+    if not found_positive_filters:
+        # If we only found negated filters, then it's probably a query like
+        # "find me all entities except for dashboards". In that case, we
+        # still want to use the default set.
+        return None
+
+    return entity_types
+
 
 class SearchClient:
     def __init__(self, client: DataHubClient):
@@ -43,8 +113,11 @@ class SearchClient:
         filter: Optional[Filter] = None,
     ) -> Iterable[Urn]:
         # TODO: Add better limit / pagination support.
+        types, compiled_filters = compile_filters(filter)
         for urn in self._client._graph.get_urns_by_filter(
             query=query,
-            extra_or_filters=compile_filters(filter),
+            status=None,
+            extra_or_filters=compiled_filters,
+            entity_types=types,
         ):
             yield Urn.from_string(urn)