acryl-datahub 0.15.0.1rc12__py3-none-any.whl → 0.15.0.1rc14__py3-none-any.whl

datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py

@@ -1,9 +1,10 @@
 import logging
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
-from typing import List, Optional
+from threading import Lock
+from typing import Dict, Iterable, List, Optional
 
 from pydantic import Field
 
@@ -18,12 +19,28 @@ from datahub.utilities.urns._urn_base import Urn
 
 logger = logging.getLogger(__name__)
 
+QUERY_QUERY_ENTITY = """
+query listQueries($input: ScrollAcrossEntitiesInput!) {
+  scrollAcrossEntities(input: $input) {
+    nextScrollId
+    count
+    searchResults {
+      entity {
+        ... on QueryEntity {
+          urn
+        }
+      }
+    }
+  }
+}
+"""
+
 
 class SoftDeletedEntitiesCleanupConfig(ConfigModel):
     enabled: bool = Field(
         default=True, description="Whether to do soft deletion cleanup."
     )
-    retention_days: Optional[int] = Field(
+    retention_days: int = Field(
         10,
         description="Number of days to retain metadata in DataHub",
     )
@@ -62,23 +79,30 @@ class SoftDeletedEntitiesCleanupConfig(ConfigModel):
         default=None,
         description="Query to filter entities",
     )
+
     limit_entities_delete: Optional[int] = Field(
         25000, description="Max number of entities to delete."
     )
 
-    runtime_limit_seconds: Optional[int] = Field(
-        None,
+    futures_max_at_time: int = Field(
+        1000, description="Max number of futures to have at a time."
+    )
+
+    runtime_limit_seconds: int = Field(
+        7200,  # 2 hours by default
         description="Runtime limit in seconds",
     )
 
 
 @dataclass
 class SoftDeletedEntitiesReport(SourceReport):
-    num_soft_deleted_entity_removed: int = 0
-    num_soft_deleted_entity_removed_by_type: TopKDict[str, int] = field(
-        default_factory=TopKDict
-    )
-    sample_soft_deleted_removed_aspects_by_type: TopKDict[str, LossyList[str]] = field(
+    num_queries_found: int = 0
+    num_soft_deleted_entity_processed: int = 0
+    num_soft_deleted_retained_due_to_age: int = 0
+    num_soft_deleted_entity_removal_started: int = 0
+    num_hard_deleted: int = 0
+    num_hard_deleted_by_type: TopKDict[str, int] = field(default_factory=TopKDict)
+    sample_hard_deleted_aspects_by_type: TopKDict[str, LossyList[str]] = field(
         default_factory=TopKDict
     )
 
@@ -103,48 +127,53 @@ class SoftDeletedEntitiesCleanup:
         self.config = config
         self.report = report
         self.dry_run = dry_run
+        self.start_time = 0.0
+        self._report_lock: Lock = Lock()
+        self.last_print_time = 0.0
+
+    def _increment_retained_count(self) -> None:
+        """Thread-safe method to update report fields"""
+        with self._report_lock:
+            self.report.num_soft_deleted_retained_due_to_age += 1
+
+    def _increment_removal_started_count(self) -> None:
+        """Thread-safe method to update report fields"""
+        with self._report_lock:
+            self.report.num_soft_deleted_entity_removal_started += 1
+
+    def _update_report(self, urn: str, entity_type: str) -> None:
+        """Thread-safe method to update report fields"""
+        with self._report_lock:
+            self.report.num_hard_deleted += 1
+
+            current_count = self.report.num_hard_deleted_by_type.get(entity_type, 0)
+            self.report.num_hard_deleted_by_type[entity_type] = current_count + 1
+            if entity_type not in self.report.sample_hard_deleted_aspects_by_type:
+                self.report.sample_hard_deleted_aspects_by_type[
+                    entity_type
+                ] = LossyList()
+            self.report.sample_hard_deleted_aspects_by_type[entity_type].append(urn)
 
     def delete_entity(self, urn: str) -> None:
         assert self.ctx.graph
 
         entity_urn = Urn.from_string(urn)
-        self.report.num_soft_deleted_entity_removed += 1
-        self.report.num_soft_deleted_entity_removed_by_type[entity_urn.entity_type] = (
-            self.report.num_soft_deleted_entity_removed_by_type.get(
-                entity_urn.entity_type, 0
-            )
-            + 1
-        )
-        if (
-            entity_urn.entity_type
-            not in self.report.sample_soft_deleted_removed_aspects_by_type
-        ):
-            self.report.sample_soft_deleted_removed_aspects_by_type[
-                entity_urn.entity_type
-            ] = LossyList()
-        self.report.sample_soft_deleted_removed_aspects_by_type[
-            entity_urn.entity_type
-        ].append(urn)
-
         if self.dry_run:
             logger.info(
                 f"Dry run is on otherwise it would have deleted {urn} with hard deletion"
             )
             return
-
+        self._increment_removal_started_count()
         self.ctx.graph.delete_entity(urn=urn, hard=True)
         self.ctx.graph.delete_references_to_urn(
             urn=urn,
             dry_run=False,
         )
+        self._update_report(urn, entity_urn.entity_type)
 
     def delete_soft_deleted_entity(self, urn: str) -> None:
         assert self.ctx.graph
 
-        if self.config.retention_days is None:
-            logger.info("Retention days is not set, skipping soft delete cleanup")
-            return
-
         retention_time = (
             int(datetime.now(timezone.utc).timestamp())
             - self.config.retention_days * 24 * 60 * 60
@@ -157,15 +186,85 @@ class SoftDeletedEntitiesCleanup:
             ]["created"]["time"] < (retention_time * 1000):
                 logger.debug(f"Hard deleting {urn}")
                 self.delete_entity(urn)
+            else:
+                self._increment_retained_count()
+
+    def _print_report(self) -> None:
+        time_taken = round(time.time() - self.last_print_time, 1)
+        # Print report every 2 minutes
+        if time_taken > 120:
+            self.last_print_time = time.time()
+            logger.info(f"\n{self.report.as_string()}")
+
+    def _process_futures(self, futures: Dict[Future, str]) -> Dict[Future, str]:
+        done, not_done = wait(futures, return_when=FIRST_COMPLETED)
+        futures = {future: urn for future, urn in futures.items() if future in not_done}
+
+        for future in done:
+            self._print_report()
+            if future.exception():
+                logger.error(
+                    f"Failed to delete entity {futures[future]}: {future.exception()}"
+                )
+                self.report.failure(
+                    f"Failed to delete entity {futures[future]}",
+                    exc=future.exception(),
+                )
+            self.report.num_soft_deleted_entity_processed += 1
+            if (
+                self.report.num_soft_deleted_entity_processed % self.config.batch_size
+                == 0
+            ):
+                if self.config.delay:
+                    logger.debug(
+                        f"Sleeping for {self.config.delay} seconds before further processing batch"
+                    )
+                    time.sleep(self.config.delay)
+        return futures
 
-    def cleanup_soft_deleted_entities(self) -> None:
-        if not self.config.enabled:
-            return
+    def _get_soft_deleted_queries(self) -> Iterable[str]:
         assert self.ctx.graph
-        start_time = time.time()
-
-        deleted_count_retention = 0
-        urns = self.ctx.graph.get_urns_by_filter(
+        scroll_id: Optional[str] = None
+        while True:
+            try:
+                result = self.ctx.graph.execute_graphql(
+                    QUERY_QUERY_ENTITY,
+                    {
+                        "input": {
+                            "types": ["QUERY"],
+                            "query": "*",
+                            "scrollId": scroll_id if scroll_id else None,
+                            "count": self.config.batch_size,
+                            "orFilters": [
+                                {
+                                    "and": [
+                                        {
+                                            "field": "removed",
+                                            "values": ["true"],
+                                            "condition": "EQUAL",
+                                        }
+                                    ]
+                                }
+                            ],
+                        }
+                    },
+                )
+            except Exception as e:
+                self.report.failure(
+                    f"While trying to get queries with {scroll_id}", exc=e
+                )
+                break
+            scroll_across_entities = result.get("scrollAcrossEntities")
+            if not scroll_across_entities:
+                break
+            scroll_id = scroll_across_entities.get("nextScrollId")
+            self.report.num_queries_found += scroll_across_entities.get("count")
+            for query in scroll_across_entities.get("searchResults"):
+                yield query["entity"]["urn"]
+
+    def _get_urns(self) -> Iterable[str]:
+        assert self.ctx.graph
+        yield from self.ctx.graph.get_urns_by_filter(
             entity_types=self.config.entity_types,
             platform=self.config.platform,
             env=self.config.env,
@@ -173,52 +272,41 @@ class SoftDeletedEntitiesCleanup:
             status=RemovedStatusFilter.ONLY_SOFT_DELETED,
             batch_size=self.config.batch_size,
         )
+        yield from self._get_soft_deleted_queries()
+
+    def cleanup_soft_deleted_entities(self) -> None:
+        if not self.config.enabled:
+            return
+        self.start_time = time.time()
 
-        futures = {}
+        futures: Dict[Future, str] = dict()
         with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor:
-            num_urns_submitted = 0
-            for urn in urns:
-                num_urns_submitted += 1
+            for urn in self._get_urns():
+                self._print_report()
+                while len(futures) >= self.config.futures_max_at_time:
+                    futures = self._process_futures(futures)
                 if (
                     self.config.limit_entities_delete
-                    and num_urns_submitted > self.config.limit_entities_delete
+                    and self.report.num_hard_deleted > self.config.limit_entities_delete
                 ):
                     logger.info(
-                        f"Limit of {self.config.limit_entities_delete} entities reached. Stopping"
+                        f"Limit of {self.config.limit_entities_delete} entities reached. Stopped adding more."
                     )
                     break
                 if (
                     self.config.runtime_limit_seconds
-                    and time.time() - start_time > self.config.runtime_limit_seconds
+                    and time.time() - self.start_time
+                    > self.config.runtime_limit_seconds
                 ):
                     logger.info(
-                        f"Runtime limit of {self.config.runtime_limit_seconds} seconds reached. Stopping"
+                        f"Runtime limit of {self.config.runtime_limit_seconds} seconds reached. Not submitting more futures."
                     )
                     break
 
                 future = executor.submit(self.delete_soft_deleted_entity, urn)
                 futures[future] = urn
 
-            if not futures:
-                return
-            for future in as_completed(futures):
-                if future.exception():
-                    logger.error(
-                        f"Failed to delete entity {futures[future]}: {future.exception()}"
-                    )
-                    self.report.failure(
-                        f"Failed to delete entity {futures[future]}",
-                        exc=future.exception(),
-                    )
-                deleted_count_retention += 1
-
-                if deleted_count_retention % self.config.batch_size == 0:
-                    logger.info(
-                        f"Processed {deleted_count_retention} soft deleted entity and deleted {self.report.num_soft_deleted_entity_removed} entities so far"
-                    )
-
-                    if self.config.delay:
-                        logger.debug(
-                            f"Sleeping for {self.config.delay} seconds before getting next batch"
-                        )
-                        time.sleep(self.config.delay)
+            logger.info(f"Waiting for {len(futures)} futures to complete")
+            while len(futures) > 0:
+                self._print_report()
+                futures = self._process_futures(futures)

datahub/ingestion/source/tableau/tableau.py

@@ -109,6 +109,7 @@ from datahub.ingestion.source.tableau.tableau_common import (
     make_filter,
     make_fine_grained_lineage_class,
     make_upstream_class,
+    optimize_query_filter,
     published_datasource_graphql_query,
     query_metadata_cursor_based_pagination,
     sheet_graphql_query,
@@ -1363,6 +1364,8 @@ class TableauSiteSource:
         query_filter: dict = {},
         page_size_override: Optional[int] = None,
     ) -> Iterable[dict]:
+        query_filter = optimize_query_filter(query_filter)
+
         # Calls the get_connection_object_page function to get the objects,
         # and automatically handles pagination.
         page_size = page_size_override or self.config.page_size

datahub/ingestion/source/tableau/tableau_common.py

@@ -1,3 +1,4 @@
+import copy
 import html
 import json
 import logging
@@ -35,6 +36,7 @@ from datahub.metadata.schema_classes import (
     UpstreamClass,
 )
 from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo, SqlParsingResult
+from datahub.utilities.ordered_set import OrderedSet
 
 logger = logging.getLogger(__name__)
 
@@ -1000,3 +1002,19 @@ def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]:
         ]
 
     return filter_pages
+
+
+def optimize_query_filter(query_filter: dict) -> dict:
+    """
+    Duplicates in the filter cause duplicates in the result,
+    leading to entities/aspects being emitted multiple times unnecessarily
+    """
+    optimized_query = copy.deepcopy(query_filter)
+
+    if query_filter.get(c.ID_WITH_IN):
+        optimized_query[c.ID_WITH_IN] = list(OrderedSet(query_filter[c.ID_WITH_IN]))
+    if query_filter.get(c.PROJECT_NAME_WITH_IN):
+        optimized_query[c.PROJECT_NAME_WITH_IN] = list(
+            OrderedSet(query_filter[c.PROJECT_NAME_WITH_IN])
+        )
+    return optimized_query

datahub/metadata/_schema_classes.py

@@ -4053,6 +4053,60 @@ class DataPlatformInstanceClass(_Aspect):
         self._inner_dict['instance'] = value
     
     
+class DataTransformClass(DictWrapper):
+    """Information about a transformation. It may be a query,"""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.common.DataTransform")
+    def __init__(self,
+        queryStatement: Union[None, "QueryStatementClass"]=None,
+    ):
+        super().__init__()
+        
+        self.queryStatement = queryStatement
+    
+    def _restore_defaults(self) -> None:
+        self.queryStatement = self.RECORD_SCHEMA.fields_dict["queryStatement"].default
+    
+    
+    @property
+    def queryStatement(self) -> Union[None, "QueryStatementClass"]:
+        """The data transform may be defined by a query statement"""
+        return self._inner_dict.get('queryStatement')  # type: ignore
+    
+    @queryStatement.setter
+    def queryStatement(self, value: Union[None, "QueryStatementClass"]) -> None:
+        self._inner_dict['queryStatement'] = value
+    
+    
+class DataTransformLogicClass(_Aspect):
+    """Information about a Query against one or more data assets (e.g. Tables or Views)."""
+
+
+    ASPECT_NAME = 'dataTransformLogic'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.common.DataTransformLogic")
+
+    def __init__(self,
+        transforms: List["DataTransformClass"],
+    ):
+        super().__init__()
+        
+        self.transforms = transforms
+    
+    def _restore_defaults(self) -> None:
+        self.transforms = list()
+    
+    
+    @property
+    def transforms(self) -> List["DataTransformClass"]:
+        """List of transformations applied"""
+        return self._inner_dict.get('transforms')  # type: ignore
+    
+    @transforms.setter
+    def transforms(self, value: List["DataTransformClass"]) -> None:
+        self._inner_dict['transforms'] = value
+    
+    
 class DeprecationClass(_Aspect):
     """Deprecation status of an entity"""
 
@@ -14624,7 +14678,7 @@ class DataJobKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'dataJobKey'
-    ASPECT_INFO = {'keyForEntity': 'dataJob', 'entityCategory': '_unset_', 'entityAspects': ['datahubIngestionRunSummary', 'datahubIngestionCheckpoint', 'domains', 'deprecation', 'versionInfo', 'dataJobInfo', 'dataJobInputOutput', 'editableDataJobProperties', 'ownership', 'status', 'globalTags', 'browsePaths', 'glossaryTerms', 'institutionalMemory', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'subTypes', 'incidentsSummary', 'testResults']}
+    ASPECT_INFO = {'keyForEntity': 'dataJob', 'entityCategory': '_unset_', 'entityAspects': ['datahubIngestionRunSummary', 'datahubIngestionCheckpoint', 'domains', 'deprecation', 'versionInfo', 'dataJobInfo', 'dataJobInputOutput', 'editableDataJobProperties', 'ownership', 'status', 'globalTags', 'browsePaths', 'glossaryTerms', 'institutionalMemory', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'subTypes', 'incidentsSummary', 'testResults', 'dataTransformLogic']}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataJobKey")
 
     def __init__(self,
@@ -24715,6 +24769,8 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.common.CostCostDiscriminator': CostCostDiscriminatorClass,
     'com.linkedin.pegasus2avro.common.CostType': CostTypeClass,
     'com.linkedin.pegasus2avro.common.DataPlatformInstance': DataPlatformInstanceClass,
+    'com.linkedin.pegasus2avro.common.DataTransform': DataTransformClass,
+    'com.linkedin.pegasus2avro.common.DataTransformLogic': DataTransformLogicClass,
     'com.linkedin.pegasus2avro.common.Deprecation': DeprecationClass,
     'com.linkedin.pegasus2avro.common.Documentation': DocumentationClass,
     'com.linkedin.pegasus2avro.common.DocumentationAssociation': DocumentationAssociationClass,
@@ -25182,6 +25238,8 @@ __SCHEMA_TYPES = {
     'CostCostDiscriminator': CostCostDiscriminatorClass,
     'CostType': CostTypeClass,
     'DataPlatformInstance': DataPlatformInstanceClass,
+    'DataTransform': DataTransformClass,
+    'DataTransformLogic': DataTransformLogicClass,
     'Deprecation': DeprecationClass,
     'Documentation': DocumentationClass,
     'DocumentationAssociation': DocumentationAssociationClass,
@@ -25588,6 +25646,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     CostClass,
     BrowsePathsClass,
     InstitutionalMemoryClass,
+    DataTransformLogicClass,
     SubTypesClass,
     FormsClass,
     DeprecationClass,
@@ -25802,6 +25861,7 @@ class AspectBag(TypedDict, total=False):
     cost: CostClass
     browsePaths: BrowsePathsClass
     institutionalMemory: InstitutionalMemoryClass
+    dataTransformLogic: DataTransformLogicClass
     subTypes: SubTypesClass
     forms: FormsClass
     deprecation: DeprecationClass

datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -19,6 +19,8 @@ from .....schema_classes import CostCostClass
 from .....schema_classes import CostCostDiscriminatorClass
 from .....schema_classes import CostTypeClass
 from .....schema_classes import DataPlatformInstanceClass
+from .....schema_classes import DataTransformClass
+from .....schema_classes import DataTransformLogicClass
 from .....schema_classes import DeprecationClass
 from .....schema_classes import DocumentationClass
 from .....schema_classes import DocumentationAssociationClass
@@ -79,6 +81,8 @@ CostCost = CostCostClass
 CostCostDiscriminator = CostCostDiscriminatorClass
 CostType = CostTypeClass
 DataPlatformInstance = DataPlatformInstanceClass
+DataTransform = DataTransformClass
+DataTransformLogic = DataTransformLogicClass
 Deprecation = DeprecationClass
 Documentation = DocumentationClass
 DocumentationAssociation = DocumentationAssociationClass

datahub/metadata/schema.avsc

@@ -400,6 +400,69 @@
     ],
     "doc": "Institutional memory of an entity. This is a way to link to relevant documentation and provide description of the documentation. Institutional or tribal knowledge is very important for users to leverage the entity."
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataTransformLogic"
+    },
+    "name": "DataTransformLogic",
+    "namespace": "com.linkedin.pegasus2avro.common",
+    "fields": [
+      {
+        "type": {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "DataTransform",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "QueryStatement",
+                    "namespace": "com.linkedin.pegasus2avro.query",
+                    "fields": [
+                      {
+                        "type": "string",
+                        "name": "value",
+                        "doc": "The query text"
+                      },
+                      {
+                        "type": {
+                          "type": "enum",
+                          "symbolDocs": {
+                            "SQL": "A SQL Query"
+                          },
+                          "name": "QueryLanguage",
+                          "namespace": "com.linkedin.pegasus2avro.query",
+                          "symbols": [
+                            "SQL"
+                          ]
+                        },
+                        "name": "language",
+                        "default": "SQL",
+                        "doc": "The language of the Query, e.g. SQL."
+                      }
+                    ],
+                    "doc": "A query statement against one or more data assets."
+                  }
+                ],
+                "name": "queryStatement",
+                "default": null,
+                "doc": "The data transform may be defined by a query statement"
+              }
+            ],
+            "doc": "Information about a transformation. It may be a query,"
+          }
+        },
+        "name": "transforms",
+        "doc": "List of transformations applied"
+      }
+    ],
+    "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+  },
   {
     "type": "record",
     "Aspect": {
@@ -11947,35 +12010,7 @@
     "namespace": "com.linkedin.pegasus2avro.query",
     "fields": [
       {
-        "type": {
-          "type": "record",
-          "name": "QueryStatement",
-          "namespace": "com.linkedin.pegasus2avro.query",
-          "fields": [
-            {
-              "type": "string",
-              "name": "value",
-              "doc": "The query text"
-            },
-            {
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "SQL": "A SQL Query"
-                },
-                "name": "QueryLanguage",
-                "namespace": "com.linkedin.pegasus2avro.query",
-                "symbols": [
-                  "SQL"
-                ]
-              },
-              "name": "language",
-              "default": "SQL",
-              "doc": "The language of the Query, e.g. SQL."
-            }
-          ],
-          "doc": "A query statement against one or more data assets."
-        },
+        "type": "com.linkedin.pegasus2avro.query.QueryStatement",
         "name": "statement",
         "doc": "The Query Statement."
       },

datahub/metadata/schemas/DataJobKey.avsc

@@ -25,7 +25,8 @@
       "forms",
       "subTypes",
       "incidentsSummary",
-      "testResults"
+      "testResults",
+      "dataTransformLogic"
     ]
   },
   "name": "DataJobKey",

datahub/metadata/schemas/DataTransformLogic.avsc

@@ -0,0 +1,63 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataTransformLogic"
+  },
+  "name": "DataTransformLogic",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "DataTransform",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "QueryStatement",
+                  "namespace": "com.linkedin.pegasus2avro.query",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "value",
+                      "doc": "The query text"
+                    },
+                    {
+                      "type": {
+                        "type": "enum",
+                        "symbolDocs": {
+                          "SQL": "A SQL Query"
+                        },
+                        "name": "QueryLanguage",
+                        "namespace": "com.linkedin.pegasus2avro.query",
+                        "symbols": [
+                          "SQL"
+                        ]
+                      },
+                      "name": "language",
+                      "default": "SQL",
+                      "doc": "The language of the Query, e.g. SQL."
+                    }
+                  ],
+                  "doc": "A query statement against one or more data assets."
+                }
+              ],
+              "name": "queryStatement",
+              "default": null,
+              "doc": "The data transform may be defined by a query statement"
+            }
+          ],
+          "doc": "Information about a transformation. It may be a query,"
+        }
+      },
+      "name": "transforms",
+      "doc": "List of transformations applied"
+    }
+  ],
+  "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+}