openmetadata-ingestion 1.6.7.1__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl

metadata/ingestion/ometa/mixins/domain_mixin.py

@@ -0,0 +1,80 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Domain and Data Product specific operations"""
+from typing import Dict, List
+
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.models.custom_pydantic import BaseModel
+from metadata.ingestion.ometa.client import REST
+
+
+class AssetsRequest(BaseModel):
+    """Request to add assets to a data product"""
+
+    assets: List[EntityReference]
+
+
+class OMetaDomainMixin:
+    """Mixin class containing Data Product specific methods"""
+
+    client: REST
+
+    def add_assets_to_data_product(
+        self, name: str, assets: List[EntityReference]
+    ) -> Dict:
+        """
+        Add assets to a data product
+
+        Args:
+            name: Name of the data product
+            assets: List of entity references to add as assets
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_assets(name, assets, "add")
+
+    def remove_assets_from_data_product(
+        self, name: str, assets: List[EntityReference]
+    ) -> Dict:
+        """
+        Remove assets from a data product
+
+        Args:
+            name: Name of the data product
+            assets: List of entity references to remove from assets
+
+        Returns:
+            API response as a dictionary
+        """
+        return self._handle_data_product_assets(name, assets, "remove")
+
+    def _handle_data_product_assets(
+        self,
+        name: str,
+        assets: List[EntityReference],
+        operation: str,
+    ) -> Dict:
+        """
+        Handle adding or removing assets from a data product
+
+        Args:
+            name: Name of the data product
+            assets: List of entity references to add/remove
+            operation: Operation to perform ("add" or "remove")
+
+        Returns:
+            API response as a dictionary
+        """
+        path = f"/dataProducts/{name}/assets/{operation}"
+        payload = AssetsRequest(assets=assets)
+
+        return self.client.put(path, payload.model_dump_json())

metadata/ingestion/ometa/mixins/es_mixin.py

@@ -16,7 +16,17 @@ To be used by OpenMetadata class
 import functools
 import json
 import traceback
-from typing import Generic, Iterable, Iterator, List, Optional, Set, Type, TypeVar
+from typing import (
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Type,
+    TypeVar,
+    Union,
+)
 from urllib.parse import quote_plus
 
 from pydantic import Field
@@ -29,7 +39,7 @@ from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.client import REST, APIError
 from metadata.ingestion.ometa.utils import quote
 from metadata.ingestion.source.models import TableView
-from metadata.utils.elasticsearch import ES_INDEX_MAP
+from metadata.utils.elasticsearch import ES_INDEX_MAP, get_entity_from_es_result
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -436,6 +446,11 @@ class ESMixin(Generic[T]):
                                                         "tableType": TableType.Dynamic.value
                                                     }
                                                 },
+                                                {
+                                                    "term": {
+                                                        "tableType": TableType.Stream.value
+                                                    }
+                                                },
                                             ]
                                         }
                                     }
@@ -487,3 +502,28 @@ class ESMixin(Generic[T]):
                         schema_name=schema_name,
                         table_name=table_name,
                     )
+
+    def search_in_any_service(
+        self,
+        entity_type: Type[T],
+        fqn_search_string: str,
+        fetch_multiple_entities: bool = False,
+    ) -> Optional[Union[List[Table], Table]]:
+        """
+        fetch table from es when with/without `db_service_name`
+        """
+        try:
+            entity_result = get_entity_from_es_result(
+                entity_list=self.es_search_from_fqn(
+                    entity_type=entity_type,
+                    fqn_search_string=fqn_search_string,
+                ),
+                fetch_multiple_entities=fetch_multiple_entities,
+            )
+            return entity_result
+        except Exception as exc:
+            logger.debug(
+                f"Error to fetch entity: fqn={fqn_search_string} from es: {exc}"
+            )
+            logger.debug(traceback.format_exc())
+        return None

metadata/ingestion/ometa/mixins/lineage_mixin.py

@@ -171,13 +171,9 @@ class OMetaLineageMixin(Generic[T]):
 
         except APIError as err:
             logger.debug(traceback.format_exc())
-            logger.error(
-                "Error %s trying to PUT lineage for %s: %s",
-                err.status_code,
-                data.model_dump_json(),
-                str(err),
-            )
-            raise err
+            error = f"Error {err.status_code} trying to PUT lineage for {data.model_dump_json()}: {str(err)}"
+            logger.error(error)
+            return {"error": error}
 
         from_entity_lineage = self.get_lineage_by_id(
             data.edge.fromEntity.type, str(data.edge.fromEntity.id.root)
@@ -397,6 +393,10 @@ class OMetaLineageMixin(Generic[T]):
                     resp = self.add_lineage(
                         lineage_request.right, check_patch=check_patch
                     )
+                    if resp.get("error"):
+                        logger.error(resp["error"])
+                        continue
+
                     entity_name = resp.get("entity", {}).get("name")
                     for node in resp.get("nodes", []):
                         logger.info(
@@ -407,37 +407,11 @@ class OMetaLineageMixin(Generic[T]):
                         f"Error while adding lineage: {lineage_request.left.error}"
                     )
 
-    def _set_permissions(self):
-        try:
-            if not hasattr(self, "is_edit_table_allowed"):
-                permissions = self.client.get(path="/permissions")
-                for permission in permissions.get("data", []):
-                    if permission.get("resource") != "table":
-                        continue
-                    for action in permission.get("permissions", []):
-                        if (
-                            action.get("operation") == "EditAll"
-                            and action.get("access") == "allow"
-                        ):
-                            self.is_edit_table_allowed = True
-                            return
-                logger.warning("Please grant EditAll permission to the user")
-        except Exception as exc:
-            logger.debug(f"Error while setting permissions: {exc}")
-            logger.debug(traceback.format_exc())
-        self.is_edit_table_allowed = False
-
     def patch_lineage_processed_flag(
         self,
         entity: Type[T],
         fqn: str,
     ) -> None:
-        """
-        Patch the processed lineage flag for an entity.
-        """
-        self._set_permissions()
-        if not self.is_edit_table_allowed:
-            return
 
         try:
             original_entity = self.get_by_name(entity=entity, fqn=fqn)

metadata/ingestion/ometa/mixins/patch_mixin.py

@@ -29,6 +29,9 @@ from metadata.generated.schema.entity.domains.domain import Domain
 from metadata.generated.schema.entity.services.connections.testConnectionResult import (
     TestConnectionResult,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.reverseIngestionResponse import (
+    ReverseIngestionResponse,
+)
 from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
 from metadata.generated.schema.type.basic import EntityLink, Markdown
 from metadata.generated.schema.type.entityReference import EntityReference
@@ -488,7 +491,7 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
     def patch_automation_workflow_response(
         self,
         automation_workflow: AutomationWorkflow,
-        test_connection_result: TestConnectionResult,
+        result: Union[TestConnectionResult, ReverseIngestionResponse],
         workflow_status: WorkflowStatus,
     ) -> None:
         """
@@ -496,14 +499,21 @@ class OMetaPatchMixin(OMetaPatchMixinBase):
         """
         result_data: Dict = {
             PatchField.PATH: PatchPath.RESPONSE,
-            PatchField.VALUE: test_connection_result.model_dump(),
+            PatchField.VALUE: result.model_dump(),
             PatchField.OPERATION: PatchOperation.ADD,
         }
 
         # for deserializing into json convert enum object to string
-        result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][
-            "status"
-        ].value
+        if isinstance(result, TestConnectionResult):
+            result_data[PatchField.VALUE]["status"] = result_data[PatchField.VALUE][
+                "status"
+            ].value
+        else:
+            # Convert UUID in string
+            data = result_data[PatchField.VALUE]
+            data["serviceId"] = str(data["serviceId"])
+            for operation_result in data["results"]:
+                operation_result["id"] = str(operation_result["id"])
 
         status_data: Dict = {
             PatchField.PATH: PatchPath.STATUS,

metadata/ingestion/ometa/mixins/pipeline_mixin.py

@@ -21,6 +21,7 @@ from metadata.generated.schema.entity.data.pipeline import (
     PipelineStatus,
     Task,
 )
+from metadata.generated.schema.type.usageRequest import UsageRequest
 from metadata.ingestion.ometa.client import REST
 from metadata.utils.logger import ometa_logger
 
@@ -83,6 +84,7 @@ class OMetaPipelineMixin:
             sourceUrl=pipeline.sourceUrl,
             concurrency=pipeline.concurrency,
             pipelineLocation=pipeline.pipelineLocation,
+            state=pipeline.state,
             startDate=pipeline.startDate,
             service=pipeline.service.fullyQualifiedName,
             tasks=all_tasks,
@@ -110,6 +112,7 @@ class OMetaPipelineMixin:
             sourceUrl=pipeline.sourceUrl,
             concurrency=pipeline.concurrency,
             pipelineLocation=pipeline.pipelineLocation,
+            state=pipeline.state,
             startDate=pipeline.startDate,
             service=pipeline.service.fullyQualifiedName,
             tasks=[task for task in pipeline.tasks if task.name in task_ids],
@@ -118,3 +121,18 @@ class OMetaPipelineMixin:
         )
 
         return self.create_or_update(updated_pipeline)
+
+    def publish_pipeline_usage(
+        self, pipeline: Pipeline, pipeline_usage_request: UsageRequest
+    ) -> None:
+        """
+        POST usage details for a Pipeline
+
+        :param pipeline: Pipeline Entity to update
+        :param pipeline_usage_request: Usage data to add
+        """
+        resp = self.client.put(
+            f"/usage/pipeline/{pipeline.id.root}",
+            data=pipeline_usage_request.model_dump_json(),
+        )
+        logger.debug("Published pipeline usage %s", resp)

metadata/ingestion/ometa/mixins/query_mixin.py

@@ -15,14 +15,20 @@ To be used by OpenMetadata class
 """
 import hashlib
 import json
+from functools import lru_cache
 from typing import List, Optional, Union
 
 from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
+from metadata.generated.schema.api.data.createQueryCostRecord import (
+    CreateQueryCostRecordRequest,
+)
 from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.data.query import Query
+from metadata.generated.schema.entity.data.queryCostRecord import QueryCostRecord
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.type.basic import Uuid
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.generated.schema.type.tableUsageCount import QueryCostWrapper
 from metadata.ingestion.lineage.masker import mask_query
 from metadata.ingestion.ometa.client import REST
 from metadata.ingestion.ometa.utils import model_str
@@ -115,3 +121,40 @@ class OMetaQueryMixin:
         if res and res.get("data"):
             return [Query(**query) for query in res.get("data")]
         return None
+
+    @lru_cache(maxsize=5000)
+    def __get_query_by_hash(
+        self, query_hash: str, service_name: str
+    ) -> Optional[Query]:
+        return self.get_by_name(entity=Query, fqn=f"{service_name}.{query_hash}")
+
+    def publish_query_cost(self, query_cost_data: QueryCostWrapper, service_name: str):
+        """
+        Create Query Cost Record
+
+        Args:
+            query_cost_record: QueryCostWrapper
+        """
+
+        masked_query = mask_query(query_cost_data.query, query_cost_data.dialect)
+
+        query_hash = self._get_query_hash(masked_query)
+
+        query = self.__get_query_by_hash(
+            query_hash=query_hash, service_name=service_name
+        )
+        if not query:
+            return None
+
+        create_request = CreateQueryCostRecordRequest(
+            timestamp=int(query_cost_data.date),
+            jsonSchema="queryCostRecord",
+            queryReference=EntityReference(id=query.id.root, type="query"),
+            cost=query_cost_data.cost,
+            count=query_cost_data.count,
+            totalDuration=query_cost_data.totalDuration,
+        )
+
+        return self.client.post(
+            self.get_suffix(QueryCostRecord), data=create_request.model_dump_json()
+        )

metadata/ingestion/ometa/ometa_api.py

@@ -17,7 +17,7 @@ working with OpenMetadata entities.
 import traceback
 from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
 
-from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from metadata.generated.schema.api.createBot import CreateBot
 from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import (
@@ -30,6 +30,7 @@ from metadata.generated.schema.type import basic
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName
 from metadata.generated.schema.type.entityHistory import EntityVersionHistory
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.models.custom_pydantic import BaseModel
 from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
 from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
 from metadata.ingestion.ometa.mixins.custom_property_mixin import (
@@ -37,6 +38,7 @@ from metadata.ingestion.ometa.mixins.custom_property_mixin import (
 )
 from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
 from metadata.ingestion.ometa.mixins.data_insight_mixin import DataInsightMixin
+from metadata.ingestion.ometa.mixins.domain_mixin import OMetaDomainMixin
 from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
 from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
     OMetaIngestionPipelineMixin,
@@ -89,6 +91,16 @@ class EmptyPayloadException(Exception):
     """
 
 
+class OpenMetadataSettings(BaseSettings):
+    """OpenMetadataConnection settings wrapper"""
+
+    model_config = SettingsConfigDict(
+        env_prefix="OPENMETADATA__", env_nested_delimiter="__", case_sensitive=True
+    )
+
+    connection: OpenMetadataConnection
+
+
 class OpenMetadata(
     OMetaPipelineMixin,
     OMetaMlModelMixin,
@@ -109,6 +121,7 @@ class OpenMetadata(
     OMetaSearchIndexMixin,
     OMetaCustomPropertyMixin,
     OMetaSuggestionsMixin,
+    OMetaDomainMixin,
     Generic[T, C],
 ):
     """
@@ -145,11 +158,14 @@ class OpenMetadata(
 
         get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
 
+        extra_headers: Optional[dict[str, str]] = None
+        if self.config.extraHeaders:
+            extra_headers = self.config.extraHeaders.root
         client_config: ClientConfig = ClientConfig(
             base_url=self.config.hostPort,
             api_version=self.config.apiVersion,
             auth_header="Authorization",
-            extra_headers=self.config.extraHeaders,
+            extra_headers=extra_headers,
             auth_token=self._auth_provider.get_access_token,
             verify=get_verify_ssl(self.config.sslConfig),
         )
@@ -158,6 +174,11 @@ class OpenMetadata(
         if self.config.enableVersionValidation:
             self.validate_versions()
 
+    @classmethod
+    def from_env(cls) -> "OpenMetadata":
+        settings = OpenMetadataSettings()
+        return cls(settings.connection)
+
     @staticmethod
     def get_suffix(entity: Type[T]) -> str:
         """
@@ -268,7 +289,11 @@ class OpenMetadata(
             )
 
         fn = getattr(self.client, method)
-        resp = fn(self.get_suffix(entity), data=data.model_dump_json())
+        resp = fn(
+            # this might be a regular pydantic model so we build the context manually
+            self.get_suffix(entity),
+            data=data.model_dump_json(context={"mask_secrets": False}),
+        )
         if not resp:
             raise EmptyPayloadException(
                 f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.model_dump_json()}"

metadata/ingestion/ometa/routes.py

@@ -46,6 +46,9 @@ from metadata.generated.schema.api.data.createMetric import CreateMetricRequest
 from metadata.generated.schema.api.data.createMlModel import CreateMlModelRequest
 from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
 from metadata.generated.schema.api.data.createQuery import CreateQueryRequest
+from metadata.generated.schema.api.data.createQueryCostRecord import (
+    CreateQueryCostRecordRequest,
+)
 from metadata.generated.schema.api.data.createSearchIndex import (
     CreateSearchIndexRequest,
 )
@@ -133,6 +136,7 @@ from metadata.generated.schema.entity.data.metric import Metric
 from metadata.generated.schema.entity.data.mlmodel import MlModel
 from metadata.generated.schema.entity.data.pipeline import Pipeline
 from metadata.generated.schema.entity.data.query import Query
+from metadata.generated.schema.entity.data.queryCostRecord import QueryCostRecord
 from metadata.generated.schema.entity.data.report import Report
 from metadata.generated.schema.entity.data.searchIndex import SearchIndex
 from metadata.generated.schema.entity.data.storedProcedure import StoredProcedure
@@ -280,4 +284,7 @@ ROUTES = {
     CreateAppMarketPlaceDefinitionRequest.__name__: "/apps/marketplace",
     # Settings
     Settings.__name__: "/system/settings",
+    # Query Cost
+    QueryCostRecord.__name__: "/queryCostRecord",
+    CreateQueryCostRecordRequest.__name__: "/queryCostRecord",
 }

metadata/ingestion/ometa/utils.py

@@ -20,6 +20,7 @@ from pydantic import BaseModel
 from requests.utils import quote as url_quote
 
 from metadata.generated.schema.type.basic import FullyQualifiedEntityName
+from metadata.generated.schema.type.entityReference import EntityReference
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -85,3 +86,15 @@ def quote(fqn: Union[FullyQualifiedEntityName, str]) -> str:
     E.g., `"foo.bar/baz"` -> `%22foo.bar%2Fbaz%22`
     """
     return url_quote(model_str(fqn), safe="")
+
+
+def build_entity_reference(entity: T) -> EntityReference:
+    """Get the EntityReference from the Entity itself"""
+    return EntityReference(
+        id=entity.id,
+        type=get_entity_type(type(entity)),
+        name=model_str(entity.name),
+        fullyQualifiedName=model_str(entity.fullyQualifiedName),
+        description=entity.description,
+        href=entity.href,
+    )

metadata/ingestion/processor/query_parser.py

@@ -66,6 +66,7 @@ def parse_sql_statement(record: TableQuery, dialect: Dialect) -> Optional[Parsed
         date=str(start_time),
         serviceName=record.serviceName,
         duration=record.duration,
+        cost=record.cost,
     )
 
 

metadata/ingestion/sink/metadata_rest.py

@@ -34,7 +34,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
 from metadata.generated.schema.dataInsight.kpi.basic import KpiResult
 from metadata.generated.schema.entity.classification.tag import Tag
 from metadata.generated.schema.entity.data.dashboard import Dashboard
-from metadata.generated.schema.entity.data.pipeline import PipelineStatus
+from metadata.generated.schema.entity.data.pipeline import Pipeline, PipelineStatus
 from metadata.generated.schema.entity.data.searchIndex import (
     SearchIndex,
     SearchIndexSampleData,
@@ -83,6 +83,7 @@ from metadata.ingestion.ometa.client import APIError
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardUsage
 from metadata.ingestion.source.database.database_service import DataModelLink
+from metadata.ingestion.source.pipeline.pipeline_service import PipelineUsage
 from metadata.profiler.api.models import ProfilerResponse
 from metadata.sampler.models import SamplerResponse
 from metadata.utils.execution_time_tracker import calculate_execution_time
@@ -250,6 +251,13 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
     @_run_dispatch.register
     def write_lineage(self, add_lineage: AddLineageRequest) -> Either[Dict[str, Any]]:
         created_lineage = self.metadata.add_lineage(add_lineage, check_patch=True)
+        if created_lineage.get("error"):
+            return Either(
+                left=StackTraceError(
+                    name="AddLineageRequestError", error=created_lineage["error"]
+                )
+            )
+
         return Either(right=created_lineage["entity"]["fullyQualifiedName"])
 
     @_run_dispatch.register
@@ -634,6 +642,18 @@ class MetadataRestSink(Sink):  # pylint: disable=too-many-public-methods
 
         return Either(right=record)
 
+    @_run_dispatch.register
+    def write_pipeline_usage(self, pipeline_usage: PipelineUsage) -> Either[Pipeline]:
+        """
+        Send a UsageRequest update to a pipeline entity
+        :param pipeline_usage: pipeline entity and usage request
+        """
+        self.metadata.publish_pipeline_usage(
+            pipeline=pipeline_usage.pipeline,
+            pipeline_usage_request=pipeline_usage.usage,
+        )
+        return Either(right=pipeline_usage.pipeline)
+
     def close(self):
         """
         We don't have anything to close since we are using the given metadata client

metadata/ingestion/source/dashboard/dashboard_service.py

@@ -198,6 +198,9 @@ class DashboardServiceTopology(ServiceTopology):
     )
 
 
+from metadata.utils.helpers import retry_with_docker_host
+
+
 # pylint: disable=too-many-public-methods
 class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     """
@@ -216,6 +219,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     dashboard_source_state: Set = set()
     datamodel_source_state: Set = set()
 
+    @retry_with_docker_host()
     def __init__(
         self,
         config: WorkflowSource,
@@ -248,7 +252,9 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
 
     @abstractmethod
     def yield_dashboard_lineage_details(
-        self, dashboard_details: Any, db_service_name: str
+        self,
+        dashboard_details: Any,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between dashboard and data sources
@@ -368,6 +374,8 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
                 yield lineage
 
         db_service_names = self.get_db_service_names()
+        if not db_service_names:
+            yield from self.yield_dashboard_lineage_details(dashboard_details) or []
         for db_service_name in db_service_names or []:
             yield from self.yield_dashboard_lineage_details(
                 dashboard_details, db_service_name

metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -248,6 +248,8 @@ class DomodashboardSource(DashboardServiceSource):
                 )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: dict, db_service_name
+        self,
+        dashboard_details: dict,
+        db_service_name: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """No lineage implemented"""

metadata/ingestion/source/dashboard/lightdash/metadata.py

@@ -176,7 +176,7 @@ class LightdashSource(DashboardServiceSource):
     def yield_dashboard_lineage_details(
         self,
         dashboard_details: LightdashDashboard,
-        db_service_name: Optional[str],
+        db_service_name: Optional[str] = None,
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Get lineage method
 

metadata/ingestion/source/dashboard/looker/metadata.py

@@ -185,7 +185,7 @@ class LookerSource(DashboardServiceSource):
     Its client uses Looker 40 from the SDK: client = looker_sdk.init40()
     """
 
-    # pylint: disable=too-many-instance-attributes
+    # pylint: disable=too-many-instance-attributes, too-many-public-methods
 
     config: WorkflowSource
     metadata: OpenMetadata
@@ -924,7 +924,9 @@ class LookerSource(DashboardServiceSource):
         )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: LookerDashboard, _: str
+        self,
+        dashboard_details: LookerDashboard,
+        _: Optional[str] = None,
     ) -> Iterable[Either[AddLineageRequest]]:
         """
         Get lineage between charts and data sources.
@@ -1196,3 +1198,7 @@ class LookerSource(DashboardServiceSource):
                     stackTrace=traceback.format_exc(),
                 )
             )
+
+    def close(self):
+        self.metadata.compute_percentile(Dashboard, self.today)
+        self.metadata.close()