openmetadata-ingestion 1.8.0.0__py3-none-any.whl → 1.8.2.0__py3-none-any.whl

metadata/ingestion/source/database/databricks/client.py

@@ -14,14 +14,23 @@ Client to interact with databricks apis
 import json
 import traceback
 from datetime import timedelta
-from typing import Iterable, List
+from typing import Iterable, List, Optional, Tuple, Union
 
 import requests
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
 
 from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
     DatabricksConnection,
 )
+from metadata.generated.schema.entity.services.connections.pipeline.databricksPipelineConnection import (
+    DatabricksPipelineConnection,
+)
 from metadata.ingestion.ometa.client import APIError
+from metadata.ingestion.source.database.databricks.queries import (
+    DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB,
+    DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB,
+)
 from metadata.utils.constants import QUERY_WITH_DBT, QUERY_WITH_OM_VERSION
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger
@@ -30,6 +39,8 @@ logger = ingestion_logger()
 API_TIMEOUT = 10
 PAGE_SIZE = 100
 QUERIES_PATH = "/sql/history/queries"
+API_VERSION = "/api/2.0"
+JOB_API_VERSION = "/api/2.1"
 
 
 class DatabricksClientException(Exception):
@@ -43,30 +54,58 @@ class DatabricksClient:
     DatabricksClient creates a Databricks connection based on DatabricksCredentials.
     """
 
-    def __init__(self, config: DatabricksConnection):
+    def __init__(
+        self,
+        config: Union[DatabricksConnection, DatabricksPipelineConnection],
+        engine: Optional[Engine] = None,
+    ):
         self.config = config
         base_url, *_ = self.config.hostPort.split(":")
-        api_version = "/api/2.0"
-        job_api_version = "/api/2.1"
         auth_token = self.config.token.get_secret_value()
-        self.base_url = f"https://{base_url}{api_version}"
+        self.base_url = f"https://{base_url}{API_VERSION}"
         self.base_query_url = f"{self.base_url}{QUERIES_PATH}"
-        self.base_job_url = f"https://{base_url}{job_api_version}/jobs"
+        self.base_job_url = f"https://{base_url}{JOB_API_VERSION}/jobs"
         self.jobs_list_url = f"{self.base_job_url}/list"
         self.jobs_run_list_url = f"{self.base_job_url}/runs/list"
         self.headers = {
             "Authorization": f"Bearer {auth_token}",
             "Content-Type": "application/json",
         }
+        self.api_timeout = self.config.connectionTimeout or 120
+        self.job_table_lineage: dict[str, list[dict[str, str]]] = {}
+        self.job_column_lineage: dict[
+            str, dict[Tuple[str, str], list[Tuple[str, str]]]
+        ] = {}
+        self.engine = engine
         self.client = requests
 
     def test_query_api_access(self) -> None:
         res = self.client.get(
-            self.base_query_url, headers=self.headers, timeout=API_TIMEOUT
+            self.base_query_url, headers=self.headers, timeout=self.api_timeout
         )
         if res.status_code != 200:
             raise APIError(res.json)
 
+    def test_lineage_query(self) -> None:
+        try:
+            with self.engine.connect() as connection:
+                test_table_lineage = connection.execute(
+                    text(DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB + " LIMIT 1")
+                )
+                test_column_lineage = connection.execute(
+                    text(DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB + " LIMIT 1")
+                )
+                # Check if queries executed successfully by fetching results
+                table_result = test_table_lineage.fetchone()
+                column_result = test_column_lineage.fetchone()
+                logger.info("Lineage queries executed successfully")
+        except Exception as exc:
+            logger.debug(f"Error testing lineage queries: {traceback.format_exc()}")
+            raise DatabricksClientException(
+                f"Failed to test lineage queries Make sure you have access"
+                "to the tables table_lineage and column_lineage: {exc}"
+            )
+
     def _run_query_paginator(self, data, result, end_time, response):
         while True:
             if response:
@@ -85,7 +124,7 @@ class DatabricksClient:
                     self.base_query_url,
                     data=json.dumps(data),
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
                 yield from response.get("res") or []
 
@@ -117,7 +156,7 @@ class DatabricksClient:
                         self.base_query_url,
                         data=json.dumps(data),
                         headers=self.headers,
-                        timeout=API_TIMEOUT,
+                        timeout=self.api_timeout,
                     ).json()
 
                     result = response.get("res") or []
@@ -145,7 +184,7 @@ class DatabricksClient:
             self.jobs_list_url,
             data=json.dumps(data),
             headers=self.headers,
-            timeout=API_TIMEOUT,
+            timeout=self.api_timeout,
         )
         if response.status_code != 200:
             raise DatabricksClientException(response.text)
@@ -154,6 +193,7 @@ class DatabricksClient:
         """
         Method returns List all the created jobs in a Databricks Workspace
         """
+        self.cache_lineage()
         try:
             iteration_count = 1
             data = {"limit": PAGE_SIZE, "expand_tasks": True, "offset": 0}
@@ -162,7 +202,7 @@ class DatabricksClient:
                 self.jobs_list_url,
                 data=json.dumps(data),
                 headers=self.headers,
-                timeout=API_TIMEOUT,
+                timeout=self.api_timeout,
             ).json()
 
             yield from response.get("jobs") or []
@@ -174,7 +214,7 @@ class DatabricksClient:
                     self.jobs_list_url,
                     data=json.dumps(data),
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
                 iteration_count += 1
                 yield from response.get("jobs") or []
@@ -200,7 +240,7 @@ class DatabricksClient:
                 self.jobs_run_list_url,
                 params=params,
                 headers=self.headers,
-                timeout=API_TIMEOUT,
+                timeout=self.api_timeout,
             ).json()
 
             yield from response.get("runs") or []
@@ -212,7 +252,7 @@ class DatabricksClient:
                     self.jobs_run_list_url,
                     params=params,
                     headers=self.headers,
-                    timeout=API_TIMEOUT,
+                    timeout=self.api_timeout,
                 ).json()
 
                 yield from response.get("runs") or []
@@ -220,3 +260,98 @@ class DatabricksClient:
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.error(exc)
+
+    def get_table_lineage(self, job_id: str) -> List[dict[str, str]]:
+        """
+        Method returns table lineage for a job by the specified job_id
+        """
+        try:
+            return self.job_table_lineage.get(str(job_id))
+        except Exception as exc:
+            logger.debug(
+                f"Error getting table lineage for job {job_id} due to {traceback.format_exc()}"
+            )
+            logger.error(exc)
+        return []
+
+    def get_column_lineage(
+        self, job_id: str, TableKey: Tuple[str, str]
+    ) -> List[Tuple[str, str]]:
+        """
+        Method returns column lineage for a job by the specified job_id and table key
+        """
+        try:
+            return self.job_column_lineage.get(str(job_id), {}).get(TableKey)
+        except Exception as exc:
+            logger.debug(
+                f"Error getting column lineage for table {TableKey} due to {traceback.format_exc()}"
+            )
+            logger.error(exc)
+        return []
+
+    def run_lineage_query(self, query: str) -> List[dict]:
+        """
+        Method runs a lineage query and returns the result
+        """
+        try:
+            with self.engine.connect() as connection:
+                result = connection.execute(text(query))
+                return result
+
+        except Exception as exc:
+            logger.debug(f"Error caching table lineage due to {traceback.format_exc()}")
+            logger.error(exc)
+        return []
+
+    def cache_lineage(self):
+        """
+        Method caches table and column lineage for a job by the specified job_id
+        """
+        logger.info(f"Caching table lineage")
+        table_lineage = self.run_lineage_query(DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB)
+        if table_lineage:
+            for row in table_lineage:
+                try:
+                    if row.job_id not in self.job_table_lineage:
+                        self.job_table_lineage[row.job_id] = []
+                    self.job_table_lineage[row.job_id].append(
+                        {
+                            "source_table_full_name": row.source_table_full_name,
+                            "target_table_full_name": row.target_table_full_name,
+                        }
+                    )
+                except Exception as exc:
+                    logger.debug(
+                        f"Error parsing row: {row} due to {traceback.format_exc()}"
+                    )
+                    continue
+
+        # Not every job has column lineage, so we need to check if the job exists in the column_lineage table
+        # we will cache the column lineage for jobs that have column lineage
+        logger.info("Caching column lineage")
+        column_lineage = self.run_lineage_query(DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB)
+        if column_lineage:
+            for row in column_lineage:
+                try:
+                    table_key = (
+                        row.source_table_full_name,
+                        row.target_table_full_name,
+                    )
+                    column_pair = (
+                        row.source_column_name,
+                        row.target_column_name,
+                    )
+
+                    if row.job_id not in self.job_column_lineage:
+                        self.job_column_lineage[row.job_id] = {}
+
+                    if table_key not in self.job_column_lineage[row.job_id]:
+                        self.job_column_lineage[row.job_id][table_key] = []
+
+                    self.job_column_lineage[row.job_id][table_key].append(column_pair)
+
+                except Exception as exc:
+                    logger.debug(
+                        f"Error parsing row: {row} due to {traceback.format_exc()}"
+                    )
+                    continue

metadata/ingestion/source/database/databricks/queries.py

@@ -86,3 +86,29 @@ DATABRICKS_GET_COLUMN_TAGS = textwrap.dedent(
 )
 
 DATABRICKS_DDL = "SHOW CREATE TABLE `{table_name}`"
+
+DATABRICKS_GET_TABLE_LINEAGE_FOR_JOB = textwrap.dedent(
+    """
+    SELECT DISTINCT
+        entity_id as job_id,
+        entity_type as job_type,
+        source_table_full_name as source_table_full_name,
+        target_table_full_name as target_table_full_name
+    FROM system.access.table_lineage
+    WHERE entity_type ILIKE 'job'
+    """
+)
+
+DATABRICKS_GET_COLUMN_LINEAGE_FOR_JOB = textwrap.dedent(
+    """
+    SELECT DISTINCT
+        entity_id as job_id,
+        entity_type as job_type,
+        source_table_full_name as source_table_full_name,
+        source_column_name as source_column_name,
+        target_table_full_name as target_table_full_name,
+        target_column_name as target_column_name
+    FROM system.access.column_lineage
+    WHERE entity_type ILIKE 'job'
+    """
+)

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -12,6 +12,7 @@
 Hosts the singledispatch to get DBT files
 """
 import json
+import os
 import traceback
 from collections import defaultdict
 from functools import singledispatch
@@ -78,11 +79,7 @@ def _(config: DbtLocalConfig):
     try:
         blob_grouped_by_directory = defaultdict(list)
 
-        subdirectory = (
-            config.dbtManifestFilePath.rsplit("/", 1)[0]
-            if "/" in config.dbtManifestFilePath
-            else ""
-        )
+        subdirectory = os.path.dirname(config.dbtManifestFilePath)
         blob_grouped_by_directory[subdirectory] = [
             config.dbtManifestFilePath,
             config.dbtCatalogFilePath,
@@ -247,8 +244,8 @@ def get_blobs_grouped_by_dir(blobs: List[str]) -> Dict[str, List[str]]:
     """
     blob_grouped_by_directory = defaultdict(list)
     for blob in blobs:
-        subdirectory = blob.rsplit("/", 1)[0] if "/" in blob else ""
-        blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+        subdirectory = os.path.dirname(blob)
+        blob_file_name = os.path.basename(blob)
         # We'll be processing multiple run_result files from a single dir
         # Grouping them together to process them in a single go
         if (
@@ -283,7 +280,7 @@ def download_dbt_files(
             for blob in blobs:
                 if blob:
                     reader = get_reader(config_source=config, client=client)
-                    blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+                    blob_file_name = os.path.basename(blob)
                     if DBT_MANIFEST_FILE_NAME == blob_file_name.lower():
                         logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}")
                         dbt_manifest = reader.read(path=blob, **kwargs)

metadata/ingestion/source/database/lineage_source.py

@@ -49,6 +49,7 @@ from metadata.ingestion.source.database.query_parser_source import QueryParserSo
 from metadata.ingestion.source.models import TableView
 from metadata.utils import fqn
 from metadata.utils.db_utils import get_view_lineage
+from metadata.utils.filters import filter_by_table
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -313,6 +314,15 @@ class LineageSource(QueryParserSource, ABC):
     ) -> Iterable[Either[AddLineageRequest]]:
         try:
             for view in views:
+                if filter_by_table(
+                    self.source_config.tableFilterPattern,
+                    view.table_name,
+                ):
+                    self.status.filter(
+                        view.table_name,
+                        "View Filtered Out",
+                    )
+                    continue
                 for lineage in get_view_lineage(
                     view=view,
                     metadata=self.metadata,

metadata/ingestion/source/database/postgres/metadata.py

@@ -40,7 +40,11 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+)
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -300,9 +304,15 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         """List stored procedures"""
         if self.source_config.includeStoredProcedures:
             yield from self._get_stored_procedures_internal(
-                POSTGRES_GET_STORED_PROCEDURES
+                POSTGRES_GET_STORED_PROCEDURES.format(
+                    schema_name=self.context.get().database_schema
+                )
+            )
+            yield from self._get_stored_procedures_internal(
+                POSTGRES_GET_FUNCTIONS.format(
+                    schema_name=self.context.get().database_schema
+                )
             )
-            yield from self._get_stored_procedures_internal(POSTGRES_GET_FUNCTIONS)
 
     def yield_stored_procedure(
         self, stored_procedure
@@ -311,7 +321,9 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
         try:
             stored_procedure_request = CreateStoredProcedureRequest(
                 name=EntityName(stored_procedure.name),
-                description=None,
+                description=Markdown(stored_procedure.description)
+                if stored_procedure.description
+                else None,
                 storedProcedureCode=StoredProcedureCode(
                     language=STORED_PROC_LANGUAGE_MAP.get(stored_procedure.language),
                     code=stored_procedure.definition,

metadata/ingestion/source/database/postgres/models.py

@@ -24,3 +24,4 @@ class PostgresStoredProcedure(BaseModel):
     definition: str
     language: Optional[str] = None
     procedure_type: Optional[str] = Field(None, alias="procedure_type")
+    description: Optional[str] = Field(None, alias="description")

metadata/ingestion/source/database/postgres/queries.py

@@ -225,10 +225,12 @@ POSTGRES_GET_STORED_PROCEDURES = """
         proargtypes AS argument_types,
         prorettype::regtype AS return_type,
         prosrc AS definition,
-        'StoredProcedure' as procedure_type
+        'StoredProcedure' as procedure_type,
+        obj_description(pg_proc.oid, 'pg_proc') AS description
     FROM pg_proc
     JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid
-    WHERE prokind = 'p';
+    WHERE prokind = 'p'
+    and pg_namespace.nspname = '{schema_name}';
 """
 
 POSTGRES_GET_FUNCTIONS = """
@@ -238,11 +240,12 @@ SELECT
     proargtypes AS argument_types,
     prorettype :: regtype AS return_type,
     prosrc AS definition,
-    'Function' as procedure_type
+    'Function' as procedure_type,
+    obj_description(pg_proc.oid, 'pg_proc') AS description
 FROM
     pg_proc
     JOIN pg_namespace ON pg_proc.pronamespace = pg_namespace.oid
 WHERE
     prokind = 'f'
-    and pg_namespace.nspname NOT IN ('pg_catalog', 'information_schema');
+    and pg_namespace.nspname = '{schema_name}';
 """

metadata/ingestion/source/database/redshift/queries.py

@@ -15,9 +15,6 @@ SQL Queries used during ingestion
 import textwrap
 from typing import List
 
-from sqlalchemy import text
-from sqlalchemy.orm.session import Session
-
 from metadata.utils.profiler_utils import QueryResult
 from metadata.utils.time_utils import datetime_to_timestamp
 
@@ -418,38 +415,6 @@ STL_QUERY = """
 """
 
 
-def get_query_results(
-    session: Session,
-    query,
-    operation,
-) -> List[QueryResult]:
-    """get query results either from cache or from the database
-
-    Args:
-        session (Session): session
-        query (_type_): query
-        operation (_type_): operation
-
-    Returns:
-        List[QueryResult]:
-    """
-    cursor = session.execute(text(query))
-    results = [
-        QueryResult(
-            database_name=row.database,
-            schema_name=row.schema,
-            table_name=row.table,
-            query_text=None,
-            query_type=operation,
-            start_time=row.starttime,
-            rows=row.rows,
-        )
-        for row in cursor
-    ]
-
-    return results
-
-
 def get_metric_result(ddls: List[QueryResult], table_name: str) -> List:
     """Given query results, retur the metric result
 

metadata/ingestion/source/database/redshift/service_spec.py

@@ -1,14 +1,14 @@
 from metadata.ingestion.source.database.redshift.lineage import RedshiftLineageSource
 from metadata.ingestion.source.database.redshift.metadata import RedshiftSource
-from metadata.ingestion.source.database.redshift.profiler.profiler import (
-    RedshiftProfiler,
-)
 from metadata.ingestion.source.database.redshift.usage import RedshiftUsageSource
+from metadata.profiler.interface.sqlalchemy.redshift.profiler_interface import (
+    RedshiftProfilerInterface,
+)
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
 ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=RedshiftSource,
     lineage_source_class=RedshiftLineageSource,
     usage_source_class=RedshiftUsageSource,
-    profiler_class=RedshiftProfiler,
+    profiler_class=RedshiftProfilerInterface,
 )

metadata/ingestion/source/database/snowflake/metadata.py

@@ -16,8 +16,9 @@ import traceback
 from datetime import datetime
 from typing import Iterable, List, Optional, Tuple
 
+import sqlalchemy.types as sqltypes
 import sqlparse
-from snowflake.sqlalchemy.custom_types import VARIANT
+from snowflake.sqlalchemy.custom_types import VARIANT, StructuredType
 from snowflake.sqlalchemy.snowdialect import SnowflakeDialect, ischema_names
 from sqlalchemy.engine.reflection import Inspector
 from sqlparse.sql import Function, Identifier, Token
@@ -124,15 +125,36 @@ from metadata.utils.sqlalchemy_utils import (
 )
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
 
+
+class MAP(StructuredType):
+    __visit_name__ = "MAP"
+
+    # Default to VARCHAR for key and value types if not provided
+    # This is a workaround to avoid the error:
+    # sqlalchemy.exc.ArgumentError: Map type requires a key_type and value_type
+    # when creating a table with a MAP column.
+    def __init__(
+        self,
+        key_type: sqltypes.TypeEngine = sqltypes.VARCHAR,
+        value_type: sqltypes.TypeEngine = sqltypes.VARCHAR,
+        not_null: bool = False,
+    ):
+        self.key_type = key_type
+        self.value_type = value_type
+        self.not_null = not_null
+        super().__init__()
+
+
 ischema_names["VARIANT"] = VARIANT
 ischema_names["GEOGRAPHY"] = create_sqlalchemy_type("GEOGRAPHY")
 ischema_names["GEOMETRY"] = create_sqlalchemy_type("GEOMETRY")
 ischema_names["VECTOR"] = create_sqlalchemy_type("VECTOR")
+ischema_names["MAP"] = MAP
 
 logger = ingestion_logger()
 
-
-SnowflakeDialect._json_deserializer = json.loads  # pylint: disable=protected-access
+# pylint: disable=protected-access
+SnowflakeDialect._json_deserializer = json.loads
 SnowflakeDialect.get_table_names = get_table_names
 SnowflakeDialect.get_view_names = get_view_names
 SnowflakeDialect.get_stream_names = get_stream_names
@@ -142,15 +164,11 @@ SnowflakeDialect.get_table_comment = get_table_comment
 SnowflakeDialect.get_all_view_definitions = get_all_view_definitions
 SnowflakeDialect.get_view_definition = get_view_definition
 SnowflakeDialect.get_unique_constraints = get_unique_constraints
-SnowflakeDialect._get_schema_columns = (  # pylint: disable=protected-access
-    get_schema_columns
-)
+SnowflakeDialect._get_schema_columns = get_schema_columns
 Inspector.get_table_names = get_table_names_reflection
 Inspector.get_view_names = get_view_names_reflection
 Inspector.get_stream_names = get_stream_names_reflection
-SnowflakeDialect._current_database_schema = (  # pylint: disable=protected-access
-    _current_database_schema
-)
+SnowflakeDialect._current_database_schema = _current_database_schema
 SnowflakeDialect.get_pk_constraint = get_pk_constraint
 SnowflakeDialect.get_foreign_keys = get_foreign_keys
 SnowflakeDialect.get_columns = get_columns
@@ -160,6 +178,7 @@ Inspector.get_stream_definition = get_stream_definition
 SnowflakeDialect._get_schema_foreign_keys = get_schema_foreign_keys
 
 
+# pylint: disable=too-many-public-methods
 class SnowflakeSource(
     ExternalTableLineageMixin,
     CommonDbSourceService,
@@ -926,4 +945,3 @@ class SnowflakeSource(
         logger.debug(
             f"Processing ownership is not supported for {self.service_connection.type.name}"
         )
-        return None

metadata/ingestion/source/database/snowflake/service_spec.py

@@ -6,10 +6,10 @@ from metadata.ingestion.source.database.snowflake.data_diff.data_diff import (
 )
 from metadata.ingestion.source.database.snowflake.lineage import SnowflakeLineageSource
 from metadata.ingestion.source.database.snowflake.metadata import SnowflakeSource
-from metadata.ingestion.source.database.snowflake.profiler.profiler import (
-    SnowflakeProfiler,
-)
 from metadata.ingestion.source.database.snowflake.usage import SnowflakeUsageSource
+from metadata.profiler.interface.sqlalchemy.snowflake.profiler_interface import (
+    SnowflakeProfilerInterface,
+)
 from metadata.sampler.sqlalchemy.snowflake.sampler import SnowflakeSampler
 from metadata.utils.service_spec.default import DefaultDatabaseSpec
 
@@ -17,7 +17,7 @@ ServiceSpec = DefaultDatabaseSpec(
     metadata_source_class=SnowflakeSource,
     lineage_source_class=SnowflakeLineageSource,
     usage_source_class=SnowflakeUsageSource,
-    profiler_class=SnowflakeProfiler,
+    profiler_class=SnowflakeProfilerInterface,
     test_suite_class=SnowflakeTestSuiteInterface,
     sampler_class=SnowflakeSampler,
     data_diff=SnowflakeTableParameter,

metadata/ingestion/source/database/stored_procedures_mixin.py

@@ -41,6 +41,7 @@ from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.models.topology import Queue
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.filters import filter_by_stored_procedure
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.stored_procedures import get_procedure_name_from_call
 from metadata.utils.time_utils import datetime_to_timestamp
@@ -289,6 +290,15 @@ class StoredProcedureLineageMixin(ABC):
             or []
         ):
             if procedure:
+                if filter_by_stored_procedure(
+                    self.source_config.storedProcedureFilterPattern,
+                    procedure.name.root,
+                ):
+                    self.status.filter(
+                        procedure.name.root,
+                        "Stored Procedure Filtered Out",
+                    )
+                    continue
                 logger.debug(f"Processing Lineage for [{procedure.name}]")
                 for query_by_procedure in (
                     queries_dict.get(procedure.name.root.lower()) or []