openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl

metadata/ingestion/source/database/oracle/metadata.py

@@ -12,18 +12,31 @@
 # pylint: disable=protected-access
 """Oracle source module"""
 import traceback
-from typing import Iterable, Optional
+from typing import Dict, Iterable, List, Optional
 
 from sqlalchemy.dialects.oracle.base import INTERVAL, OracleDialect, ischema_names
 from sqlalchemy.engine import Inspector
 
+from metadata.generated.schema.api.data.createStoredProcedure import (
+    CreateStoredProcedureRequest,
+)
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.storedProcedure import (
+    Language,
+    StoredProcedureCode,
+)
 from metadata.generated.schema.entity.data.table import TableType
 from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
     OracleConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.basic import EntityName
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
@@ -31,6 +44,14 @@ from metadata.ingestion.source.database.common_db_source import (
     CommonDbSourceService,
     TableNameAndType,
 )
+from metadata.ingestion.source.database.oracle.models import (
+    FetchProcedureList,
+    OracleStoredProcedure,
+)
+from metadata.ingestion.source.database.oracle.queries import (
+    ORACLE_GET_STORED_PROCEDURE_QUERIES,
+    ORACLE_GET_STORED_PROCEDURES,
+)
 from metadata.ingestion.source.database.oracle.utils import (
     _get_col_type,
     get_columns,
@@ -41,6 +62,12 @@ from metadata.ingestion.source.database.oracle.utils import (
     get_table_names,
     get_view_definition,
 )
+from metadata.ingestion.source.database.stored_procedures_mixin import (
+    QueryByProcedure,
+    StoredProcedureMixin,
+)
+from metadata.utils import fqn
+from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
@@ -70,7 +97,7 @@ Inspector.get_mview_definition = get_mview_definition
 OracleDialect.get_mview_names = get_mview_names_dialect
 
 
-class OracleSource(CommonDbSourceService):
+class OracleSource(StoredProcedureMixin, CommonDbSourceService):
     """
     Implements the necessary methods to extract
     Database metadata from Oracle Source
@@ -131,3 +158,82 @@ class OracleSource(CommonDbSourceService):
             logger.debug(traceback.format_exc())
             logger.warning(f"Failed to fetch view definition for {table_name}: {exc}")
         return None
+
+    def process_result(self, data: FetchProcedureList):
+        """Process data as per our stored procedure format"""
+        result_dict = {}
+
+        for row in data:
+            owner, name, line, text = row
+            key = (owner, name)
+            if key not in result_dict:
+                result_dict[key] = {"lines": [], "text": ""}
+            result_dict[key]["lines"].append(line)
+            result_dict[key]["text"] += text
+
+        # Return the concatenated text for each procedure name, ordered by line
+        return result_dict
+
+    def get_stored_procedures(self) -> Iterable[OracleStoredProcedure]:
+        """List Oracle Stored Procedures"""
+        if self.source_config.includeStoredProcedures:
+            results: FetchProcedureList = self.engine.execute(
+                ORACLE_GET_STORED_PROCEDURES.format(
+                    schema=self.context.database_schema.upper()
+                )
+            ).all()
+            results = self.process_result(data=results)
+            for row in results.items():
+                stored_procedure = OracleStoredProcedure(
+                    name=row[0][1], definition=row[1]["text"], owner=row[0][0]
+                )
+                yield stored_procedure
+
+    def yield_stored_procedure(
+        self, stored_procedure: OracleStoredProcedure
+    ) -> Iterable[Either[CreateStoredProcedureRequest]]:
+        """Prepare the stored procedure payload"""
+
+        try:
+            stored_procedure_request = CreateStoredProcedureRequest(
+                name=EntityName(__root__=stored_procedure.name),
+                storedProcedureCode=StoredProcedureCode(
+                    language=Language.SQL,
+                    code=stored_procedure.definition,
+                ),
+                owner=self.metadata.get_reference_by_name(
+                    name=stored_procedure.owner.lower()
+                ),
+                databaseSchema=fqn.build(
+                    metadata=self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.database_service,
+                    database_name=self.context.database,
+                    schema_name=self.context.database_schema,
+                ),
+            )
+            yield Either(right=stored_procedure_request)
+            self.register_record_stored_proc_request(stored_procedure_request)
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=stored_procedure.name,
+                    error=f"Error yielding Stored Procedure [{stored_procedure.name}] due to [{exc}]",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
+
+    def get_stored_procedure_queries_dict(self) -> Dict[str, List[QueryByProcedure]]:
+        """
+        Return the dictionary associating stored procedures to the
+        queries they triggered
+        """
+        start, _ = get_start_and_end(self.source_config.queryLogDuration)
+        query = ORACLE_GET_STORED_PROCEDURE_QUERIES.format(
+            start_date=start,
+        )
+        queries_dict = self.procedure_queries_dict(
+            query=query,
+        )
+
+        return queries_dict

metadata/ingestion/source/database/oracle/models.py

@@ -0,0 +1,30 @@
+"""
+Oracle models
+"""
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class OracleStoredProcedure(BaseModel):
+    """Oracle Stored Procedure list query results"""
+
+    name: str
+    definition: str
+    language: Optional[str] = Field(
+        None, description="Will only be informed for non-SQL routines."
+    )
+    owner: str
+
+
+class FetchProcedure(BaseModel):
+    """Oracle Fetch Stored Procedure Raw Model"""
+
+    owner: Optional[str]
+    name: str
+    line: int
+    text: str
+
+
+class FetchProcedureList(BaseModel):
+    __name__: List[FetchProcedure]

metadata/ingestion/source/database/oracle/queries.py

@@ -12,58 +12,135 @@
 SQL Queries used during ingestion
 """
 
-ORACLE_ALL_TABLE_COMMENTS = """
-SELECT 
+import textwrap
+
+ORACLE_ALL_TABLE_COMMENTS = textwrap.dedent(
+    """
+SELECT
 	comments table_comment,
 	LOWER(table_name) "table_name",
 	LOWER(owner) "schema" 	
-FROM all_tab_comments
+FROM dba_tab_comments
 where comments is not null and owner not in ('SYSTEM', 'SYS')
 """
+)
 
 
-ORACLE_ALL_VIEW_DEFINITIONS = """
+ORACLE_ALL_VIEW_DEFINITIONS = textwrap.dedent(
+    """
 SELECT
 LOWER(view_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('VIEW', view_name, owner) AS view_def
-FROM all_views
+FROM DBA_VIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 UNION ALL
 SELECT
 LOWER(mview_name) AS "view_name",
 LOWER(owner) AS "schema",
 DBMS_METADATA.GET_DDL('MATERIALIZED_VIEW', mview_name, owner) AS view_def
-FROM all_mviews
+FROM DBA_MVIEWS
 WHERE owner NOT IN ('SYSTEM', 'SYS')
 """
+)
 
-GET_MATERIALIZED_VIEW_NAMES = """
-SELECT mview_name FROM all_mviews WHERE owner = :owner
+GET_MATERIALIZED_VIEW_NAMES = textwrap.dedent(
+    """
+SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner
 """
+)
 
-ORACLE_GET_TABLE_NAMES = """
-SELECT table_name FROM all_tables WHERE 
+ORACLE_GET_TABLE_NAMES = textwrap.dedent(
+    """
+SELECT table_name FROM DBA_TABLES WHERE 
 {tablespace}
 OWNER = :owner  
 AND IOT_NAME IS NULL 
 AND DURATION IS NULL
 AND TABLE_NAME NOT IN 
-(SELECT mview_name FROM all_mviews WHERE owner = :owner)
+(SELECT mview_name FROM DBA_MVIEWS WHERE owner = :owner)
 """
+)
 
-ORACLE_IDENTITY_TYPE = """\
+ORACLE_IDENTITY_TYPE = textwrap.dedent(
+    """\
 col.default_on_null,
 (
 	SELECT id.generation_type || ',' || id.IDENTITY_OPTIONS
-	FROM ALL_TAB_IDENTITY_COLS{dblink} id
+	FROM DBA_TAB_IDENTITY_COLS{dblink} id
 	WHERE col.table_name = id.table_name
 	AND col.column_name = id.column_name
 	AND col.owner = id.owner
 ) AS identity_options
 """
+)
 
-ORACLE_GET_COLUMNS = """
+ORACLE_GET_STORED_PROCEDURES = textwrap.dedent(
+    """
+SELECT
+    OWNER,
+    NAME,
+    LINE,
+    TEXT
+FROM
+    DBA_SOURCE
+WHERE
+    type = 'PROCEDURE' and owner = '{schema}'
+"""
+)
+
+ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
+    """
+WITH SP_HISTORY AS (SELECT
+	sql_text AS query_text,
+    TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') AS start_time,
+    TO_TIMESTAMP(LAST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') + NUMTODSINTERVAL(ELAPSED_TIME / 1000, 'SECOND') AS end_time,
+    PARSING_SCHEMA_NAME as user_name
+  FROM gv$sql
+  WHERE sql_text LIKE 'CALL%%'
+  AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') >= TO_TIMESTAMP('{start_date}', 'YYYY-MM-DD HH24:MI:SS')
+ ),
+ Q_HISTORY AS (SELECT
+      sql_text AS query_text,
+      CASE 
+      	WHEN UPPER(SQL_TEXT) LIKE 'INSERT%' THEN 'INSERT'
+      	WHEN UPPER(SQL_TEXT) LIKE 'SELECT%' THEN 'SELECT'
+      	ELSE 'OTHER'
+    	END AS QUERY_TYPE,
+      TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') AS start_time,
+      TO_TIMESTAMP(LAST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') 
+      + NUMTODSINTERVAL(ELAPSED_TIME / 1000, 'SECOND') AS end_time,
+      PARSING_SCHEMA_NAME AS user_name,
+      PARSING_SCHEMA_NAME AS SCHEMA_NAME,
+      NULL AS DATABASE_NAME
+    FROM gv$sql
+    WHERE sql_text NOT LIKE '%CALL%'
+      AND SQL_FULLTEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
+      AND SQL_FULLTEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%'
+      AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'YYYY-MM-DD HH24:MI:SS') 
+      >= TO_TIMESTAMP('{start_date}', 'YYYY-MM-DD HH24:MI:SS')
+)
+SELECT
+  Q.QUERY_TYPE AS QUERY_TYPE,
+  Q.DATABASE_NAME AS QUERY_DATABASE_NAME,
+  Q.SCHEMA_NAME AS QUERY_SCHEMA_NAME,
+  SP.QUERY_TEXT AS PROCEDURE_TEXT,
+  SP.START_TIME AS PROCEDURE_START_TIME,
+  SP.END_TIME AS PROCEDURE_END_TIME,
+  Q.START_TIME AS QUERY_START_TIME,
+  Q.QUERY_TEXT AS QUERY_TEXT,
+  Q.USER_NAME AS QUERY_USER_NAME
+FROM SP_HISTORY SP
+JOIN Q_HISTORY Q
+  ON Q.start_time between SP.start_time and SP.end_time
+  AND Q.end_time between SP.start_time and SP.end_time
+  AND Q.user_name = SP.user_name
+ORDER BY PROCEDURE_START_TIME DESC
+"""
+)
+
+ORACLE_GET_COLUMNS = textwrap.dedent(
+    """
         SELECT
             col.column_name,
             col.data_type,
@@ -75,7 +152,7 @@ ORACLE_GET_COLUMNS = """
             com.comments,
             col.virtual_column,
             {identity_cols}
-        FROM all_tab_cols{dblink} col
+        FROM DBA_TAB_COLS{dblink} col
         LEFT JOIN all_col_comments{dblink} com
         ON col.table_name = com.table_name
         AND col.column_name = com.column_name
@@ -83,8 +160,10 @@ ORACLE_GET_COLUMNS = """
         WHERE col.table_name = CAST(:table_name AS VARCHAR2(128))
         AND col.hidden_column = 'NO'
     """
+)
 
-ORACLE_QUERY_HISTORY_STATEMENT = """
+ORACLE_QUERY_HISTORY_STATEMENT = textwrap.dedent(
+    """
 SELECT 
   NULL AS user_name,
   NULL AS database_name,
@@ -100,7 +179,9 @@ WHERE OBJECT_STATUS = 'VALID'
   AND SQL_FULLTEXT NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%'
   AND SQL_FULLTEXT NOT LIKE '/* {{"app": "dbt", %%}} */%%'
   AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'yy-MM-dd/HH24:MI:SS') >= TO_TIMESTAMP('{start_time}', 'yy-MM-dd HH24:MI:SS')
-  AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'yy-MM-dd/HH24:MI:SS') + NUMTODSINTERVAL(ELAPSED_TIME / 1000, 'SECOND') < TO_TIMESTAMP('{end_time}', 'yy-MM-dd HH24:MI:SS')
+  AND TO_TIMESTAMP(FIRST_LOAD_TIME, 'yy-MM-dd/HH24:MI:SS') + NUMTODSINTERVAL(ELAPSED_TIME / 1000, 'SECOND') 
+  < TO_TIMESTAMP('{end_time}', 'yy-MM-dd HH24:MI:SS')
 ORDER BY FIRST_LOAD_TIME DESC 
 OFFSET 0 ROWS FETCH NEXT {result_limit} ROWS ONLY
 """
+)

metadata/ingestion/source/database/oracle/utils.py

@@ -62,7 +62,6 @@ def get_view_definition(
     dblink="",
     **kw,
 ):
-
     return get_view_definition_wrapper(
         self,
         connection,

metadata/ingestion/source/database/postgres/lineage.py

@@ -11,15 +11,17 @@
 """
 Postgres lineage module
 """
+import traceback
+from datetime import datetime
 from typing import Iterable
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresScheme,
 )
+from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.models import Either
-from metadata.ingestion.lineage.models import Dialect
-from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
+from metadata.ingestion.source.connections import get_connection
 from metadata.ingestion.source.database.lineage_source import LineageSource
 from metadata.ingestion.source.database.postgres.pgspider.lineage import (
     get_lineage_from_multi_tenant_table,
@@ -55,19 +57,8 @@ class PostgresLineageSource(PostgresQueryParserSource, LineageSource):
         Based on the query logs, prepare the lineage
         and send it to the sink
         """
-        for table_queries in self.get_table_query():
-            for table_query in table_queries.queries:
-                lineages = get_lineage_by_query(
-                    self.metadata,
-                    query=table_query.query,
-                    service_name=table_query.serviceName,
-                    database_name=table_query.databaseName,
-                    schema_name=table_query.databaseSchema,
-                    dialect=Dialect.POSTGRES,
-                )
 
-                for lineage_request in lineages or []:
-                    yield lineage_request
+        yield from super()._iter()
 
         if self.service_connection.scheme == PostgresScheme.pgspider_psycopg2:
             lineages = get_lineage_from_multi_tenant_table(
@@ -78,3 +69,30 @@ class PostgresLineageSource(PostgresQueryParserSource, LineageSource):
 
             for lineage_request in lineages or []:
                 yield lineage_request
+
+    def process_table_query(self) -> Iterable[TableQuery]:
+        """
+        Process Query
+        """
+        try:
+            with get_connection(self.service_connection).connect() as conn:
+                rows = conn.execute(self.get_sql_statement())
+                for row in rows:
+                    row = dict(row)
+                    try:
+                        yield TableQuery(
+                            query=row["query_text"],
+                            userName=row["usename"],
+                            analysisDate=datetime.now(),
+                            aborted=self.get_aborted_status(row),
+                            databaseName=self.get_database_name(row),
+                            serviceName=self.config.serviceName,
+                            databaseSchema=self.get_schema_name(row),
+                            duration=row.get("duration"),
+                        )
+                    except Exception as err:
+                        logger.debug(traceback.format_exc())
+                        logger.error(str(err))
+        except Exception as err:
+            logger.error(f"Source usage processing error - {err}")
+            logger.debug(traceback.format_exc())

metadata/ingestion/source/database/postgres/metadata.py

@@ -17,7 +17,7 @@ from typing import Iterable, Optional, Tuple
 
 from sqlalchemy import sql
 from sqlalchemy.dialects.postgresql.base import PGDialect, ischema_names
-from sqlalchemy.engine.reflection import Inspector
+from sqlalchemy.engine import Inspector
 
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.table import (
@@ -28,10 +28,13 @@ from metadata.generated.schema.entity.data.table import (
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
 )
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.ingestion.api.models import Either, StackTraceError
+from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -50,7 +53,9 @@ from metadata.ingestion.source.database.postgres.queries import (
 from metadata.ingestion.source.database.postgres.utils import (
     get_column_info,
     get_columns,
+    get_etable_owner,
     get_table_comment,
+    get_table_owner,
     get_view_definition,
 )
 from metadata.utils import fqn
@@ -58,6 +63,7 @@ from metadata.utils.filters import filter_by_database
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
+    get_all_table_owners,
     get_all_view_definitions,
 )
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
@@ -109,8 +115,12 @@ PGDialect.get_view_definition = get_view_definition
 PGDialect.get_columns = get_columns
 PGDialect.get_all_view_definitions = get_all_view_definitions
 
+PGDialect.get_all_table_owners = get_all_table_owners
+PGDialect.get_table_owner = get_table_owner
 PGDialect.ischema_names = ischema_names
 
+Inspector.get_table_owner = get_etable_owner
+
 
 class PostgresSource(CommonDbSourceService, MultiDBSource):
     """
@@ -188,12 +198,10 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
                     )
 
     def get_table_partition_details(
-        self, table_name: str, schema_name: str, inspector: Inspector
+        self, table_name: str, schema_name: str, inspector
     ) -> Tuple[bool, TablePartition]:
         result = self.engine.execute(
-            POSTGRES_PARTITION_DETAILS.format(
-                table_name=table_name, schema_name=schema_name
-            )
+            POSTGRES_PARTITION_DETAILS, table_name=table_name, schema_name=schema_name
         ).all()
         if result:
             partition_details = TablePartition(
@@ -236,6 +244,6 @@ class PostgresSource(CommonDbSourceService, MultiDBSource):
                 left=StackTraceError(
                     name="Tags and Classification",
                     error=f"Skipping Policy Tag: {exc}",
-                    stack_trace=traceback.format_exc(),
+                    stackTrace=traceback.format_exc(),
                 )
             )

metadata/ingestion/source/database/postgres/pgspider/lineage.py

@@ -75,7 +75,6 @@ def get_lineage_from_multi_tenant_table(
     connection: any,
     service_name: str,
 ) -> Iterator[Either[AddLineageRequest]]:
-
     """
     For PGSpider, firstly, get list of multi-tenant tables.
     Next, get child foreign tables of each multi-tenant tables.

metadata/ingestion/source/database/postgres/queries.py

@@ -40,7 +40,9 @@ POSTGRES_GET_TABLE_NAMES = """
     JOIN pg_namespace n ON n.oid = c.relnamespace
     WHERE n.nspname = :schema AND c.relkind in ('r', 'p', 'f') AND relispartition = false
 """
-
+POSTGRES_TABLE_OWNERS = """
+select schemaname, tablename, tableowner from pg_catalog.pg_tables where schemaname <> 'pg_catalog' order by schemaname,tablename;
+"""
 POSTGRES_PARTITION_DETAILS = textwrap.dedent(
     """
     select
@@ -69,7 +71,7 @@ POSTGRES_PARTITION_DETAILS = textwrap.dedent(
         col.table_schema = par.relnamespace::regnamespace::text
         and col.table_name = par.relname
         and ordinal_position = pt.column_index
-     where par.relname='{table_name}' and  par.relnamespace::regnamespace::text='{schema_name}'
+     where par.relname=%(table_name)s and  par.relnamespace::regnamespace::text=%(schema_name)s
     """
 )
 

metadata/ingestion/source/database/postgres/query_parser.py

@@ -11,11 +11,9 @@
 """
 Postgres Query parser module
 """
-import csv
 import traceback
 from abc import ABC
-from datetime import datetime
-from typing import Iterable, Optional
+from typing import Iterable
 
 from sqlalchemy.engine.base import Engine
 
@@ -25,7 +23,7 @@ from metadata.generated.schema.entity.services.connections.database.postgresConn
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
-from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
+from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection
@@ -75,46 +73,11 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
             time_column_name=get_postgres_time_column_name(engine=self.engine),
         )
 
+    # pylint: disable=no-member
     def get_table_query(self) -> Iterable[TableQuery]:
         try:
             if self.config.sourceConfig.config.queryLogFilePath:
-                table_query_list = []
-                with open(
-                    self.config.sourceConfig.config.queryLogFilePath,
-                    "r",
-                    encoding="utf-8",
-                ) as query_log_file:
-                    for record in csv.DictReader(query_log_file):
-                        query_dict = dict(record)
-
-                        analysis_date = (
-                            datetime.utcnow()
-                            if not query_dict.get("session_start_time")
-                            else datetime.strptime(
-                                query_dict.get("session_start_time"),
-                                "%Y-%m-%d %H:%M:%S+%f",
-                            )
-                        )
-
-                        query_dict["aborted"] = query_dict["sql_state_code"] == "00000"
-                        if "statement" in query_dict["message"]:
-                            query_dict["message"] = query_dict["message"].split(":")[1]
-
-                        table_query_list.append(
-                            TableQuery(
-                                query=query_dict["message"],
-                                userName=query_dict.get("user_name", ""),
-                                startTime=query_dict.get("session_start_time", ""),
-                                endTime=query_dict.get("log_time", ""),
-                                analysisDate=analysis_date,
-                                aborted=self.get_aborted_status(query_dict),
-                                databaseName=self.get_database_name(query_dict),
-                                serviceName=self.config.serviceName,
-                                databaseSchema=self.get_schema_name(query_dict),
-                            )
-                        )
-                yield TableQueries(queries=table_query_list)
-
+                yield from super().yield_table_queries_from_logs()
             else:
                 database = self.config.serviceConnection.__root__.config.database
                 if database:
@@ -133,37 +96,6 @@ class PostgresQueryParserSource(QueryParserSource, ABC):
             logger.error(f"Source usage processing error - {err}")
             logger.debug(traceback.format_exc())
 
-    def process_table_query(self) -> Optional[Iterable[TableQuery]]:
-        """
-        Process Query
-        """
-        try:
-            with get_connection(self.service_connection).connect() as conn:
-                rows = conn.execute(self.get_sql_statement())
-                queries = []
-                for row in rows:
-                    row = dict(row)
-                    try:
-                        queries.append(
-                            TableQuery(
-                                query=row["query_text"],
-                                userName=row["usename"],
-                                analysisDate=datetime.now(),
-                                aborted=self.get_aborted_status(row),
-                                databaseName=self.get_database_name(row),
-                                serviceName=self.config.serviceName,
-                                databaseSchema=self.get_schema_name(row),
-                                duration=row.get("duration"),
-                            )
-                        )
-                    except Exception as err:
-                        logger.debug(traceback.format_exc())
-                        logger.error(str(err))
-            yield TableQueries(queries=queries)
-        except Exception as err:
-            logger.error(f"Source usage processing error - {err}")
-            logger.debug(traceback.format_exc())
-
     @staticmethod
     def get_database_name(data: dict) -> str:
         """