openmetadata-ingestion 1.3.4.0__py3-none-any.whl → 1.4.0.0__py3-none-any.whl

metadata/ingestion/source/database/redshift/incremental_table_processor.py

@@ -0,0 +1,192 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Incremental Processor for Redshift
+"""
+import re
+from datetime import datetime
+from typing import Iterable, List, Optional, Tuple
+
+from sqlalchemy.engine import Connection
+
+from metadata.ingestion.source.database.redshift.models import (
+    RedshiftTable,
+    RedshiftTableChangeQueryRegex,
+    RedshiftTableMap,
+    SchemaName,
+    TableName,
+)
+from metadata.ingestion.source.database.redshift.queries import (
+    REDSHIFT_TABLE_CHANGES_QUERY,
+)
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+
+# ---- Regex Definitions
+TABLE_NAME_RE = r"(\w+\.){0,2}\w+"
+
+CREATE_TABLE = rf"^.*CREATE\s+(LOCAL\s+|EXTERNAL\s+)?(TEMPORARY\s+|TEMP\s+)?TABLE\s+(IF\s+NOT\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+ALTER_TABLE = rf"^.*(ALTER\s+TABLE)\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+DROP_TABLE = rf"^.*DROP\s+TABLE\s+(IF\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+CREATE_VIEW = rf"^.*CREATE\s+(OR\s+REPLACE\s+)?(EXTERNAL\s+|MATERIALIZED\s+)?VIEW\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+ALTER_VIEW = rf"^.*ALTER\s+(EXTERNAL\s+)?VIEW\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+DROP_VIEW = rf"^.*DROP\s+(EXTERNAL\s+|MATERIALIZED\s+)?VIEW\s+(IF\s+EXISTS\s+)?(?P<table>{TABLE_NAME_RE}).*$"
+
+# Not supporting Comment changes on Constraint
+COMMENT = rf"^.*COMMENT\s+ON\s+(TABLE|COLUMN|VIEW)\s+(?P<table>{TABLE_NAME_RE}).*$"
+
+
+REGEX_LIST = [
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(ALTER_TABLE, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(CREATE_TABLE, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(DROP_TABLE, re.IGNORECASE), deleted=True
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(ALTER_VIEW, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(CREATE_VIEW, re.IGNORECASE), deleted=False
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(DROP_VIEW, re.IGNORECASE), deleted=True
+    ),
+    RedshiftTableChangeQueryRegex(
+        regex=re.compile(COMMENT, re.IGNORECASE), deleted=False
+    ),
+]
+
+
+class RedshiftIncrementalTableProcessor:
+    """Incremental Table Processor class.
+
+    Responsible for the Incremental Extraction flow."""
+
+    def __init__(
+        self,
+        table_map: RedshiftTableMap,
+        table_changes_query: str,
+        regex_list: List[RedshiftTableChangeQueryRegex],
+        connection: Connection,
+        default_schema: SchemaName,
+    ):
+        self.table_map = table_map
+        self.table_changes_query = table_changes_query
+        self.regex_list = regex_list
+        self.connection = connection
+        self.default_schema = default_schema
+
+    @classmethod
+    def create(
+        cls, connection: Connection, default_schema: SchemaName
+    ) -> "RedshiftIncrementalTableProcessor":
+        """Creates a new instance based on a connection and the default schema."""
+        return cls(
+            table_map=RedshiftTableMap.default(),
+            table_changes_query=REDSHIFT_TABLE_CHANGES_QUERY,
+            regex_list=REGEX_LIST,
+            connection=connection,
+            default_schema=default_schema,
+        )
+
+    def _query_for_changes(self, database: str, start_date: datetime) -> Iterable[str]:
+        """Queries the Redshift database for the Table Changes."""
+        for row in (
+            self.connection.execute(
+                self.table_changes_query.format(
+                    database=database, start_date=start_date
+                )
+            )
+            or []
+        ):
+            yield row[0]
+
+    def _clean_statement(self, statement: str) -> str:
+        """Gets rid of unwanted characters"""
+        return (
+            statement.replace("\n", " ")
+            .replace("\t", " ")
+            .replace("\v", " ")
+            .replace('"', "")
+        )  # Removes '"' to make the Regex Match easier.
+
+    def _get_schema_and_table(
+        self, full_table_name: str, statement: str
+    ) -> Tuple[SchemaName, TableName]:
+        """From the full table name, retrieves the Schema and Table Name.
+        If no Schema is present, falls back to the default schema."""
+        full_table_name_as_list = full_table_name.split(".")
+
+        # Check if Schema is present
+        if len(full_table_name_as_list) >= 2:
+            schema = full_table_name_as_list[-2]
+        else:
+            schema = self.default_schema
+            logger.debug(
+                "Schema not present for statement %s. TableName: %s",
+                statement,
+                full_table_name,
+            )
+            logger.debug("Falling back to the Default Schema: %s", self.default_schema)
+
+        table_name = full_table_name_as_list[-1]
+
+        return schema, table_name
+
+    def set_table_map(self, database: str, start_date: datetime):
+        """Sets the RedshiftTableMap for the given database, filtering by the given start_date."""
+        for statement in self._query_for_changes(database, start_date):
+            statement = self._clean_statement(statement)
+            match_found = False
+
+            for possible_match in self.regex_list:
+                match = possible_match.regex.match(statement)
+
+                if not match:
+                    continue
+
+                else:
+                    match_found = True
+                    schema, table_name = self._get_schema_and_table(
+                        match.group("table"), statement
+                    )
+                    deleted = possible_match.deleted
+
+                    self.table_map.update(
+                        schema, RedshiftTable(name=table_name, deleted=deleted)
+                    )
+
+                    break
+
+            if not match_found:
+                logger.debug("Match not found for %s", statement)
+
+    def get_deleted(
+        self, schema_name: Optional[SchemaName] = None
+    ) -> List[Tuple[SchemaName, TableName]]:
+        """Returns the deleted table names present in the table_map for a given schema."""
+        return self.table_map.get_deleted(schema_name)
+
+    def get_not_deleted(self, schema_name: SchemaName) -> List[TableName]:
+        """Returns the not deleted table names present in the table_map for a given schema."""
+        return self.table_map.get_not_deleted(schema_name)

metadata/ingestion/source/database/redshift/metadata.py

@@ -16,7 +16,7 @@ import re
 import traceback
 from typing import Dict, Iterable, List, Optional, Tuple
 
-from sqlalchemy import inspect, sql
+from sqlalchemy import sql
 from sqlalchemy.dialects.postgresql.base import PGDialect
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy_redshift.dialect import RedshiftDialect, RedshiftDialectMixin
@@ -32,7 +32,9 @@ from metadata.generated.schema.entity.data.storedProcedure import (
 )
 from metadata.generated.schema.entity.data.table import (
     ConstraintType,
-    IntervalType,
+    PartitionColumnDetails,
+    PartitionIntervalTypes,
+    Table,
     TableConstraint,
     TablePartition,
     TableType,
@@ -47,6 +49,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.generated.schema.type.basic import EntityName
+from metadata.ingestion.api.delete import delete_entity_by_name
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -54,10 +57,16 @@ from metadata.ingestion.source.database.common_db_source import (
     CommonDbSourceService,
     TableNameAndType,
 )
+from metadata.ingestion.source.database.incremental_metadata_extraction import (
+    IncrementalConfig,
+)
 from metadata.ingestion.source.database.life_cycle_query_mixin import (
     LifeCycleQueryMixin,
 )
 from metadata.ingestion.source.database.multi_db_source import MultiDBSource
+from metadata.ingestion.source.database.redshift.incremental_table_processor import (
+    RedshiftIncrementalTableProcessor,
+)
 from metadata.ingestion.source.database.redshift.models import RedshiftStoredProcedure
 from metadata.ingestion.source.database.redshift.queries import (
     REDSHIFT_GET_ALL_RELATION_INFO,
@@ -80,10 +89,18 @@ from metadata.ingestion.source.database.stored_procedures_mixin import (
     StoredProcedureMixin,
 )
 from metadata.utils import fqn
+from metadata.utils.execution_time_tracker import (
+    calculate_execution_time,
+    calculate_execution_time_generator,
+)
 from metadata.utils.filters import filter_by_database
 from metadata.utils.helpers import get_start_and_end
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.sqlalchemy_utils import get_all_table_comments
+from metadata.utils.sqlalchemy_utils import (
+    get_all_table_comments,
+    get_all_table_ddls,
+    get_table_ddl,
+)
 
 logger = ingestion_logger()
 
@@ -109,6 +126,9 @@ RedshiftDialect._get_all_relation_info = (  # pylint: disable=protected-access
     _get_all_relation_info
 )
 
+Inspector.get_all_table_ddls = get_all_table_ddls
+Inspector.get_table_ddl = get_table_ddl
+
 
 class RedshiftSource(
     LifeCycleQueryMixin, StoredProcedureMixin, CommonDbSourceService, MultiDBSource
@@ -118,20 +138,41 @@ class RedshiftSource(
     Database metadata from Redshift Source
     """
 
-    def __init__(self, config, metadata):
+    def __init__(
+        self,
+        config: WorkflowSource,
+        metadata,
+        incremental_configuration: IncrementalConfig,
+    ):
         super().__init__(config, metadata)
         self.partition_details = {}
         self.life_cycle_query = REDSHIFT_LIFE_CYCLE_QUERY
+        self.context.get_global().deleted_tables = []
+        self.incremental = incremental_configuration
+        self.incremental_table_processor: Optional[
+            RedshiftIncrementalTableProcessor
+        ] = None
+
+        if self.incremental.enabled:
+            logger.info(
+                "Starting Incremental Metadata Extraction.\n\t Considering Table changes from %s",
+                self.incremental.start_datetime_utc,
+            )
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: RedshiftConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, RedshiftConnection):
             raise InvalidSourceException(
                 f"Expected RedshiftConnection, but got {connection}"
             )
-        return cls(config, metadata)
+        incremental_config = IncrementalConfig.create(
+            config.sourceConfig.config.incremental, pipeline_name, metadata
+        )
+        return cls(config, metadata, incremental_config)
 
     def get_partition_details(self) -> None:
         """
@@ -139,7 +180,7 @@ class RedshiftSource(
         """
         try:
             self.partition_details.clear()
-            results = self.engine.execute(REDSHIFT_PARTITION_DETAILS).fetchall()
+            results = self.connection.execute(REDSHIFT_PARTITION_DETAILS).fetchall()
             for row in results:
                 self.partition_details[f"{row.schema}.{row.table}"] = row.diststyle
         except Exception as exe:
@@ -158,6 +199,16 @@ class RedshiftSource(
             {"schema": schema_name},
         )
 
+        if self.incremental.enabled:
+            result = [
+                (name, relkind)
+                for name, relkind in result
+                if name
+                in self.incremental_table_processor.get_not_deleted(
+                    schema_name=schema_name
+                )
+            ]
+
         return [
             TableNameAndType(
                 name=name, type_=STANDARD_TABLE_TYPES.get(relkind, TableType.Regular)
@@ -165,6 +216,35 @@ class RedshiftSource(
             for name, relkind in result
         ]
 
+    def query_view_names_and_types(
+        self, schema_name: str
+    ) -> Iterable[TableNameAndType]:
+        """
+        Connect to the source database to get the view
+        name and type. By default, use the inspector method
+        to get the names and pass the View type.
+
+        This is useful for sources where we need fine-grained
+        logic on how to handle table types, e.g., material views,...
+        """
+
+        result = self.inspector.get_view_names(schema_name) or []
+
+        if self.incremental.enabled:
+            result = [
+                name
+                for name in result
+                if name
+                in self.incremental_table_processor.get_not_deleted(
+                    schema_name=schema_name
+                )
+            ]
+
+        return [
+            TableNameAndType(name=table_name, type_=TableType.View)
+            for table_name in result
+        ]
+
     def get_configured_database(self) -> Optional[str]:
         if not self.service_connection.ingestAllDatabases:
             return self.service_connection.database
@@ -173,25 +253,59 @@ class RedshiftSource(
     def get_database_names_raw(self) -> Iterable[str]:
         yield from self._execute_database_query(REDSHIFT_GET_DATABASE_NAMES)
 
+    def _set_incremental_table_processor(self, database: str):
+        """Prepares the needed data for doing incremental metadata extration for a given database.
+
+        1. Queries Redshift to get the changes done after the `self.incremental.start_datetime_utc`
+        2. Sets the table map with the changes within the RedshiftIncrementalTableProcessor
+        3. Sets the deleted tables in the context
+        """
+        if self.incremental.enabled:
+            self.incremental_table_processor = RedshiftIncrementalTableProcessor.create(
+                self.connection, self.inspector.default_schema_name
+            )
+
+            self.incremental_table_processor.set_table_map(
+                database=database, start_date=self.incremental.start_datetime_utc
+            )
+
+            self.context.get_global().deleted_tables.extend(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Table,
+                    service_name=self.context.get().database_service,
+                    database_name=database,
+                    schema_name=schema_name,
+                    table_name=table_name,
+                )
+                for schema_name, table_name in self.incremental_table_processor.get_deleted()
+            )
+
     def get_database_names(self) -> Iterable[str]:
         if not self.config.serviceConnection.__root__.config.ingestAllDatabases:
-            self.inspector = inspect(self.engine)
             self.get_partition_details()
+
+            self._set_incremental_table_processor(
+                self.config.serviceConnection.__root__.config.database
+            )
+
             yield self.config.serviceConnection.__root__.config.database
         else:
             for new_database in self.get_database_names_raw():
                 database_fqn = fqn.build(
                     self.metadata,
                     entity_type=Database,
-                    service_name=self.context.database_service,
+                    service_name=self.context.get().database_service,
                     database_name=new_database,
                 )
 
                 if filter_by_database(
                     self.source_config.databaseFilterPattern,
-                    database_fqn
-                    if self.source_config.useFqnForFiltering
-                    else new_database,
+                    (
+                        database_fqn
+                        if self.source_config.useFqnForFiltering
+                        else new_database
+                    ),
                 ):
                     self.status.filter(database_fqn, "Database Filtered Out")
                     continue
@@ -199,6 +313,9 @@ class RedshiftSource(
                 try:
                     self.set_inspector(database_name=new_database)
                     self.get_partition_details()
+
+                    self._set_incremental_table_processor(new_database)
+
                     yield new_database
                 except Exception as exc:
                     logger.debug(traceback.format_exc())
@@ -206,26 +323,34 @@ class RedshiftSource(
                         f"Error trying to connect to database {new_database}: {exc}"
                     )
 
-    def _get_partition_key(self, diststyle: str) -> Optional[List[str]]:
+    def _get_partition_key(self, diststyle: str) -> Optional[str]:
         try:
             regex = re.match(r"KEY\((\w+)\)", diststyle)
             if regex:
-                return [regex.group(1)]
+                return regex.group(1)
         except Exception as err:
             logger.debug(traceback.format_exc())
             logger.warning(err)
         return None
 
+    @calculate_execution_time()
     def get_table_partition_details(
         self, table_name: str, schema_name: str, inspector: Inspector
-    ) -> Tuple[bool, TablePartition]:
+    ) -> Tuple[bool, Optional[TablePartition]]:
         diststyle = self.partition_details.get(f"{schema_name}.{table_name}")
         if diststyle:
-            partition_details = TablePartition(
-                columns=self._get_partition_key(diststyle),
-                intervalType=IntervalType.COLUMN_VALUE,
-            )
-            return True, partition_details
+            distkey = self._get_partition_key(diststyle)
+            if distkey is not None:
+                partition_details = TablePartition(
+                    columns=[
+                        PartitionColumnDetails(
+                            columnName=distkey,
+                            intervalType=PartitionIntervalTypes.COLUMN_VALUE,
+                            interval=None,
+                        )
+                    ]
+                )
+                return True, partition_details
         return False, None
 
     def process_additional_table_constraints(
@@ -254,15 +379,16 @@ class RedshiftSource(
     def get_stored_procedures(self) -> Iterable[RedshiftStoredProcedure]:
         """List Snowflake stored procedures"""
         if self.source_config.includeStoredProcedures:
-            results = self.engine.execute(
+            results = self.connection.execute(
                 REDSHIFT_GET_STORED_PROCEDURES.format(
-                    schema_name=self.context.database_schema,
+                    schema_name=self.context.get().database_schema,
                 )
             ).all()
             for row in results:
                 stored_procedure = RedshiftStoredProcedure.parse_obj(dict(row))
                 yield stored_procedure
 
+    @calculate_execution_time_generator()
     def yield_stored_procedure(
         self, stored_procedure: RedshiftStoredProcedure
     ) -> Iterable[Either[CreateStoredProcedureRequest]]:
@@ -278,9 +404,9 @@ class RedshiftSource(
                 databaseSchema=fqn.build(
                     metadata=self.metadata,
                     entity_type=DatabaseSchema,
-                    service_name=self.context.database_service,
-                    database_name=self.context.database,
-                    schema_name=self.context.database_schema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=self.context.get().database_schema,
                 ),
             )
             yield Either(right=stored_procedure_request)
@@ -304,7 +430,7 @@ class RedshiftSource(
         start, _ = get_start_and_end(self.source_config.queryLogDuration)
         query = REDSHIFT_GET_STORED_PROCEDURE_QUERIES.format(
             start_date=start,
-            database_name=self.context.database,
+            database_name=self.context.get().database,
         )
 
         queries_dict = self.procedure_queries_dict(
@@ -312,3 +438,26 @@ class RedshiftSource(
         )
 
         return queries_dict
+
+    def mark_tables_as_deleted(self):
+        """
+        Use the current inspector to mark tables as deleted
+        """
+        if self.incremental.enabled:
+            if not self.context.get().__dict__.get("database"):
+                raise ValueError(
+                    "No Database found in the context. We cannot run the table deletion."
+                )
+
+            if self.source_config.markDeletedTables:
+                logger.info(
+                    f"Mark Deleted Tables set to True. Processing database [{self.context.get().database}]"
+                )
+                yield from delete_entity_by_name(
+                    self.metadata,
+                    entity_type=Table,
+                    entity_names=self.context.get_global().deleted_tables,
+                    mark_deleted_entity=self.source_config.markDeletedTables,
+                )
+        else:
+            yield from super().mark_tables_as_deleted()

metadata/ingestion/source/database/redshift/models.py

@@ -11,10 +11,14 @@
 """
 Redshift models
 """
-from typing import Optional
+import re
+from typing import Dict, List, Optional, Tuple
 
 from pydantic import BaseModel
 
+TableName = str
+SchemaName = str
+
 
 class RedshiftStoredProcedure(BaseModel):
     """Redshift stored procedure list query results"""
@@ -22,3 +26,72 @@ class RedshiftStoredProcedure(BaseModel):
     name: str
     owner: Optional[str]
     definition: str
+
+
+class RedshiftTableChangeQueryRegex(BaseModel):
+    """Redshift Table Change Query Regex Model. Used for Incremental Extraction."""
+
+    regex: re.Pattern
+    deleted: bool
+
+
+class RedshiftTable(BaseModel):
+    """Redshift Simplified Table Model. Used for Incremental Extraction."""
+
+    name: str
+    deleted: bool
+
+
+class RedshiftTableMap(BaseModel):
+    """Redshift TableMap Model. Used for Incremental Extraction"""
+
+    table_map: Dict[SchemaName, Dict[TableName, RedshiftTable]]
+
+    @classmethod
+    def default(cls) -> "RedshiftTableMap":
+        """Creates a new RedshiftTableMap with an Empty dict."""
+        return cls(table_map={})
+
+    def update(self, schema: SchemaName, table: RedshiftTable):
+        """Updates the Redshift Table Map.
+
+        If the table is already registered in the map, it does nothing.
+        This relies on the processing of the table being ordered by latest changes first.
+        """
+        if schema not in self.table_map:
+            self.table_map[schema] = {table.name: table}
+        else:
+            if table.name not in self.table_map[schema]:
+                self.table_map[schema][table.name] = table
+
+    def get_deleted(
+        self, schema_name: Optional[SchemaName] = None
+    ) -> List[Tuple[SchemaName, TableName]]:
+        """Returns all deleted table names for a given schema."""
+        if schema_name:
+            return [
+                (schema_name, table.name)
+                for table in self.table_map.get(schema_name, {}).values()
+                if table.deleted
+            ]
+
+        deleted_tables = []
+
+        for schema_name in self.table_map:
+            deleted_tables.extend(
+                [
+                    (schema_name, table.name)
+                    for table in self.table_map.get(schema_name, {}).values()
+                    if table.deleted
+                ]
+            )
+
+        return deleted_tables
+
+    def get_not_deleted(self, schema_name: SchemaName) -> List[TableName]:
+        """Returns all not deleted table names for a given schema."""
+        return [
+            table.name
+            for table in self.table_map.get(schema_name, {}).values()
+            if not table.deleted
+        ]

metadata/ingestion/source/database/redshift/queries.py

@@ -248,7 +248,7 @@ REDSHIFT_GET_ALL_RELATIONS = """
         AS "diststyle",
         c.relowner AS "owner_id",
         u.usename AS "owner_name",
-        TRIM(TRAILING ';' FROM 
+        TRIM(TRAILING ';' FROM
         'create view ' || n.nspname || '.' || c.relname || ' as ' ||pg_catalog.pg_get_viewdef(c.oid, true))
         AS "view_definition",
         pg_catalog.array_to_string(c.relacl, '\n') AS "privileges"
@@ -345,7 +345,7 @@ select
 from SP_HISTORY sp
   join Q_HISTORY q
     on sp.procedure_session_id = q.query_session_id
-   and q.query_start_time between sp.procedure_start_time and sp.procedure_end_time 
+   and q.query_start_time between sp.procedure_start_time and sp.procedure_end_time
    and q.query_end_time between sp.procedure_start_time and sp.procedure_end_time
 order by procedure_start_time DESC
     """
@@ -360,3 +360,18 @@ where o.schema = '{schema_name}'
 and o.database = '{database_name}'
 """
 )
+
+REDSHIFT_TABLE_CHANGES_QUERY = """
+SELECT
+    query_text
+FROM SYS_QUERY_HISTORY
+WHERE status = 'success'
+  AND (
+    query_type = 'DDL' OR
+    (query_type = 'UTILITY' AND query_text ilike '%%COMMENT ON%%') OR
+    (query_type = 'CTAS' AND query_text ilike '%%CREATE TABLE%%')
+  )
+  and database_name = '{database}'
+  and end_time >= '{start_date}'
+ORDER BY end_time DESC
+"""

metadata/ingestion/source/database/redshift/query_parser.py

@@ -38,7 +38,9 @@ class RedshiftQueryParserSource(QueryParserSource, ABC):
     filters: str
 
     @classmethod
-    def create(cls, config_dict, metadata: OpenMetadata):
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
         config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
         connection: RedshiftConnection = config.serviceConnection.__root__.config
         if not isinstance(connection, RedshiftConnection):