openmetadata-ingestion 1.5.0.0rc1__py3-none-any.whl → 1.5.0.0rc2__py3-none-any.whl

metadata/ingestion/source/dashboard/looker/utils.py

@@ -28,6 +28,9 @@ from metadata.generated.schema.security.credentials.bitbucketCredentials import
 from metadata.generated.schema.security.credentials.githubCredentials import (
     GitHubCredentials,
 )
+from metadata.generated.schema.security.credentials.gitlabCredentials import (
+    GitlabCredentials,
+)
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -38,9 +41,7 @@ def _clone_repo(
     path: str,
     credential: Optional[
         Union[
-            NoGitCredentials,
-            GitHubCredentials,
-            BitBucketCredentials,
+            NoGitCredentials, GitHubCredentials, BitBucketCredentials, GitlabCredentials
         ]
     ],
     overwrite: Optional[bool] = False,
@@ -60,6 +61,8 @@ def _clone_repo(
         elif isinstance(credential, BitBucketCredentials):
             url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@bitbucket.org/{repo_name}.git"
             allow_unsafe_protocols = True
+        elif isinstance(credential, GitlabCredentials):
+            url = f"https://x-token-auth:{credential.token.root.get_secret_value()}@gitlab.com/{repo_name}.git"
 
         assert url is not None
 

metadata/ingestion/source/dashboard/metabase/client.py

@@ -33,6 +33,7 @@ from metadata.ingestion.source.dashboard.metabase.models import (
     MetabaseUser,
 )
 from metadata.utils.constants import AUTHORIZATION_HEADER, NO_ACCESS_TOKEN
+from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -78,7 +79,7 @@ class MetabaseClient:
         self.config = config
         session_token = self._get_metabase_session()
         client_config: ClientConfig = ClientConfig(
-            base_url=str(self.config.hostPort)[:-1],  # remove trailing slash
+            base_url=clean_uri(str(self.config.hostPort)),
             api_version=API_VERSION,
             auth_header=AUTHORIZATION_HEADER,
             auth_token=lambda: (NO_ACCESS_TOKEN, 0),

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -102,7 +102,8 @@ class PowerbiSource(DashboardServiceSource):
 
     def close(self):
         self.metadata.close()
-        self.client.file_client.delete_tmp_files()
+        if self.client.file_client:
+            self.client.file_client.delete_tmp_files()
 
     def get_filtered_workspaces(self, groups: List[Group]) -> List[Group]:
         """

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -145,7 +145,7 @@ class QlikCloudClient:
         resp_apps = self.client.get("/v1/items?resourceType=app")
         if resp_apps:
             resp = QlikAppResponse(**resp_apps)
-            yield from resp.apps
+            return list(resp.apps)
 
     def get_dashboard_details(self, dashboard_id: str) -> Optional[QlikApp]:
         """

metadata/ingestion/source/dashboard/superset/mixin.py

@@ -223,6 +223,10 @@ class SupersetSourceMixin(DashboardServiceSource):
             )
         return None
 
+    def _clearn_column_datatype(self, datatype: str) -> str:
+        """clean datatype of column fetched from superset"""
+        return datatype.replace("()", "")
+
     def get_column_info(
         self, data_source: List[Union[DataSourceResult, FetchColumn]]
     ) -> Optional[List[Column]]:
@@ -236,6 +240,7 @@ class SupersetSourceMixin(DashboardServiceSource):
         for field in data_source or []:
             try:
                 if field.type:
+                    field.type = self._clearn_column_datatype(field.type)
                     col_parse = ColumnTypeParser._parse_datatype_string(  # pylint: disable=protected-access
                         field.type
                     )

metadata/ingestion/source/database/bigquery/metadata.py

@@ -11,6 +11,7 @@
 """
 Bigquery source module
 """
+import ast
 import os
 import traceback
 from typing import Dict, Iterable, List, Optional, Tuple
@@ -412,7 +413,10 @@ class BigquerySource(
             )
 
             query_result = [result.schema_description for result in query_resp.result()]
-            return fqn.unquote_name(query_result[0])
+
+            return str(
+                ast.literal_eval(query_result[0])
+            )  # To safely evaluate the string, unquote and interpret escaped characters
         except IndexError:
             logger.debug(f"No dataset description found for {schema_name}")
         except Exception as err:

metadata/ingestion/source/database/column_type_parser.py

@@ -292,6 +292,8 @@ class ColumnTypeParser:
         "CURR": "DECIMAL",
         "STRG": "STRING",
         "RSTR": "STRING",
+        # azuresql
+        "HIERARCHYID": "UNKNOWN",
     }
 
     _COMPLEX_TYPE = re.compile("^(struct|map|array|uniontype)")
@@ -347,7 +349,6 @@ class ColumnTypeParser:
             arr_data_type = ColumnTypeParser._parse_primitive_datatype_string(
                 data_type[6:-1]
             )["dataType"]
-
             data_type_string = {
                 "dataType": "ARRAY",
                 "arrayDataType": arr_data_type,

metadata/ingestion/source/database/common_db_source.py

@@ -61,7 +61,10 @@ from metadata.ingestion.connections.session import create_and_bind_thread_safe_s
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.connections import get_connection
+from metadata.ingestion.source.connections import (
+    get_connection,
+    kill_active_connections,
+)
 from metadata.ingestion.source.database.database_service import DatabaseServiceSource
 from metadata.ingestion.source.database.sql_column_handler import SqlColumnHandlerMixin
 from metadata.ingestion.source.database.sqlalchemy_source import SqlAlchemySource
@@ -142,6 +145,8 @@ class CommonDbSourceService(
         to setup multiple inspectors. They can use this function.
         :param database_name: new database to set
         """
+
+        kill_active_connections(self.engine)
         logger.info(f"Ingesting from database: {database_name}")
 
         new_service_connection = deepcopy(self.service_connection)
@@ -403,7 +408,7 @@ class CommonDbSourceService(
                 schema_definition = inspector.get_view_definition(
                     table_name, schema_name
                 )
-            elif hasattr(inspector, "get_table_ddl"):
+            elif hasattr(inspector, "get_table_ddl") and self.source_config.includeDDL:
                 schema_definition = inspector.get_table_ddl(
                     self.connection, table_name, schema_name
                 )
@@ -415,11 +420,11 @@ class CommonDbSourceService(
             return schema_definition
 
         except NotImplementedError:
-            logger.warning("Schema definition not implemented")
+            logger.debug("Schema definition not implemented")
 
         except Exception as exc:
             logger.debug(traceback.format_exc())
-            logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
+            logger.debug(f"Failed to fetch schema definition for {table_name}: {exc}")
         return None
 
     def is_partition(  # pylint: disable=unused-argument

metadata/ingestion/source/database/database_service.py

@@ -316,7 +316,7 @@ class DatabaseServiceSource(
         """
 
     def yield_table_tag_details(
-        self, table_name_and_type: str
+        self, table_name_and_type: Tuple[str, TableType]
     ) -> Iterable[Either[OMetaTagAndClassification]]:
         """
         From topology. To be run for each table

metadata/ingestion/source/database/databricks/lineage.py

@@ -42,8 +42,8 @@ class DatabricksLineageSource(DatabricksQueryParserSource, LineageSource):
                     yield TableQuery(
                         query=row.get("query_text"),
                         userName=row.get("user_name"),
-                        startTime=row.get("query_start_time_ms"),
-                        endTime=row.get("execution_end_time_ms"),
+                        startTime=str(row.get("query_start_time_ms")),
+                        endTime=str(row.get("execution_end_time_ms")),
                         analysisDate=DateTime(datetime.now()),
                         serviceName=self.config.serviceName,
                     )

metadata/ingestion/source/database/databricks/metadata.py

@@ -15,8 +15,10 @@ import traceback
 from copy import deepcopy
 from typing import Iterable, Optional, Tuple, Union
 
+from pydantic import EmailStr
+from pydantic_core import PydanticCustomError
 from pyhive.sqlalchemy_hive import _type_map
-from sqlalchemy import types, util
+from sqlalchemy import exc, types, util
 from sqlalchemy.engine import reflection
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy.exc import DatabaseError
@@ -35,6 +37,7 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
+from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -162,16 +165,19 @@ def get_columns(self, connection, table_name, schema=None, **kw):
         }
         if col_type in {"array", "struct", "map"}:
             col_name = f"`{col_name}`" if "." in col_name else col_name
-            rows = dict(
-                connection.execute(
-                    f"DESCRIBE {schema}.{table_name} {col_name}"
-                    if schema
-                    else f"DESCRIBE {table_name} {col_name}"
-                ).fetchall()
-            )
-
-            col_info["system_data_type"] = rows["data_type"]
-            col_info["is_complex"] = True
+            try:
+                rows = dict(
+                    connection.execute(
+                        f"DESCRIBE TABLE {kw.get('db_name')}.{schema}.{table_name} {col_name}"
+                    ).fetchall()
+                )
+                col_info["system_data_type"] = rows["data_type"]
+                col_info["is_complex"] = True
+            except DatabaseError as err:
+                logger.error(
+                    f"Failed to fetch column details for column {col_name} in table {table_name} due to: {err}"
+                )
+                logger.debug(traceback.format_exc())
         result.append(col_info)
     return result
 
@@ -221,7 +227,9 @@ def get_table_comment(  # pylint: disable=unused-argument
     """
     cursor = connection.execute(
         DATABRICKS_GET_TABLE_COMMENTS.format(
-            schema_name=schema_name, table_name=table_name
+            database_name=self.context.get().database,
+            schema_name=schema_name,
+            table_name=table_name,
         )
     )
     try:
@@ -283,15 +291,46 @@ def get_table_names(
         # if it is > 1, we use spark thrift server with 3 columns in the result (schema, table, is_temporary)
         # else it is hive with 1 column in the result
         if len(row) > 1:
-            tables.append(row[1])
+            table_name = row[1]
         else:
-            tables.append(row[0])
+            table_name = row[0]
+        if schema:
+            database = kw.get("db_name")
+            table_type = get_table_type(connection, database, schema, table_name)
+            if not table_type or table_type == "FOREIGN":
+                # skip the table if it's foreign table / error in fetching table_type
+                logger.debug(
+                    f"Skipping metadata ingestion for unsupported foreign table {table_name}"
+                )
+                continue
+        tables.append(table_name)
+
     # "SHOW TABLES" command in hive also fetches view names
     # Below code filters out view names from table names
     views = self.get_view_names(connection, schema)
     return [table for table in tables if table not in views]
 
 
+def get_table_type(connection, database, schema, table):
+    """get table type (regular/foreign)"""
+    try:
+        if database:
+            query = DATABRICKS_GET_TABLE_COMMENTS.format(
+                database_name=database, schema_name=schema, table_name=table
+            )
+        else:
+            query = f"DESCRIBE TABLE EXTENDED {schema}.{table}"
+        rows = connection.execute(query)
+        for row in rows:
+            row_dict = dict(row)
+            if row_dict.get("col_name") == "Type":
+                # get type of table
+                return row_dict.get("data_type")
+    except DatabaseError as err:
+        logger.error(f"Failed to fetch table type for table {table} due to: {err}")
+    return
+
+
 DatabricksDialect.get_table_comment = get_table_comment
 DatabricksDialect.get_view_names = get_view_names
 DatabricksDialect.get_columns = get_columns
@@ -476,9 +515,11 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
                 )
                 if filter_by_database(
                     self.source_config.databaseFilterPattern,
-                    database_fqn
-                    if self.source_config.useFqnForFiltering
-                    else new_catalog,
+                    (
+                        database_fqn
+                        if self.source_config.useFqnForFiltering
+                        else new_catalog
+                    ),
                 ):
                     self.status.filter(database_fqn, "Database Filtered Out")
                     continue
@@ -638,7 +679,9 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
         try:
             cursor = self.connection.execute(
                 DATABRICKS_GET_TABLE_COMMENTS.format(
-                    schema_name=schema_name, table_name=table_name
+                    database_name=self.context.get().database,
+                    schema_name=schema_name,
+                    table_name=table_name,
                 )
             )
             for result in list(cursor):
@@ -661,3 +704,42 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
                 f"Table description error for table [{schema_name}.{table_name}]: {exc}"
             )
         return description
+
+    def _filter_owner_name(self, owner_name: str) -> str:
+        """remove unnecessary keyword from name"""
+        pattern = r"\(Unknown\)"
+        filtered_name = re.sub(pattern, "", owner_name).strip()
+        return filtered_name
+
+    def get_owner_ref(self, table_name: str) -> Optional[EntityReferenceList]:
+        """
+        Method to process the table owners
+        """
+        try:
+            query = DATABRICKS_GET_TABLE_COMMENTS.format(
+                database_name=self.context.get().database,
+                schema_name=self.context.get().database_schema,
+                table_name=table_name,
+            )
+            result = self.connection.engine.execute(query)
+            owner = None
+            for row in result:
+                row_dict = dict(row)
+                if row_dict.get("col_name") == "Owner":
+                    owner = row_dict.get("data_type")
+                    break
+            if not owner:
+                return
+
+            owner = self._filter_owner_name(owner)
+            owner_ref = None
+            try:
+                owner_email = EmailStr._validate(owner)
+                owner_ref = self.metadata.get_reference_by_email(email=owner_email)
+            except PydanticCustomError:
+                owner_ref = self.metadata.get_reference_by_name(name=owner)
+            return owner_ref
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error processing owner for table {table_name}: {exc}")
+        return

metadata/ingestion/source/database/databricks/queries.py

@@ -24,7 +24,9 @@ DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent(
     """
 )
 
-DATABRICKS_GET_TABLE_COMMENTS = "DESCRIBE TABLE EXTENDED {schema_name}.{table_name}"
+DATABRICKS_GET_TABLE_COMMENTS = (
+    "DESCRIBE TABLE EXTENDED {database_name}.{schema_name}.{table_name}"
+)
 
 DATABRICKS_GET_CATALOGS = "SHOW CATALOGS"
 

metadata/ingestion/source/database/databricks/usage.py

@@ -47,8 +47,8 @@ class DatabricksUsageSource(DatabricksQueryParserSource, UsageSource):
                         TableQuery(
                             query=row.get("query_text"),
                             userName=row.get("user_name"),
-                            startTime=row.get("query_start_time_ms"),
-                            endTime=row.get("execution_end_time_ms"),
+                            startTime=str(row.get("query_start_time_ms")),
+                            endTime=str(row.get("execution_end_time_ms")),
                             analysisDate=DateTime(datetime.now()),
                             serviceName=self.config.serviceName,
                             duration=row.get("duration")

metadata/ingestion/source/database/datalake/clients/gcs.py

@@ -38,6 +38,10 @@ class DatalakeGcsClient(DatalakeBaseClient):
         self._client = client
         self._temp_credentials_file_path_list = temp_credentials_file_path_list
 
+    @property
+    def project(self):
+        return self._client.project
+
     @staticmethod
     def get_gcs_client(config: GCSConfig) -> storage.Client:
         gcs_config = deepcopy(config)

metadata/ingestion/source/database/db2/metadata.py

@@ -10,7 +10,7 @@
 #  limitations under the License.
 """Db2 source module"""
 import traceback
-from typing import Optional
+from typing import Iterable, Optional
 
 from ibm_db_sa.base import ischema_names
 from sqlalchemy.engine.reflection import Inspector
@@ -52,6 +52,13 @@ class Db2Source(CommonDbSourceService):
             )
         return cls(config, metadata)
 
+    def get_raw_database_schema_names(self) -> Iterable[str]:
+        if self.service_connection.__dict__.get("databaseSchema"):
+            yield self.service_connection.databaseSchema
+        else:
+            for schema_name in self.inspector.get_schema_names():
+                yield schema_name.rstrip()
+
     @staticmethod
     def get_table_description(
         schema_name: str, table_name: str, inspector: Inspector

metadata/ingestion/source/database/dbt/constants.py

@@ -26,13 +26,7 @@ NONE_KEYWORDS_LIST = ["none", "null"]
 
 DBT_CATALOG_FILE_NAME = "catalog.json"
 DBT_MANIFEST_FILE_NAME = "manifest.json"
-DBT_RUN_RESULTS_FILE_NAME = "run_results.json"
-
-DBT_FILE_NAMES_LIST = [
-    DBT_CATALOG_FILE_NAME,
-    DBT_MANIFEST_FILE_NAME,
-    DBT_RUN_RESULTS_FILE_NAME,
-]
+DBT_RUN_RESULTS_FILE_NAME = "run_results"
 
 
 class SkipResourceTypeEnum(Enum):

metadata/ingestion/source/database/dbt/dbt_config.py

@@ -41,7 +41,6 @@ from metadata.generated.schema.metadataIngestion.dbtconfig.dbtS3Config import (
 )
 from metadata.ingestion.source.database.dbt.constants import (
     DBT_CATALOG_FILE_NAME,
-    DBT_FILE_NAMES_LIST,
     DBT_MANIFEST_FILE_NAME,
     DBT_RUN_RESULTS_FILE_NAME,
 )
@@ -129,7 +128,7 @@ def _(config: DbtHttpConfig):
         yield DbtFiles(
             dbt_catalog=dbt_catalog.json() if dbt_catalog else None,
             dbt_manifest=dbt_manifest.json(),
-            dbt_run_results=dbt_run_results.json() if dbt_run_results else None,
+            dbt_run_results=[dbt_run_results.json()] if dbt_run_results else None,
         )
     except DBTConfigException as exc:
         raise exc
@@ -216,7 +215,7 @@ def _(config: DbtCloudConfig):  # pylint: disable=too-many-locals
         yield DbtFiles(
             dbt_catalog=dbt_catalog,
             dbt_manifest=dbt_manifest,
-            dbt_run_results=dbt_run_results,
+            dbt_run_results=[dbt_run_results] if dbt_run_results else None,
         )
     except DBTConfigException as exc:
         raise exc
@@ -233,13 +232,12 @@ def get_blobs_grouped_by_dir(blobs: List[str]) -> Dict[str, List[str]]:
     for blob in blobs:
         subdirectory = blob.rsplit("/", 1)[0] if "/" in blob else ""
         blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
-        if next(
-            (
-                file_name
-                for file_name in DBT_FILE_NAMES_LIST
-                if file_name.lower() == blob_file_name.lower()
-            ),
-            None,
+        # We'll be processing multiple run_result files from a single dir
+        # Grouping them together to process them in a single go
+        if (
+            DBT_MANIFEST_FILE_NAME == blob_file_name.lower()
+            or DBT_CATALOG_FILE_NAME == blob_file_name.lower()
+            or DBT_RUN_RESULTS_FILE_NAME in blob_file_name.lower()
         ):
             blob_grouped_by_directory[subdirectory].append(blob)
     return blob_grouped_by_directory
@@ -257,7 +255,7 @@ def download_dbt_files(
     ) in blob_grouped_by_directory.items():
         dbt_catalog = None
         dbt_manifest = None
-        dbt_run_results = None
+        dbt_run_results = []
         kwargs = {}
         if bucket_name:
             kwargs = {"bucket_name": bucket_name}
@@ -265,10 +263,11 @@ def download_dbt_files(
             for blob in blobs:
                 if blob:
                     reader = get_reader(config_source=config, client=client)
-                    if DBT_MANIFEST_FILE_NAME in blob:
+                    blob_file_name = blob.rsplit("/", 1)[1] if "/" in blob else blob
+                    if DBT_MANIFEST_FILE_NAME == blob_file_name.lower():
                         logger.debug(f"{DBT_MANIFEST_FILE_NAME} found in {key}")
                         dbt_manifest = reader.read(path=blob, **kwargs)
-                    if DBT_CATALOG_FILE_NAME in blob:
+                    if DBT_CATALOG_FILE_NAME == blob_file_name.lower():
                         try:
                             logger.debug(f"{DBT_CATALOG_FILE_NAME} found in {key}")
                             dbt_catalog = reader.read(path=blob, **kwargs)
@@ -276,10 +275,12 @@ def download_dbt_files(
                             logger.warning(
                                 f"{DBT_CATALOG_FILE_NAME} not found in {key}: {exc}"
                             )
-                    if DBT_RUN_RESULTS_FILE_NAME in blob:
+                    if DBT_RUN_RESULTS_FILE_NAME in blob_file_name.lower():
                         try:
-                            logger.debug(f"{DBT_RUN_RESULTS_FILE_NAME} found in {key}")
-                            dbt_run_results = reader.read(path=blob, **kwargs)
+                            logger.debug(f"{blob_file_name} found in {key}")
+                            dbt_run_result = reader.read(path=blob, **kwargs)
+                            if dbt_run_result:
+                                dbt_run_results.append(json.loads(dbt_run_result))
                         except Exception as exc:
                             logger.warning(
                                 f"{DBT_RUN_RESULTS_FILE_NAME} not found in {key}: {exc}"
@@ -289,9 +290,7 @@ def download_dbt_files(
             yield DbtFiles(
                 dbt_catalog=json.loads(dbt_catalog) if dbt_catalog else None,
                 dbt_manifest=json.loads(dbt_manifest),
-                dbt_run_results=json.loads(dbt_run_results)
-                if dbt_run_results
-                else None,
+                dbt_run_results=dbt_run_results if dbt_run_results else None,
             )
         except DBTConfigException as exc:
             logger.warning(exc)

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -184,9 +184,10 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
             if self.context.get().dbt_file.dbt_catalog
             else None,
             dbt_manifest=parse_manifest(self.context.get().dbt_file.dbt_manifest),
-            dbt_run_results=parse_run_results(
-                self.context.get().dbt_file.dbt_run_results
-            )
+            dbt_run_results=[
+                parse_run_results(run_result_file)
+                for run_result_file in self.context.get().dbt_file.dbt_run_results
+            ]
             if self.context.get().dbt_file.dbt_run_results
             else None,
         )

metadata/ingestion/source/database/dbt/metadata.py

@@ -1,3 +1,4 @@
+#  pylint: disable=too-many-lines
 #  Copyright 2021 Collate
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -156,6 +157,13 @@ class DbtSource(DbtServiceSource):
                 owner = self.metadata.get_reference_by_name(
                     name=dbt_owner, is_owner=True
                 )
+
+                if owner:
+                    return owner
+
+                # If owner is not found, try to find the owner in OMD using email
+                owner = self.metadata.get_reference_by_email(name=dbt_owner)
+
                 if not owner:
                     logger.warning(
                         "Unable to ingest owner from DBT since no user or"
@@ -312,7 +320,8 @@ class DbtSource(DbtServiceSource):
         self.context.get().dbt_tests[key][DbtCommonEnum.RESULTS.value] = next(
             (
                 item
-                for item in dbt_objects.dbt_run_results.results
+                for run_result in dbt_objects.dbt_run_results
+                for item in run_result.results
                 if item.unique_id == key
             ),
             None,
@@ -340,12 +349,14 @@ class DbtSource(DbtServiceSource):
             self.context.get().data_model_links = []
             self.context.get().dbt_tests = {}
             self.context.get().run_results_generate_time = None
+            # Since we'll be processing multiple run_results for a single project
+            # we'll only consider the first run_results generated_at time
             if (
                 dbt_objects.dbt_run_results
-                and dbt_objects.dbt_run_results.metadata.generated_at
+                and dbt_objects.dbt_run_results[0].metadata.generated_at
             ):
                 self.context.get().run_results_generate_time = (
-                    dbt_objects.dbt_run_results.metadata.generated_at
+                    dbt_objects.dbt_run_results[0].metadata.generated_at
                 )
             for key, manifest_node in manifest_entities.items():
                 try:

metadata/ingestion/source/database/dbt/models.py

@@ -20,13 +20,13 @@ from pydantic import BaseModel
 class DbtFiles(BaseModel):
     dbt_catalog: Optional[dict] = None
     dbt_manifest: dict
-    dbt_run_results: Optional[dict] = None
+    dbt_run_results: Optional[List[dict]] = None
 
 
 class DbtObjects(BaseModel):
     dbt_catalog: Optional[Any] = None
     dbt_manifest: Any
-    dbt_run_results: Optional[Any] = None
+    dbt_run_results: Optional[List[Any]] = None
 
 
 class DbtFilteredModel(BaseModel):

metadata/ingestion/source/database/mssql/lineage.py

@@ -26,10 +26,6 @@ from metadata.ingestion.source.database.mssql.utils import (
 
 
 class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.start = self.start.replace(tzinfo=None)
-        self.end = self.end.replace(tzinfo=None)
 
     sql_stmt = MSSQL_SQL_STATEMENT
 

metadata/ingestion/source/database/mssql/metadata.py

@@ -224,7 +224,7 @@ class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
         current_datetime_format = MSSQL_DATEFORMAT_DATETIME_MAP.get(
             server_date_format, DEFAULT_DATETIME_FORMAT
         )
-        start = start.replace(tzinfo=None).strftime(current_datetime_format)
+        start = start.strftime(current_datetime_format)
         query = MSSQL_GET_STORED_PROCEDURE_QUERIES.format(
             start_date=start,
         )