openmetadata-ingestion 1.6.1.0__py3-none-any.whl → 1.6.2.1__py3-none-any.whl

metadata/ingestion/source/dashboard/powerbi/client.py

@@ -127,7 +127,13 @@ class PowerBiApiClient:
             List[PowerBIDashboard]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/dashboards")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(
+                f"/myorg/{admin}groups/{group_id}/dashboards"
+            )
+            if not response_data:
+                logger.debug(f"No dashboards found for workspace_id: {group_id}")
+                return None
             response = DashboardsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -142,7 +148,11 @@ class PowerBiApiClient:
             List[PowerBIReport]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/reports")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/reports")
+            if not response_data:
+                logger.debug(f"No reports found for workspace_id: {group_id}")
+                return None
             response = ReportsResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -157,7 +167,11 @@ class PowerBiApiClient:
             List[Dataset]
         """
         try:
-            response_data = self.client.get(f"/myorg/groups/{group_id}/datasets")
+            admin = "admin/" if self.config.useAdminApis else ""
+            response_data = self.client.get(f"/myorg/{admin}groups/{group_id}/datasets")
+            if not response_data:
+                logger.debug(f"No datasets found for workspace_id: {group_id}")
+                return None
             response = DatasetResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -174,9 +188,13 @@ class PowerBiApiClient:
             List[Tile]
         """
         try:
+            admin = "admin/" if self.config.useAdminApis else ""
             response_data = self.client.get(
-                f"/myorg/groups/{group_id}/dashboards/{dashboard_id}/tiles"
+                f"/myorg/{admin}dashboards/{dashboard_id}/tiles"
             )
+            if not response_data:
+                logger.debug(f"No dashboard tiles found for workspace_id: {group_id}")
+                return None
             response = TilesResponse(**response_data)
             return response.value
         except Exception as exc:  # pylint: disable=broad-except
@@ -216,6 +234,9 @@ class PowerBiApiClient:
             entities_per_page = self.config.pagination_entity_per_page
             params_data = {"$top": "1"}
             response_data = self.client.get(api_url, data=params_data)
+            if not response_data:
+                logger.debug("No groups/workspaces found")
+                return None
             response = GroupsResponse(**response_data)
             count = response.odata_count
             indexes = math.ceil(count / entities_per_page)
@@ -227,6 +248,9 @@ class PowerBiApiClient:
                     "$skip": str(index * entities_per_page),
                 }
                 response_data = self.client.get(api_url, data=params_data)
+                if not response_data:
+                    logger.debug("No more groups/workspaces found")
+                    continue
                 response = GroupsResponse(**response_data)
                 workspaces.extend(response.value)
             return workspaces
@@ -262,6 +286,7 @@ class PowerBiApiClient:
     def fetch_workspace_scan_status(
         self, scan_id: str
     ) -> Optional[WorkSpaceScanResponse]:
+        # deprecated in favour to avoide bulk data prepare
         """Get Workspace scan status by id method
         Args:
             scan_id:
@@ -280,6 +305,7 @@ class PowerBiApiClient:
         return None
 
     def fetch_workspace_scan_result(self, scan_id: str) -> Optional[Workspaces]:
+        # deprecated in favour to avoide bulk data prepare
         """Get Workspace scan result by id method
         Args:
             scan_id:
@@ -301,6 +327,7 @@ class PowerBiApiClient:
         """
         Method to poll the scan status endpoint until the timeout
         """
+        # deprecated in favour to avoide bulk data prepare
         min_sleep_time = 3
         if min_sleep_time > timeout:
             logger.info(f"Timeout is set to minimum sleep time: {timeout}")

metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -49,6 +49,7 @@ from metadata.generated.schema.type.basic import (
 )
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
@@ -92,12 +93,6 @@ class PowerbiSource(DashboardServiceSource):
         self.datamodel_file_mappings = []
 
     def prepare(self):
-        if self.service_connection.useAdminApis:
-            groups = self.get_admin_workspace_data()
-        else:
-            groups = self.get_org_workspace_data()
-        if groups:
-            self.workspace_data = self.get_filtered_workspaces(groups)
         return super().prepare()
 
     def close(self):
@@ -123,91 +118,6 @@ class PowerbiSource(DashboardServiceSource):
             filtered_groups.append(group)
         return filtered_groups
 
-    def get_org_workspace_data(self) -> Optional[List[Group]]:
-        """
-        fetch all the group workspace ids
-        """
-        groups = self.client.api_client.fetch_all_workspaces()
-        for group in groups:
-            # add the dashboards to the groups
-            group.dashboards.extend(
-                self.client.api_client.fetch_all_org_dashboards(group_id=group.id) or []
-            )
-            for dashboard in group.dashboards:
-                # add the tiles to the dashboards
-                dashboard.tiles.extend(
-                    self.client.api_client.fetch_all_org_tiles(
-                        group_id=group.id, dashboard_id=dashboard.id
-                    )
-                    or []
-                )
-
-            # add the reports to the groups
-            group.reports.extend(
-                self.client.api_client.fetch_all_org_reports(group_id=group.id) or []
-            )
-
-            # add the datasets to the groups
-            group.datasets.extend(
-                self.client.api_client.fetch_all_org_datasets(group_id=group.id) or []
-            )
-            for dataset in group.datasets:
-                # add the tables to the datasets
-                dataset.tables.extend(
-                    self.client.api_client.fetch_dataset_tables(
-                        group_id=group.id, dataset_id=dataset.id
-                    )
-                    or []
-                )
-        return groups
-
-    def get_admin_workspace_data(self) -> Optional[List[Group]]:
-        """
-        fetch all the workspace ids
-        """
-        groups = []
-        workspaces = self.client.api_client.fetch_all_workspaces()
-        if workspaces:
-            workspace_id_list = [workspace.id for workspace in workspaces]
-
-            # Start the scan of the available workspaces for dashboard metadata
-            workspace_paginated_list = [
-                workspace_id_list[i : i + self.pagination_entity_per_page]
-                for i in range(
-                    0, len(workspace_id_list), self.pagination_entity_per_page
-                )
-            ]
-            count = 1
-            for workspace_ids_chunk in workspace_paginated_list:
-                logger.info(
-                    f"Scanning {count}/{len(workspace_paginated_list)} set of workspaces"
-                )
-                workspace_scan = self.client.api_client.initiate_workspace_scan(
-                    workspace_ids_chunk
-                )
-
-                # Keep polling the scan status endpoint to check if scan is succeeded
-                workspace_scan_status = self.client.api_client.wait_for_scan_complete(
-                    scan_id=workspace_scan.id
-                )
-                if workspace_scan_status:
-                    response = self.client.api_client.fetch_workspace_scan_result(
-                        scan_id=workspace_scan.id
-                    )
-                    groups.extend(
-                        [
-                            active_workspace
-                            for active_workspace in response.workspaces
-                            if active_workspace.state == "Active"
-                        ]
-                    )
-                else:
-                    logger.error("Error in fetching dashboards and charts")
-                count += 1
-        else:
-            logger.error("Unable to fetch any PowerBI workspaces")
-        return groups or None
-
     @classmethod
     def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
@@ -220,12 +130,51 @@ class PowerbiSource(DashboardServiceSource):
             )
         return cls(config, metadata)
 
+    def prepare_workspace_data(self, workspace: Group):
+        """prepare one workspace data at a time"""
+        # add the dashboards to the groups
+        workspace.dashboards.extend(
+            self.client.api_client.fetch_all_org_dashboards(group_id=workspace.id) or []
+        )
+        for dashboard in workspace.dashboards:
+            # add the tiles to the dashboards
+            dashboard.tiles.extend(
+                self.client.api_client.fetch_all_org_tiles(
+                    group_id=workspace.id, dashboard_id=dashboard.id
+                )
+                or []
+            )
+
+        # add the reports to the groups
+        workspace.reports.extend(
+            self.client.api_client.fetch_all_org_reports(group_id=workspace.id) or []
+        )
+
+        # add the datasets to the groups
+        workspace.datasets.extend(
+            self.client.api_client.fetch_all_org_datasets(group_id=workspace.id) or []
+        )
+        for dataset in workspace.datasets:
+            # add the tables to the datasets
+            dataset.tables.extend(
+                self.client.api_client.fetch_dataset_tables(
+                    group_id=workspace.id, dataset_id=dataset.id
+                )
+                or []
+            )
+
     def get_dashboard(self) -> Any:
         """
         Method to iterate through dashboard lists filter dashboards & yield dashboard details
         """
-        for workspace in self.workspace_data:
+        # fetch all workspaces/groups & apply filter pattern
+        all_workspaces = self.client.api_client.fetch_all_workspaces() or []
+        all_workspaces = self.get_filtered_workspaces(all_workspaces)
+        for workspace in all_workspaces:
+            # prepare additional data for specific workspace (datasets, reports, dashboards)
+            self.prepare_workspace_data(workspace)
             self.context.get().workspace = workspace
+            self.workspace_data.append(workspace)
             for dashboard in self.get_dashboards_list():
                 try:
                     dashboard_details = self.get_dashboard_details(dashboard)
@@ -309,54 +258,45 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/{chart_url_postfix}"
         )
 
-    def list_datamodels(self) -> Iterable[Dataset]:
+    def yield_datamodel(
+        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
+    ) -> Iterable[Either[CreateDashboardRequest]]:
         """
-        Get All the Powerbi Datasets
+        Method to yield datamodel for each workspace
         """
-        if self.source_config.includeDataModels:
+        workspace_datasets = self.context.get().workspace.datasets
+        for dataset in workspace_datasets:
+            if filter_by_datamodel(
+                self.source_config.dataModelFilterPattern, dataset.name
+            ):
+                self.status.filter(dataset.name, "Data model filtered out.")
+                continue
             try:
-                for workspace in self.workspace_data:
-                    for dataset in workspace.datasets or []:
-                        if filter_by_datamodel(
-                            self.source_config.dataModelFilterPattern, dataset.name
-                        ):
-                            self.status.filter(dataset.name, "Data model filtered out.")
-                            continue
-                        yield dataset
-            except Exception as err:
-                logger.debug(traceback.format_exc())
-                logger.error(f"Unexpected error fetching PowerBI datasets - {err}")
-
-    def yield_bulk_datamodel(
-        self, dataset: Dataset
-    ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
-        """
-        Method to fetch DataModels in bulk
-        """
-        try:
-            data_model_request = CreateDashboardDataModelRequest(
-                name=EntityName(dataset.id),
-                displayName=dataset.name,
-                description=Markdown(dataset.description)
-                if dataset.description
-                else None,
-                service=FullyQualifiedEntityName(self.context.get().dashboard_service),
-                dataModelType=DataModelType.PowerBIDataModel.value,
-                serviceType=DashboardServiceType.PowerBI.value,
-                columns=self._get_column_info(dataset),
-                project=self._fetch_dataset_workspace(dataset_id=dataset.id),
-            )
-            yield Either(right=data_model_request)
-            self.register_record_datamodel(datamodel_request=data_model_request)
+                data_model_request = CreateDashboardDataModelRequest(
+                    name=EntityName(dataset.id),
+                    displayName=dataset.name,
+                    description=Markdown(dataset.description)
+                    if dataset.description
+                    else None,
+                    service=FullyQualifiedEntityName(
+                        self.context.get().dashboard_service
+                    ),
+                    dataModelType=DataModelType.PowerBIDataModel.value,
+                    serviceType=DashboardServiceType.PowerBI.value,
+                    columns=self._get_column_info(dataset),
+                    project=self.get_project_name(dashboard_details),
+                )
+                yield Either(right=data_model_request)
+                self.register_record_datamodel(datamodel_request=data_model_request)
 
-        except Exception as exc:
-            yield Either(
-                left=StackTraceError(
-                    name=dataset.name,
-                    error=f"Error yielding Data Model [{dataset.name}]: {exc}",
-                    stackTrace=traceback.format_exc(),
+            except Exception as exc:
+                yield Either(
+                    left=StackTraceError(
+                        name=dataset.name,
+                        error=f"Error yielding Data Model [{dataset.name}]: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
                 )
-            )
 
     def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
         """
@@ -822,3 +762,85 @@ class PowerbiSource(DashboardServiceSource):
                 f"Error fetching project name for {dashboard_details.id}: {exc}"
             )
         return None
+
+    def yield_dashboard_lineage(
+        self, dashboard_details: Any
+    ) -> Iterable[Either[OMetaLineageRequest]]:
+        """
+        Yields lineage if config is enabled.
+
+        We will look for the data in all the services
+        we have informed.
+        """
+        for lineage in self.yield_datamodel_dashboard_lineage(dashboard_details) or []:
+            if lineage.right is not None:
+                yield Either(
+                    right=OMetaLineageRequest(
+                        lineage_request=lineage.right,
+                        override_lineage=self.source_config.overrideLineage,
+                    )
+                )
+            else:
+                yield lineage
+
+        db_service_names = self.get_db_service_names()
+        for db_service_name in db_service_names or []:
+            yield from self.yield_dashboard_lineage_details(
+                dashboard_details, db_service_name
+            ) or []
+
+    def yield_datamodel_dashboard_lineage(
+        self, dashboard_details: Any
+    ) -> Iterable[Either[AddLineageRequest]]:
+        """
+        Returns:
+            Lineage request between Data Models and Dashboards
+        """
+        dashboard_fqn = fqn.build(
+            self.metadata,
+            entity_type=Dashboard,
+            service_name=self.context.get().dashboard_service,
+            dashboard_name=dashboard_details.id,
+        )
+        dashboard_entity = self.metadata.get_by_name(
+            entity=Dashboard, fqn=dashboard_fqn
+        )
+        if isinstance(dashboard_details, PowerBIReport):
+            datamodel_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=DashboardDataModel,
+                service_name=self.context.get().dashboard_service,
+                data_model_name=dashboard_details.datasetId,
+            )
+            datamodel_entity = self.metadata.get_by_name(
+                entity=DashboardDataModel, fqn=datamodel_fqn
+            )
+            if dashboard_entity and datamodel_entity:
+                yield self._get_add_lineage_request(
+                    to_entity=dashboard_entity, from_entity=datamodel_entity
+                )
+        else:
+            if (
+                hasattr(self.context.get(), "dataModels")
+                and self.context.get().dataModels
+            ):
+                for datamodel in self.context.get().dataModels:
+                    try:
+                        datamodel_fqn = fqn.build(
+                            metadata=self.metadata,
+                            entity_type=DashboardDataModel,
+                            service_name=self.context.get().dashboard_service,
+                            data_model_name=datamodel,
+                        )
+                        datamodel_entity = self.metadata.get_by_name(
+                            entity=DashboardDataModel, fqn=datamodel_fqn
+                        )
+                        if dashboard_entity and datamodel_entity:
+                            yield self._get_add_lineage_request(
+                                to_entity=dashboard_entity, from_entity=datamodel_entity
+                            )
+                    except Exception as err:
+                        logger.debug(traceback.format_exc())
+                        logger.error(
+                            f"Error to yield dashboard lineage details for data model name [{str(datamodel)}]: {err}"
+                        )

metadata/ingestion/source/database/db2/metadata.py

@@ -13,6 +13,7 @@ import traceback
 from typing import Iterable, Optional
 
 from ibm_db_sa.base import ischema_names
+from ibm_db_sa.reflection import DB2Reflector, OS390Reflector
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy.engine.row import LegacyRow
 from sqlalchemy.sql.sqltypes import BOOLEAN
@@ -26,6 +27,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+from metadata.ingestion.source.database.db2.utils import get_unique_constraints
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -34,6 +36,10 @@ logger = ingestion_logger()
 ischema_names.update({"BOOLEAN": BOOLEAN})
 
 
+DB2Reflector.get_unique_constraints = get_unique_constraints
+OS390Reflector.get_unique_constraints = get_unique_constraints
+
+
 class Db2Source(CommonDbSourceService):
     """
     Implements the necessary methods to extract

metadata/ingestion/source/database/db2/utils.py

@@ -0,0 +1,63 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Module to define overriden dialect methods
+"""
+from sqlalchemy import and_, join, sql
+from sqlalchemy.engine import reflection
+
+
+@reflection.cache
+def get_unique_constraints(
+    self, connection, table_name, schema=None, **kw
+):  # pylint: disable=unused-argument
+    """Small Method to override the Dialect default as it is not filtering properly the Schema and Table Name."""
+    current_schema = self.denormalize_name(schema or self.default_schema_name)
+    table_name = self.denormalize_name(table_name)
+    syskeycol = self.sys_keycoluse
+    sysconst = self.sys_tabconst
+    query = (
+        sql.select(syskeycol.c.constname, syskeycol.c.colname)
+        .select_from(
+            join(
+                syskeycol,
+                sysconst,
+                and_(
+                    syskeycol.c.constname == sysconst.c.constname,
+                    syskeycol.c.tabschema == sysconst.c.tabschema,
+                    syskeycol.c.tabname == sysconst.c.tabname,
+                ),
+            )
+        )
+        .where(
+            and_(
+                sysconst.c.tabname == table_name,
+                sysconst.c.tabschema == current_schema,
+                sysconst.c.type == "U",
+            )
+        )
+        .order_by(syskeycol.c.constname)
+    )
+    unique_consts = []
+    curr_const = None
+    for r in connection.execute(query):
+        if curr_const == r[0]:
+            unique_consts[-1]["column_names"].append(self.normalize_name(r[1]))
+        else:
+            curr_const = r[0]
+            unique_consts.append(
+                {
+                    "name": self.normalize_name(curr_const),
+                    "column_names": [self.normalize_name(r[1])],
+                }
+            )
+    return unique_consts

metadata/ingestion/source/database/dbt/dbt_service.py

@@ -122,6 +122,11 @@ class DbtServiceTopology(ServiceTopology):
                 processor="process_dbt_descriptions",
                 nullable=True,
             ),
+            NodeStage(
+                type_=DataModelLink,
+                processor="process_dbt_owners",
+                nullable=True,
+            ),
         ],
     )
     process_dbt_tests: Annotated[
@@ -293,6 +298,12 @@ class DbtServiceSource(TopologyRunnerMixin, Source, ABC):
         Method to process DBT descriptions using patch APIs
         """
 
+    @abstractmethod
+    def process_dbt_owners(self, data_model_link: DataModelLink):
+        """
+        Method to process DBT owners using patch APIs
+        """
+
     def get_dbt_tests(self) -> dict:
         """
         Prepare the DBT tests

metadata/ingestion/source/database/dbt/metadata.py

@@ -61,6 +61,7 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
 from metadata.ingestion.lineage.sql_lineage import get_lineage_by_query
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
+from metadata.ingestion.models.patch_request import PatchedEntity, PatchRequest
 from metadata.ingestion.models.table_metadata import ColumnDescription
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
@@ -886,6 +887,47 @@ class DbtSource(DbtServiceSource):
                     f"to update dbt description: {exc}"
                 )
 
+    def process_dbt_owners(
+        self, data_model_link: DataModelLink
+    ) -> Iterable[Either[PatchedEntity]]:
+        """
+        Method to process DBT owners
+        """
+        table_entity: Table = data_model_link.table_entity
+        if table_entity:
+            logger.debug(
+                f"Processing DBT owners for: {table_entity.fullyQualifiedName.root}"
+            )
+            try:
+                data_model = data_model_link.datamodel
+                if (
+                    data_model.resourceType != DbtCommonEnum.SOURCE.value
+                    and self.source_config.dbtUpdateOwners
+                ):
+                    logger.debug(
+                        f"Overwriting owners with DBT owners: {table_entity.fullyQualifiedName.root}"
+                    )
+                    if data_model.owners:
+                        new_entity = deepcopy(table_entity)
+                        new_entity.owners = data_model.owners
+                        yield Either(
+                            right=PatchRequest(
+                                original_entity=table_entity,
+                                new_entity=new_entity,
+                                override_metadata=True,
+                            )
+                        )
+
+            except Exception as exc:  # pylint: disable=broad-except
+                yield Either(
+                    left=StackTraceError(
+                        name=str(table_entity.fullyQualifiedName.root),
+                        error=f"Failed to parse the node"
+                        f"{table_entity.fullyQualifiedName.root} to update dbt owner: {exc}",
+                        stackTrace=traceback.format_exc(),
+                    )
+                )
+
     def create_dbt_tests_definition(
         self, dbt_test: dict
     ) -> Iterable[Either[CreateTestDefinitionRequest]]:

metadata/ingestion/source/database/redshift/queries.py

@@ -328,7 +328,7 @@ Q_HISTORY as (
         pid as query_session_id,
         starttime as query_start_time,
         endtime as query_end_time,
-        b.usename as query_user_name
+        cast(b.usename as varchar) as query_user_name
     from STL_QUERY q
     join pg_catalog.pg_user b
       on b.usesysid = q.userid

metadata/ingestion/source/database/sample_data.py

@@ -1548,9 +1548,7 @@ class SampleDataSource(
                     test_suite=CreateTestSuiteRequest(
                         name=test_suite["testSuiteName"],
                         description=test_suite["testSuiteDescription"],
-                        executableEntityReference=test_suite[
-                            "executableEntityReference"
-                        ],
+                        basicEntityReference=test_suite["executableEntityReference"],
                     )
                 )
             )

metadata/ingestion/source/database/snowflake/queries.py

@@ -296,7 +296,7 @@ SELECT
   ARGUMENT_SIGNATURE AS signature,
   COMMENT as comment,
   'StoredProcedure' as procedure_type
-FROM INFORMATION_SCHEMA.PROCEDURES
+FROM SNOWFLAKE.ACCOUNT_USAGE.PROCEDURES
 WHERE PROCEDURE_CATALOG = '{database_name}'
   AND PROCEDURE_SCHEMA = '{schema_name}'
     """
@@ -312,7 +312,7 @@ SELECT
   ARGUMENT_SIGNATURE AS signature,
   COMMENT as comment,
   'UDF' as procedure_type
-FROM INFORMATION_SCHEMA.FUNCTIONS
+FROM SNOWFLAKE.ACCOUNT_USAGE.FUNCTIONS
 WHERE FUNCTION_CATALOG = '{database_name}'
   AND FUNCTION_SCHEMA = '{schema_name}'
     """

metadata/ingestion/source/database/sql_column_handler.py

@@ -94,6 +94,12 @@ class SqlColumnHandlerMixin:
                 parsed_string["name"] = column["name"]
         else:
             col_type = ColumnTypeParser.get_column_type(column["type"])
+            # For arrays, we'll get the item type if possible, or parse the string representation of the column
+            # if SQLAlchemy does not provide any further information
+            if col_type == "ARRAY" and getattr(column["type"], "item_type"):
+                arr_data_type = ColumnTypeParser.get_column_type(
+                    column["type"].item_type
+                )
             if col_type == "ARRAY" and re.match(
                 r"(?:\w*)(?:\()(\w*)(?:.*)", str(column["type"])
             ):