acryl-datahub 1.0.0.1rc4__py3-none-any.whl → 1.0.0.1rc6__py3-none-any.whl

datahub/ingestion/source/powerbi/m_query/pattern_handler.py

@@ -30,7 +30,13 @@ from datahub.ingestion.source.powerbi.m_query.data_classes import (
     ReferencedTable,
 )
 from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table
-from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult
+from datahub.metadata.schema_classes import SchemaFieldDataTypeClass
+from datahub.sql_parsing.sqlglot_lineage import (
+    ColumnLineageInfo,
+    ColumnRef,
+    DownstreamColumnRef,
+    SqlParsingResult,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -262,6 +268,33 @@ class AbstractLineage(ABC):
             ),
         )
 
+    def create_table_column_lineage(self, urn: str) -> List[ColumnLineageInfo]:
+        column_lineage = []
+
+        if self.table.columns is not None:
+            for column in self.table.columns:
+                downstream = DownstreamColumnRef(
+                    table=self.table.name,
+                    column=column.name,
+                    column_type=SchemaFieldDataTypeClass(type=column.datahubDataType),
+                    native_column_type=column.dataType or "UNKNOWN",
+                )
+
+                upstreams = [
+                    ColumnRef(
+                        table=urn,
+                        column=column.name.lower(),
+                    )
+                ]
+
+                column_lineage_info = ColumnLineageInfo(
+                    downstream=downstream, upstreams=upstreams
+                )
+
+                column_lineage.append(column_lineage_info)
+
+        return column_lineage
+
 
 class AmazonRedshiftLineage(AbstractLineage):
     def get_platform_pair(self) -> DataPlatformPair:
@@ -299,6 +332,8 @@ class AmazonRedshiftLineage(AbstractLineage):
             qualified_table_name=qualified_table_name,
         )
 
+        column_lineage = self.create_table_column_lineage(urn)
+
         return Lineage(
             upstreams=[
                 DataPlatformTable(
@@ -306,7 +341,7 @@ class AmazonRedshiftLineage(AbstractLineage):
                     urn=urn,
                 )
             ],
-            column_lineage=[],
+            column_lineage=column_lineage,
         )
 
 
@@ -364,6 +399,8 @@ class OracleLineage(AbstractLineage):
             qualified_table_name=qualified_table_name,
         )
 
+        column_lineage = self.create_table_column_lineage(urn)
+
         return Lineage(
             upstreams=[
                 DataPlatformTable(
@@ -371,7 +408,7 @@ class OracleLineage(AbstractLineage):
                     urn=urn,
                 )
             ],
-            column_lineage=[],
+            column_lineage=column_lineage,
         )
 
 
@@ -449,6 +486,8 @@ class DatabricksLineage(AbstractLineage):
                 qualified_table_name=qualified_table_name,
             )
 
+            column_lineage = self.create_table_column_lineage(urn)
+
             return Lineage(
                 upstreams=[
                     DataPlatformTable(
@@ -456,7 +495,7 @@ class DatabricksLineage(AbstractLineage):
                         urn=urn,
                     )
                 ],
-                column_lineage=[],
+                column_lineage=column_lineage,
             )
 
         return Lineage.empty()
@@ -509,6 +548,9 @@ class TwoStepDataAccessPattern(AbstractLineage, ABC):
             server=server,
             qualified_table_name=qualified_table_name,
         )
+
+        column_lineage = self.create_table_column_lineage(urn)
+
         return Lineage(
             upstreams=[
                 DataPlatformTable(
@@ -516,10 +558,62 @@ class TwoStepDataAccessPattern(AbstractLineage, ABC):
                     urn=urn,
                 )
             ],
-            column_lineage=[],
+            column_lineage=column_lineage,
         )
 
 
+class MySQLLineage(AbstractLineage):
+    def create_lineage(
+        self, data_access_func_detail: DataAccessFunctionDetail
+    ) -> Lineage:
+        logger.debug(
+            f"Processing {self.get_platform_pair().powerbi_data_platform_name} data-access function detail {data_access_func_detail}"
+        )
+
+        server, db_name = self.get_db_detail_from_argument(
+            data_access_func_detail.arg_list
+        )
+        if server is None or db_name is None:
+            return Lineage.empty()  # Return an empty list
+
+        schema_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Schema"]
+
+        table_name: str = cast(
+            IdentifierAccessor, data_access_func_detail.identifier_accessor
+        ).items["Item"]
+
+        qualified_table_name: str = f"{schema_name}.{table_name}"
+
+        logger.debug(
+            f"Platform({self.get_platform_pair().datahub_data_platform_name}) qualified_table_name= {qualified_table_name}"
+        )
+
+        urn = make_urn(
+            config=self.config,
+            platform_instance_resolver=self.platform_instance_resolver,
+            data_platform_pair=self.get_platform_pair(),
+            server=server,
+            qualified_table_name=qualified_table_name,
+        )
+
+        column_lineage = self.create_table_column_lineage(urn)
+
+        return Lineage(
+            upstreams=[
+                DataPlatformTable(
+                    data_platform_pair=self.get_platform_pair(),
+                    urn=urn,
+                )
+            ],
+            column_lineage=column_lineage,
+        )
+
+    def get_platform_pair(self) -> DataPlatformPair:
+        return SupportedDataPlatform.MYSQL.value
+
+
 class PostgresLineage(TwoStepDataAccessPattern):
     def create_lineage(
         self, data_access_func_detail: DataAccessFunctionDetail
@@ -671,6 +765,8 @@ class ThreeStepDataAccessPattern(AbstractLineage, ABC):
             qualified_table_name=qualified_table_name,
         )
 
+        column_lineage = self.create_table_column_lineage(urn)
+
         return Lineage(
             upstreams=[
                 DataPlatformTable(
@@ -678,7 +774,7 @@ class ThreeStepDataAccessPattern(AbstractLineage, ABC):
                     urn=urn,
                 )
             ],
-            column_lineage=[],
+            column_lineage=column_lineage,
         )
 
 
@@ -726,6 +822,7 @@ class NativeQueryLineage(AbstractLineage):
 
         tables: List[str] = native_sql_parser.get_tables(query)
 
+        column_lineage = []
         for qualified_table_name in tables:
             if len(qualified_table_name.split(".")) != 3:
                 logger.debug(
@@ -748,12 +845,11 @@ class NativeQueryLineage(AbstractLineage):
                 )
             )
 
+            column_lineage = self.create_table_column_lineage(urn)
+
         logger.debug(f"Generated dataplatform_tables {dataplatform_tables}")
 
-        return Lineage(
-            upstreams=dataplatform_tables,
-            column_lineage=[],
-        )
+        return Lineage(upstreams=dataplatform_tables, column_lineage=column_lineage)
 
     def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]:
         if (
@@ -885,6 +981,11 @@ class SupportedPattern(Enum):
         FunctionName.AMAZON_REDSHIFT_DATA_ACCESS,
     )
 
+    MYSQL = (
+        MySQLLineage,
+        FunctionName.MYSQL_DATA_ACCESS,
+    )
+
     NATIVE_QUERY = (
         NativeQueryLineage,
         FunctionName.NATIVE_QUERY,

datahub/ingestion/source/powerbi/m_query/resolver.py

@@ -361,6 +361,9 @@ class MQueryResolver(AbstractDataAccessMQueryResolver, ABC):
         )
 
         if output_variable is None:
+            logger.debug(
+                f"Table: {self.table.full_name}: output-variable not found in tree"
+            )
             self.reporter.report_warning(
                 f"{self.table.full_name}-output-variable",
                 "output-variable not found in table expression",
@@ -374,6 +377,9 @@ class MQueryResolver(AbstractDataAccessMQueryResolver, ABC):
 
         # Each item is data-access function
         for f_detail in table_links:
+            logger.debug(
+                f"Processing data-access-function {f_detail.data_access_function_name}"
+            )
             # Get & Check if we support data-access-function available in M-Query
             supported_resolver = SupportedPattern.get_pattern_handler(
                 f_detail.data_access_function_name
@@ -390,6 +396,10 @@ class MQueryResolver(AbstractDataAccessMQueryResolver, ABC):
 
             # From supported_resolver enum get respective handler like AmazonRedshift or Snowflake or Oracle or NativeQuery and create instance of it
             # & also pass additional information that will be need to generate lineage
+            logger.debug(
+                f"Creating instance of {supported_resolver.handler().__name__} "
+                f"for data-access-function {f_detail.data_access_function_name}"
+            )
             pattern_handler: AbstractLineage = supported_resolver.handler()(
                 ctx=ctx,
                 table=self.table,

datahub/ingestion/source/powerbi/powerbi.py

@@ -3,6 +3,7 @@
 # Meta Data Ingestion From the Power BI Source
 #
 #########################################################
+import functools
 import logging
 from datetime import datetime
 from typing import Iterable, List, Optional, Tuple, Union
@@ -24,6 +25,7 @@ from datahub.ingestion.api.decorators import (
     support_status,
 )
 from datahub.ingestion.api.incremental_lineage_helper import (
+    auto_incremental_lineage,
     convert_dashboard_info_to_patch,
 )
 from datahub.ingestion.api.source import (
@@ -238,6 +240,10 @@ class Mapper:
         upstream: List[UpstreamClass] = []
         cll_lineage: List[FineGrainedLineage] = []
 
+        logger.debug(
+            f"Extracting lineage for table {table.full_name} in dataset {table.dataset.name if table.dataset else None}"
+        )
+
         upstream_lineage: List[
             datahub.ingestion.source.powerbi.m_query.data_classes.Lineage
         ] = parser.get_upstream_tables(
@@ -1306,7 +1312,9 @@ class PowerBiDashboardSource(StatefulIngestionSourceBase, TestableSource):
 
         allowed_workspaces = []
         for workspace in all_workspaces:
-            if not self.source_config.workspace_id_pattern.allowed(workspace.id):
+            if not self.source_config.workspace_id_pattern.allowed(
+                workspace.id
+            ) or not self.source_config.workspace_name_pattern.allowed(workspace.name):
                 self.reporter.filtered_workspace_names.append(
                     f"{workspace.id} - {workspace.name}"
                 )
@@ -1522,6 +1530,9 @@ class PowerBiDashboardSource(StatefulIngestionSourceBase, TestableSource):
         else:
             return [
                 *super().get_workunit_processors(),
+                functools.partial(
+                    auto_incremental_lineage, self.source_config.incremental_lineage
+                ),
                 self.stale_entity_removal_handler.workunit_processor,
             ]
 

datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py

@@ -115,7 +115,7 @@ class PowerBiAPI:
         if scan_result is None:
             return results
 
-        for scanned_dashboard in scan_result.get(Constant.DASHBOARDS, []):
+        for scanned_dashboard in scan_result.get(Constant.DASHBOARDS) or []:
             # Iterate through response and create a list of PowerBiAPI.Dashboard
             dashboard_id = scanned_dashboard.get("id")
             tags = self._parse_endorsement(
@@ -133,17 +133,17 @@ class PowerBiAPI:
         if scan_result is None:
             return results
 
-        reports: List[dict] = scan_result.get(Constant.REPORTS, [])
+        reports: List[dict] = scan_result.get(Constant.REPORTS) or []
 
         for report in reports:
-            report_id = report.get(Constant.ID, None)
+            report_id = report.get(Constant.ID)
             if report_id is None:
                 logger.warning(
                     f"Report id is none. Skipping endorsement tag for report instance {report}"
                 )
                 continue
             endorsements = self._parse_endorsement(
-                report.get(Constant.ENDORSEMENT_DETAIL, None)
+                report.get(Constant.ENDORSEMENT_DETAIL)
             )
             results[report_id] = endorsements
 
@@ -339,7 +339,7 @@ class PowerBiAPI:
         if not endorsements:
             return []
 
-        endorsement = endorsements.get(Constant.ENDORSEMENT, None)
+        endorsement = endorsements.get(Constant.ENDORSEMENT)
         if not endorsement:
             return []
 
@@ -396,7 +396,7 @@ class PowerBiAPI:
 
             if self.__config.extract_endorsements_to_tags:
                 dataset_instance.tags = self._parse_endorsement(
-                    dataset_dict.get(Constant.ENDORSEMENT_DETAIL, None)
+                    dataset_dict.get(Constant.ENDORSEMENT_DETAIL)
                 )
 
             dataset_map[dataset_instance.id] = dataset_instance
@@ -407,7 +407,7 @@ class PowerBiAPI:
                 else dataset_instance.id
             )
             logger.debug(f"dataset_dict = {dataset_dict}")
-            for table in dataset_dict.get(Constant.TABLES, []):
+            for table in dataset_dict.get(Constant.TABLES) or []:
                 expression: Optional[str] = (
                     table[Constant.SOURCE][0][Constant.EXPRESSION]
                     if table.get(Constant.SOURCE) is not None
@@ -430,10 +430,10 @@ class PowerBiAPI:
                                 column["dataType"], FIELD_TYPE_MAPPING["Null"]
                             ),
                         )
-                        for column in table.get("columns", [])
+                        for column in table.get("columns") or []
                     ],
                     measures=[
-                        Measure(**measure) for measure in table.get("measures", [])
+                        Measure(**measure) for measure in table.get("measures") or []
                     ],
                     dataset=dataset_instance,
                     row_count=None,
@@ -480,7 +480,7 @@ class PowerBiAPI:
                     )
                 )
                 if app_id is None:  # In PowerBI one workspace can have one app
-                    app_id = report.get(Constant.APP_ID)
+                    app_id = report[Constant.APP_ID]
 
         raw_app_dashboards: List[Dict] = []
         # Filter app dashboards
@@ -488,7 +488,7 @@ class PowerBiAPI:
             if dashboard.get(Constant.APP_ID):
                 raw_app_dashboards.append(dashboard)
                 if app_id is None:  # In PowerBI, one workspace contains one app
-                    app_id = report[Constant.APP_ID]
+                    app_id = dashboard[Constant.APP_ID]
 
         # workspace doesn't have an App. Above two loops can be avoided
         # if app_id is available at root level in workspace_metadata

datahub/ingestion/source/redshift/lineage_v2.py

@@ -230,7 +230,8 @@ class RedshiftSqlLineageV2(Closeable):
             )
 
         # Populate lineage for external tables.
-        self._process_external_tables(all_tables=all_tables, db_schemas=db_schemas)
+        if not self.config.skip_external_tables:
+            self._process_external_tables(all_tables=all_tables, db_schemas=db_schemas)
 
     def _populate_lineage_agg(
         self,

datahub/ingestion/source/sigma/config.py

@@ -9,6 +9,7 @@ from datahub.configuration.source_common import (
     EnvConfigMixin,
     PlatformInstanceConfigMixin,
 )
+from datahub.ingestion.api.report import EntityFilterReport
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
     StaleEntityRemovalSourceReport,
     StatefulStaleMetadataRemovalConfig,
@@ -54,16 +55,14 @@ class Constant:
 
 @dataclass
 class SigmaSourceReport(StaleEntityRemovalSourceReport):
-    number_of_workspaces: int = 0
+    workspaces: EntityFilterReport = EntityFilterReport.field(type="workspace")
+    number_of_workspaces: Optional[int] = None
     non_accessible_workspaces_count: int = 0
     shared_entities_count: int = 0
     number_of_datasets: int = 0
     number_of_workbooks: int = 0
     number_of_files_metadata: Dict[str, int] = field(default_factory=dict)
 
-    def report_number_of_workspaces(self, number_of_workspaces: int) -> None:
-        self.number_of_workspaces = number_of_workspaces
-
 
 class PlatformDetail(PlatformInstanceConfigMixin, EnvConfigMixin):
     data_source_platform: str = pydantic.Field(

datahub/ingestion/source/sigma/sigma.py

@@ -162,14 +162,17 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
 
     def _get_allowed_workspaces(self) -> List[Workspace]:
         all_workspaces = self.sigma_api.workspaces.values()
-        allowed_workspaces = [
-            workspace
-            for workspace in all_workspaces
-            if self.config.workspace_pattern.allowed(workspace.name)
-        ]
         logger.info(f"Number of workspaces = {len(all_workspaces)}")
-        self.reporter.report_number_of_workspaces(len(all_workspaces))
+        self.reporter.number_of_workspaces = len(all_workspaces)
+
+        allowed_workspaces = []
+        for workspace in all_workspaces:
+            if self.config.workspace_pattern.allowed(workspace.name):
+                allowed_workspaces.append(workspace)
+            else:
+                self.reporter.workspaces.dropped(workspace.workspaceId)
         logger.info(f"Number of allowed workspaces = {len(allowed_workspaces)}")
+
         return allowed_workspaces
 
     def _gen_workspace_workunit(
@@ -658,6 +661,7 @@ class SigmaSource(StatefulIngestionSourceBase, TestableSource):
             yield from self._gen_workbook_workunit(workbook)
 
         for workspace in self._get_allowed_workspaces():
+            self.reporter.workspaces.processed(workspace.workspaceId)
             yield from self._gen_workspace_workunit(workspace)
         yield from self._gen_sigma_dataset_upstream_lineage_workunit()
 

datahub/ingestion/source/sql/oracle.py

@@ -31,7 +31,9 @@ from datahub.ingestion.source.sql.sql_common import (
     SQLAlchemySource,
     make_sqlalchemy_type,
 )
-from datahub.ingestion.source.sql.sql_config import BasicSQLAlchemyConfig
+from datahub.ingestion.source.sql.sql_config import (
+    BasicSQLAlchemyConfig,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -71,10 +73,12 @@ class OracleConfig(BasicSQLAlchemyConfig):
         description="Will be set automatically to default value.",
     )
     service_name: Optional[str] = Field(
-        default=None, description="Oracle service name. If using, omit `database`."
+        default=None,
+        description="Oracle service name. If using, omit `database`.",
     )
     database: Optional[str] = Field(
-        default=None, description="If using, omit `service_name`."
+        default=None,
+        description="If using, omit `service_name`.",
     )
     add_database_name_to_urn: Optional[bool] = Field(
         default=False,
@@ -631,7 +635,6 @@ class OracleSource(SQLAlchemySource):
     - Table, row, and column statistics via optional SQL profiling
 
     Using the Oracle source requires that you've also installed the correct drivers; see the [cx_Oracle docs](https://cx-oracle.readthedocs.io/en/latest/user_guide/installation.html). The easiest one is the [Oracle Instant Client](https://www.oracle.com/database/technologies/instant-client.html).
-
     """
 
     config: OracleConfig
@@ -661,6 +664,8 @@ class OracleSource(SQLAlchemySource):
         database name from Connection URL, which does not work when using
         service instead of database.
         In that case, it tries to retrieve the database name by sending a query to the DB.
+
+        Note: This is used as a fallback if database is not specified in the config.
         """
 
         # call default implementation first
@@ -687,7 +692,49 @@ class OracleSource(SQLAlchemySource):
                 # To silent the mypy lint error
                 yield cast(Inspector, inspector)
 
+    def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]:
+        """
+        Override the get_db_schema method to ensure proper schema name extraction.
+        This method is used during view lineage extraction to determine the default schema
+        for unqualified table names in view definitions.
+        """
+        try:
+            # Try to get the schema from the dataset identifier
+            parts = dataset_identifier.split(".")
+
+            # Handle the identifier format differently based on add_database_name_to_urn flag
+            if self.config.add_database_name_to_urn:
+                if len(parts) >= 3:
+                    # Format is: database.schema.view when add_database_name_to_urn=True
+                    db_name = parts[-3]
+                    schema_name = parts[-2]
+                    return db_name, schema_name
+                elif len(parts) >= 2:
+                    # Handle the case where database might be missing even with flag enabled
+                    # If we have a database in the config, use that
+                    db_name = str(self.config.database)
+                    schema_name = parts[-2]
+                    return db_name, schema_name
+            else:
+                # Format is: schema.view when add_database_name_to_urn=False
+                if len(parts) >= 2:
+                    # When add_database_name_to_urn is False, don't include database in the result
+                    db_name = None
+                    schema_name = parts[-2]
+                    return db_name, schema_name
+        except Exception as e:
+            logger.warning(
+                f"Error extracting schema from identifier {dataset_identifier}: {e}"
+            )
+
+        # Fall back to parent implementation if our approach fails
+        db_name, schema_name = super().get_db_schema(dataset_identifier)
+        return db_name, schema_name
+
     def get_workunits(self):
+        """
+        Override get_workunits to patch Oracle dialect for custom types.
+        """
         with patch.dict(
             "sqlalchemy.dialects.oracle.base.OracleDialect.ischema_names",
             {klass.__name__: klass for klass in extra_oracle_types},

datahub/ingestion/source/usage/usage_common.py

@@ -12,11 +12,9 @@ from typing import (
     Optional,
     Tuple,
     TypeVar,
-    Union,
 )
 
 import pydantic
-from deprecated import deprecated
 from pydantic.fields import Field
 
 import datahub.emitter.mce_builder as builder
@@ -28,19 +26,13 @@ from datahub.configuration.time_window_config import (
 )
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetUsageStatistics
 from datahub.metadata.schema_classes import (
-    CalendarIntervalClass,
     DatasetFieldUsageCountsClass,
     DatasetUsageStatisticsClass,
     DatasetUserUsageCountsClass,
     TimeWindowSizeClass,
-    UsageAggregationClass,
-    WindowDurationClass,
 )
 from datahub.utilities.sql_formatter import format_sql_query, trim_query
-from datahub.utilities.urns.dataset_urn import DatasetUrn
-from datahub.utilities.urns.urn import guess_entity_type
 
 logger = logging.getLogger(__name__)
 
@@ -295,60 +287,3 @@ class UsageAggregator(Generic[ResourceType]):
                     user_urn_builder=user_urn_builder,
                     queries_character_limit=self.config.queries_character_limit,
                 )
-
-
-@deprecated
-def convert_usage_aggregation_class(
-    obj: UsageAggregationClass,
-) -> MetadataChangeProposalWrapper:
-    # Legacy usage aggregation only supported dataset usage stats
-    if guess_entity_type(obj.resource) == DatasetUrn.ENTITY_TYPE:
-        aspect = DatasetUsageStatistics(
-            timestampMillis=obj.bucket,
-            eventGranularity=TimeWindowSizeClass(
-                unit=convert_window_to_interval(obj.duration)
-            ),
-            uniqueUserCount=obj.metrics.uniqueUserCount,
-            totalSqlQueries=obj.metrics.totalSqlQueries,
-            topSqlQueries=obj.metrics.topSqlQueries,
-            userCounts=(
-                [
-                    DatasetUserUsageCountsClass(
-                        user=u.user, count=u.count, userEmail=u.userEmail
-                    )
-                    for u in obj.metrics.users
-                    if u.user is not None
-                ]
-                if obj.metrics.users
-                else None
-            ),
-            fieldCounts=(
-                [
-                    DatasetFieldUsageCountsClass(fieldPath=f.fieldName, count=f.count)
-                    for f in obj.metrics.fields
-                ]
-                if obj.metrics.fields
-                else None
-            ),
-        )
-        return MetadataChangeProposalWrapper(entityUrn=obj.resource, aspect=aspect)
-    else:
-        raise Exception(
-            f"Skipping unsupported usage aggregation - invalid entity type: {obj}"
-        )
-
-
-@deprecated
-def convert_window_to_interval(window: Union[str, WindowDurationClass]) -> str:
-    if window == WindowDurationClass.YEAR:
-        return CalendarIntervalClass.YEAR
-    elif window == WindowDurationClass.MONTH:
-        return CalendarIntervalClass.MONTH
-    elif window == WindowDurationClass.WEEK:
-        return CalendarIntervalClass.WEEK
-    elif window == WindowDurationClass.DAY:
-        return CalendarIntervalClass.DAY
-    elif window == WindowDurationClass.HOUR:
-        return CalendarIntervalClass.HOUR
-    else:
-        raise Exception(f"Unsupported window duration: {window}")