acryl-datahub 1.1.1rc4__py3-none-any.whl → 1.3.0.1rc9__py3-none-any.whl

datahub/ingestion/source/mock_data/table_naming_helper.py

@@ -0,0 +1,97 @@
+from typing import Dict, Optional
+
+
+class TableNamingHelper:
+    """
+    Helper class for managing table naming conventions in mock data generation.
+
+    Table naming pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+    """
+
+    @staticmethod
+    def generate_table_name(
+        lineage_hops: int,
+        lineage_fan_out: int,
+        level: int,
+        table_index: int,
+        prefix: Optional[str] = None,
+    ) -> str:
+        """
+        Generate a table name following the standard naming convention.
+
+        Args:
+            lineage_hops: Total number of hops in the lineage graph
+            lineage_fan_out: Number of downstream tables per upstream table
+            level: Level of the table in the lineage graph (0-based)
+            table_index: Index of the table within its level (0-based)
+            prefix: Optional prefix to add to the table name
+
+        Returns:
+            Table name following the pattern: "{prefix}hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+        """
+        base_name = f"hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+        return f"{prefix}{base_name}" if prefix else base_name
+
+    @staticmethod
+    def parse_table_name(table_name: str) -> Dict[str, int]:
+        """
+        Parse a table name to extract its components.
+
+        Args:
+            table_name: Table name following the standard naming convention
+
+        Returns:
+            Dictionary containing parsed components:
+            - lineage_hops: Total number of hops in the lineage graph
+            - lineage_fan_out: Number of downstream tables per upstream table
+            - level: Level of the table in the lineage graph (0-based)
+            - table_index: Index of the table within its level (0-based)
+
+        Raises:
+            ValueError: If the table name doesn't follow the expected pattern
+        """
+        try:
+            # Expected pattern: "hops_{lineage_hops}_f_{lineage_fan_out}_h{level}_t{table_index}"
+            parts = table_name.split("_")
+
+            if (
+                len(parts) != 6
+                or parts[0] != "hops"
+                or parts[2] != "f"
+                or not parts[4].startswith("h")
+                or not parts[5].startswith("t")
+            ):
+                raise ValueError(f"Invalid table name format: {table_name}")
+
+            lineage_hops = int(parts[1])
+            lineage_fan_out = int(parts[3])  # lineage_fan_out is at index 3
+            level = int(parts[4][1:])  # Remove 'h' prefix from parts[4]
+            table_index = int(parts[5][1:])  # Remove 't' prefix from parts[5]
+
+            return {
+                "lineage_hops": lineage_hops,
+                "lineage_fan_out": lineage_fan_out,
+                "level": level,
+                "table_index": table_index,
+            }
+        except (ValueError, IndexError) as e:
+            raise ValueError(
+                f"Failed to parse table name '{table_name}': {str(e)}"
+            ) from e
+
+    @staticmethod
+    def is_valid_table_name(table_name: str) -> bool:
+        """
+        Check if a table name follows the expected naming convention.
+
+        Args:
+            table_name: Table name to validate
+
+        Returns:
+            True if the table name follows the expected pattern, False otherwise
+        """
+        try:
+            TableNamingHelper.parse_table_name(table_name)
+            return True
+        except ValueError:
+            return False

datahub/ingestion/source/mode.py

@@ -7,7 +7,16 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from functools import lru_cache
 from json import JSONDecodeError
-from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
+from typing import (
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 import dateutil.parser as dp
 import psutil
@@ -24,7 +33,7 @@ from requests.models import HTTPBasicAuth, HTTPError
 from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, HiddenFromDocs
 from datahub.configuration.source_common import (
     DatasetLineageProviderConfigBase,
 )
@@ -200,10 +209,13 @@ class ModeConfig(
         default=True, description="Tag measures and dimensions in the schema"
     )
 
-    items_per_page: int = Field(
-        default=DEFAULT_API_ITEMS_PER_PAGE,
+    items_per_page: HiddenFromDocs[int] = Field(
+        DEFAULT_API_ITEMS_PER_PAGE,
         description="Number of items per page for paginated API requests.",
-        hidden_from_docs=True,
+    )
+
+    exclude_archived: bool = Field(
+        default=False, description="Exclude archived reports"
     )
 
     @validator("connect_uri")
@@ -1465,6 +1477,15 @@ class ModeSource(StatefulIngestionSourceBase):
                     logger.debug(
                         f"Read {len(reports_page)} reports records from workspace {self.workspace_uri} space {space_token}"
                     )
+                    if self.config.exclude_archived:
+                        logger.debug(
+                            f"Excluding archived reports since exclude_archived: {self.config.exclude_archived}"
+                        )
+                        reports_page = [
+                            report
+                            for report in reports_page
+                            if not report.get("archived", False)
+                        ]
                     yield reports_page
         except ModeRequestError as e:
             if isinstance(e, HTTPError) and e.response.status_code == 404:

datahub/ingestion/source/mongodb.py

@@ -36,7 +36,10 @@ from datahub.ingestion.api.decorators import (
 )
 from datahub.ingestion.api.source import MetadataWorkUnitProcessor
 from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.source.common.subtypes import DatasetContainerSubTypes
+from datahub.ingestion.source.common.subtypes import (
+    DatasetContainerSubTypes,
+    SourceCapabilityModifier,
+)
 from datahub.ingestion.source.schema_inference.object import (
     SchemaDescription,
     construct_schema,
@@ -249,6 +252,13 @@ def construct_schema_pymongo(
 @support_status(SupportStatus.CERTIFIED)
 @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
 @capability(SourceCapability.SCHEMA_METADATA, "Enabled by default")
+@capability(
+    SourceCapability.CONTAINERS,
+    "Enabled by default",
+    subtype_modifier=[
+        SourceCapabilityModifier.DATABASE,
+    ],
+)
 @dataclass
 class MongoDBSource(StatefulIngestionSourceBase):
     """

datahub/ingestion/source/neo4j/neo4j_source.py

@@ -1,7 +1,6 @@
 import logging
-import time
 from dataclasses import dataclass
-from typing import Any, Dict, Iterable, List, Optional, Type, Union
+from typing import Dict, Iterable, List, Optional, Tuple
 
 import pandas as pd
 from neo4j import GraphDatabase
@@ -11,11 +10,6 @@ from datahub.configuration.source_common import (
     EnvConfigMixin,
     PlatformInstanceConfigMixin,
 )
-from datahub.emitter.mce_builder import (
-    make_data_platform_urn,
-    make_dataset_urn_with_platform_instance,
-)
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -28,7 +22,6 @@ from datahub.ingestion.api.source import (
     MetadataWorkUnitProcessor,
     SourceCapability,
 )
-from datahub.ingestion.api.source_helpers import auto_workunit
 from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.ingestion.source.common.subtypes import DatasetSubTypes
 from datahub.ingestion.source.state.stale_entity_removal_handler import (
@@ -40,36 +33,14 @@ from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionReport,
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaFieldDataType
-from datahub.metadata.schema_classes import (
-    AuditStampClass,
-    BooleanTypeClass,
-    DatasetPropertiesClass,
-    DateTypeClass,
-    NullTypeClass,
-    NumberTypeClass,
-    OtherSchemaClass,
-    SchemaFieldClass,
-    SchemaMetadataClass,
-    StringTypeClass,
-    SubTypesClass,
-    UnionTypeClass,
-)
+from datahub.sdk.dataset import Dataset
 
 log = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-_type_mapping: Dict[Union[Type, str], Type] = {
-    "list": UnionTypeClass,
-    "boolean": BooleanTypeClass,
-    "integer": NumberTypeClass,
-    "local_date_time": DateTypeClass,
-    "float": NumberTypeClass,
-    "string": StringTypeClass,
-    "date": DateTypeClass,
-    "node": StringTypeClass,
-    "relationship": StringTypeClass,
-}
+# Neo4j object types
+_NODE = "node"
+_RELATIONSHIP = "relationship"
 
 
 class Neo4jConfig(
@@ -78,7 +49,6 @@ class Neo4jConfig(
     username: str = Field(description="Neo4j Username")
     password: str = Field(description="Neo4j Password")
     uri: str = Field(description="The URI for the Neo4j server")
-    env: str = Field(description="Neo4j env")
 
     stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None
 
@@ -96,8 +66,6 @@ class Neo4jSourceReport(StatefulIngestionReport):
 )
 @support_status(SupportStatus.CERTIFIED)
 class Neo4jSource(StatefulIngestionSourceBase):
-    NODE = "node"
-    RELATIONSHIP = "relationship"
     config: Neo4jConfig
     report: Neo4jSourceReport
 
@@ -113,78 +81,59 @@ class Neo4jSource(StatefulIngestionSourceBase):
         config = Neo4jConfig.parse_obj(config_dict)
         return cls(config, ctx)
 
-    def get_field_type(self, attribute_type: Union[type, str]) -> SchemaFieldDataType:
-        type_class: type = _type_mapping.get(attribute_type, NullTypeClass)
-        return SchemaFieldDataType(type=type_class())
-
-    def get_schema_field_class(
-        self, col_name: str, col_type: str, **kwargs: Any
-    ) -> SchemaFieldClass:
-        if kwargs["obj_type"] == self.NODE and col_type == self.RELATIONSHIP:
-            col_type = self.NODE
-        else:
-            col_type = col_type
-        return SchemaFieldClass(
-            fieldPath=col_name,
-            type=self.get_field_type(col_type),
-            nativeDataType=col_type,
-            description=col_type.upper()
-            if col_type in (self.NODE, self.RELATIONSHIP)
-            else col_type,
-            lastModified=AuditStampClass(
-                time=round(time.time() * 1000), actor="urn:li:corpuser:ingestion"
-            ),
+    def create_schema_field_tuple(
+        self, col_name: str, col_type: str, obj_type: Optional[str]
+    ) -> Tuple[str, str, str]:
+        """Convert Neo4j property to (field_name, field_type, description) tuple."""
+        # Special case: when a node has a relationship-typed property, treat it as a node reference
+        # This ensures relationship properties within nodes are described as "NODE" rather than "RELATIONSHIP"
+        column_type = (
+            _NODE if obj_type == _NODE and col_type == _RELATIONSHIP else col_type
+        )
+
+        description = (
+            column_type.upper()
+            if column_type in (_NODE, _RELATIONSHIP)
+            else column_type
         )
 
-    def add_properties(
+        return (col_name, column_type, description)
+
+    def get_subtype_from_obj_type(self, obj_type: str) -> str:
+        """Map Neo4j object type to DataHub subtype."""
+        if obj_type == _NODE:
+            return DatasetSubTypes.NEO4J_NODE
+        elif obj_type == _RELATIONSHIP:
+            return DatasetSubTypes.NEO4J_RELATIONSHIP
+        return DatasetSubTypes.NEO4J_NODE  # default fallback
+
+    def create_neo4j_dataset(
         self,
         dataset: str,
+        columns: list,
+        obj_type: Optional[str] = None,
         description: Optional[str] = None,
-        custom_properties: Optional[Dict[str, str]] = None,
-    ) -> Iterable[MetadataWorkUnit]:
-        dataset_properties = DatasetPropertiesClass(
-            description=description,
-            customProperties=custom_properties,
-        )
-        yield MetadataChangeProposalWrapper(
-            entityUrn=make_dataset_urn_with_platform_instance(
-                platform=self.platform,
-                name=dataset,
-                platform_instance=self.config.platform_instance,
-                env=self.config.env,
-            ),
-            aspect=dataset_properties,
-        ).as_workunit()
-
-    def generate_neo4j_object(
-        self, dataset: str, columns: list, obj_type: Optional[str] = None
-    ) -> Optional[MetadataChangeProposalWrapper]:
+    ) -> Optional[Dataset]:
+        """Create Dataset entity with Neo4j schema and metadata."""
         try:
-            fields = [
-                self.get_schema_field_class(key, value.lower(), obj_type=obj_type)
+            schema_fields = [
+                self.create_schema_field_tuple(
+                    col_name=key, col_type=value.lower(), obj_type=obj_type
+                )
                 for d in columns
                 for key, value in d.items()
             ]
-            return MetadataChangeProposalWrapper(
-                entityUrn=make_dataset_urn_with_platform_instance(
-                    platform=self.platform,
-                    name=dataset,
-                    platform_instance=self.config.platform_instance,
-                    env=self.config.env,
-                ),
-                aspect=SchemaMetadataClass(
-                    schemaName=dataset,
-                    platform=make_data_platform_urn(self.platform),
-                    version=0,
-                    hash="",
-                    platformSchema=OtherSchemaClass(rawSchema=""),
-                    lastModified=AuditStampClass(
-                        time=round(time.time() * 1000),
-                        actor="urn:li:corpuser:ingestion",
-                    ),
-                    fields=fields,
-                ),
+
+            return Dataset(
+                platform=self.platform,
+                name=dataset,
+                platform_instance=self.config.platform_instance,
+                env=self.config.env,
+                schema=schema_fields,
+                subtype=self.get_subtype_from_obj_type(obj_type or _NODE),
+                description=description,
             )
+
         except Exception as e:
             log.error(e)
             self.report.report_failure(
@@ -199,21 +148,24 @@ class Neo4jSource(StatefulIngestionSourceBase):
             self.config.uri, auth=(self.config.username, self.config.password)
         )
         """
-        This process retrieves the metadata for Neo4j objects using an APOC query, which returns a dictionary
-        with two columns: key and value. The key represents the Neo4j object, while the value contains the
-        corresponding metadata.
+        This process retrieves the metadata for Neo4j objects using an APOC query,
+        which returns a dictionary with two columns: key and value. The key represents
+        the Neo4j object, while the value contains the corresponding metadata.
 
-        When data is returned from Neo4j, much of the relationship metadata is stored with the relevant node's
-        metadata. Consequently, the objects are organized into two separate dataframes: one for nodes and one for
-        relationships.
+        When data is returned from Neo4j, much of the relationship metadata is stored
+        with the relevant node's metadata. Consequently, the objects are organized
+        into two separate dataframes: one for nodes and one for relationships.
 
-        In the node dataframe, several fields are extracted and added as new columns. Similarly, in the relationship
-        dataframe, certain fields are parsed out, while others require metadata from the nodes dataframe.
+        In the node dataframe, several fields are extracted and added as new columns.
+        Similarly, in the relationship dataframe, certain fields are parsed out,
+        while others require metadata from the nodes dataframe.
 
-        Once the data is parsed and these two dataframes are created, we combine a subset of their columns into a
-        single dataframe, which will be used to create the DataHub objects.
+        Once the data is parsed and these two dataframes are created, we combine
+        a subset of their columns into a single dataframe, which will be used to
+        create the DataHub objects.
 
-        See the docs for examples of metadata:  metadata-ingestion/docs/sources/neo4j/neo4j.md
+        See the docs for examples of metadata:
+        metadata-ingestion/docs/sources/neo4j/neo4j.md
         """
         try:
             log.info(f"{query}")
@@ -238,7 +190,7 @@ class Neo4jSource(StatefulIngestionSourceBase):
         return None
 
     def process_nodes(self, data: list) -> pd.DataFrame:
-        nodes = [record for record in data if record["value"]["type"] == self.NODE]
+        nodes = [record for record in data if record["value"]["type"] == _NODE]
         node_df = pd.DataFrame(
             nodes,
             columns=["key", "value"],
@@ -261,9 +213,7 @@ class Neo4jSource(StatefulIngestionSourceBase):
         return node_df
 
     def process_relationships(self, data: list, node_df: pd.DataFrame) -> pd.DataFrame:
-        rels = [
-            record for record in data if record["value"]["type"] == self.RELATIONSHIP
-        ]
+        rels = [record for record in data if record["value"]["type"] == _RELATIONSHIP]
         rel_df = pd.DataFrame(rels, columns=["key", "value"])
         rel_df["obj_type"] = rel_df["value"].apply(
             lambda record: self.get_obj_type(record)
@@ -331,51 +281,40 @@ class Neo4jSource(StatefulIngestionSourceBase):
         ]
 
     def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
-        df = self.get_neo4j_metadata(
-            "CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;"
+        query = (
+            "CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key "
+            "RETURN key, value[key] AS value;"
         )
+        df = self.get_neo4j_metadata(query)
         if df is None:
             log.warning("No metadata retrieved from Neo4j")
             return
 
         for _, row in df.iterrows():
             try:
-                neo4j_obj = self.generate_neo4j_object(
-                    columns=row["property_data_types"],
-                    dataset=row["key"],
-                )
-                if neo4j_obj:
-                    yield from auto_workunit([neo4j_obj])
-
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=make_dataset_urn_with_platform_instance(
-                        platform=self.platform,
-                        name=row["key"],
-                        platform_instance=self.config.platform_instance,
-                        env=self.config.env,
-                    ),
-                    aspect=SubTypesClass(
-                        typeNames=[
-                            DatasetSubTypes.NEO4J_NODE
-                            if row["obj_type"] == self.NODE
-                            else DatasetSubTypes.NEO4J_RELATIONSHIP
-                        ]
-                    ),
-                ).as_workunit()
-
-                yield from self.add_properties(
+                dataset_obj = self.create_neo4j_dataset(
                     dataset=row["key"],
-                    custom_properties=None,
+                    columns=row["property_data_types"],
+                    obj_type=row["obj_type"],
                     description=row["description"],
                 )
 
+                if dataset_obj:
+                    yield from dataset_obj.as_workunits()
+                    self.report.obj_created += 1
+                else:
+                    log.warning(f"Failed to create dataset object for {row['key']}")
+                    self.report.obj_failures += 1
+
             except Exception as e:
-                log.error(f"Failed to process row {row['key']}: {str(e)}")
-                self.report.report_failure(
-                    message="Error processing Neo4j metadata",
+                log.warning(f"Failed to process row {row['key']}: {str(e)}")
+                self.report.report_warning(
+                    title="Error processing Neo4j metadata",
+                    message="Some entities will be missed",
                     context=row["key"],
                     exc=e,
                 )
+                self.report.obj_failures += 1
 
     def get_report(self) -> "Neo4jSourceReport":
         return self.report

datahub/ingestion/source/nifi.py

@@ -72,7 +72,7 @@ NIFI = "nifi"
 # and here - https://github.com/psf/requests/issues/1573
 class SSLAdapter(HTTPAdapter):
     def __init__(self, certfile, keyfile, password=None):
-        self.context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        self.context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
         self.context.load_cert_chain(
             certfile=certfile, keyfile=keyfile, password=password
         )
@@ -166,7 +166,7 @@ class NifiSourceConfig(StatefulIngestionConfigBase, EnvConfigMixin):
     )
 
     @root_validator(skip_on_failure=True)
-    def validate_auth_params(cla, values):
+    def validate_auth_params(cls, values):
         if values.get("auth") is NifiAuthType.CLIENT_CERT and not values.get(
             "client_cert_file"
         ):

datahub/ingestion/source/openapi.py

@@ -333,7 +333,7 @@ class APISource(Source, ABC):
                     ),
                 )
                 yield wu
-            elif endpoint_dets["method"] != "get":
+            elif endpoint_dets["method"] != "GET":
                 self.report.report_warning(
                     title="Failed to Extract Endpoint Metadata",
                     message=f"No example provided for {endpoint_dets['method']}",

datahub/ingestion/source/powerbi/config.py

@@ -4,11 +4,10 @@ from enum import Enum
 from typing import Dict, List, Literal, Optional, Union
 
 import pydantic
-from pydantic import validator
-from pydantic.class_validators import root_validator
+from pydantic import root_validator, validator
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, HiddenFromDocs
 from datahub.configuration.source_common import DatasetSourceConfigMixin, PlatformDetail
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.api.incremental_lineage_helper import (
@@ -291,22 +290,18 @@ class PowerBiProfilingConfig(ConfigModel):
 class PowerBiDashboardSourceConfig(
     StatefulIngestionConfigBase, DatasetSourceConfigMixin, IncrementalLineageConfigMixin
 ):
-    platform_name: str = pydantic.Field(
-        default=Constant.PLATFORM_NAME, hidden_from_docs=True
-    )
+    platform_name: HiddenFromDocs[str] = pydantic.Field(default=Constant.PLATFORM_NAME)
 
-    platform_urn: str = pydantic.Field(
+    platform_urn: HiddenFromDocs[str] = pydantic.Field(
         default=builder.make_data_platform_urn(platform=Constant.PLATFORM_NAME),
-        hidden_from_docs=True,
     )
 
     # Organization Identifier
     tenant_id: str = pydantic.Field(description="PowerBI tenant identifier")
     # PowerBi workspace identifier
-    workspace_id: Optional[str] = pydantic.Field(
+    workspace_id: HiddenFromDocs[Optional[str]] = pydantic.Field(
         default=None,
         description="[deprecated] Use workspace_id_pattern instead",
-        hidden_from_docs=True,
     )
     # PowerBi workspace identifier
     workspace_id_pattern: AllowDenyPattern = pydantic.Field(
@@ -326,15 +321,14 @@ class PowerBiDashboardSourceConfig(
     # Dataset type mapping PowerBI support many type of data-sources. Here user needs to define what type of PowerBI
     # DataSource needs to be mapped to corresponding DataHub Platform DataSource. For example, PowerBI `Snowflake` is
     # mapped to DataHub `snowflake` PowerBI `PostgreSQL` is mapped to DataHub `postgres` and so on.
-    dataset_type_mapping: Union[Dict[str, str], Dict[str, PlatformDetail]] = (
-        pydantic.Field(
-            default_factory=default_for_dataset_type_mapping,
-            description="[deprecated] Use server_to_platform_instance instead. Mapping of PowerBI datasource type to "
-            "DataHub supported datasources."
-            "You can configured platform instance for dataset lineage. "
-            "See Quickstart Recipe for mapping",
-            hidden_from_docs=True,
-        )
+    dataset_type_mapping: HiddenFromDocs[
+        Union[Dict[str, str], Dict[str, PlatformDetail]]
+    ] = pydantic.Field(
+        default_factory=default_for_dataset_type_mapping,
+        description="[deprecated] Use server_to_platform_instance instead. Mapping of PowerBI datasource type to "
+        "DataHub supported datasources."
+        "You can configured platform instance for dataset lineage. "
+        "See Quickstart Recipe for mapping",
     )
     # PowerBI datasource's server to platform instance mapping
     server_to_platform_instance: Dict[
@@ -353,6 +347,19 @@ class PowerBiDashboardSourceConfig(
         "For example with an ODBC connection string 'DSN=database' where the database type "
         "is 'PostgreSQL' you would configure the mapping as 'database: postgres'.",
     )
+    # ODBC DSN to database (or database.schema) mapping
+    dsn_to_database_schema: Dict[str, str] = pydantic.Field(
+        default={},
+        description="A mapping of ODBC DSN to database names with optional schema names "
+        "(some database platforms such a MySQL use the table name pattern 'database.table', "
+        "while others use the pattern 'database.schema.table'). "
+        "This mapping is used in conjunction with ODBC SQL query parsing. "
+        "If SQL queries used with ODBC do not reference fully qualified tables names, "
+        "then you should configure mappings for your DSNs. "
+        "For example with an ODBC connection string 'DSN=database' where the database "
+        "is 'prod' you would configure the mapping as 'database: prod'. "
+        "If the database is 'prod' and the schema is 'data' then mapping would be 'database: prod.data'.",
+    )
     # deprecated warning
     _dataset_type_mapping = pydantic_field_deprecated(
         "dataset_type_mapping",
@@ -528,10 +535,9 @@ class PowerBiDashboardSourceConfig(
         "Increase this value if you encounter the 'M-Query Parsing Timeout' message in the connector report.",
     )
 
-    metadata_api_timeout: int = pydantic.Field(
+    metadata_api_timeout: HiddenFromDocs[int] = pydantic.Field(
         default=30,
         description="timeout in seconds for Metadata Rest Api.",
-        hidden_from_docs=True,
     )
 
     @root_validator(skip_on_failure=True)
@@ -614,3 +620,23 @@ class PowerBiDashboardSourceConfig(
                 "Please use `extract_dataset_schema: true`, otherwise dataset schema extraction will be skipped."
             )
         return values
+
+    @root_validator(skip_on_failure=True)
+    def validate_dsn_to_database_schema(cls, values: Dict) -> Dict:
+        if values.get("dsn_to_database_schema") is not None:
+            dsn_mapping = values.get("dsn_to_database_schema")
+            if not isinstance(dsn_mapping, dict):
+                raise ValueError("dsn_to_database_schema must contain key-value pairs")
+
+            for _key, value in dsn_mapping.items():
+                if not isinstance(value, str):
+                    raise ValueError(
+                        "dsn_to_database_schema mapping values must be strings"
+                    )
+                parts = value.split(".")
+                if len(parts) != 1 and len(parts) != 2:
+                    raise ValueError(
+                        f"dsn_to_database_schema invalid mapping value: {value}"
+                    )
+
+        return values

datahub/ingestion/source/powerbi/m_query/data_classes.py

@@ -76,3 +76,4 @@ class FunctionName(Enum):
     DATABRICK_MULTI_CLOUD_DATA_ACCESS = "DatabricksMultiCloud.Catalogs"
     MYSQL_DATA_ACCESS = "MySQL.Database"
     ODBC_DATA_ACCESS = "Odbc.DataSource"
+    ODBC_QUERY = "Odbc.Query"