acryl-datahub 1.2.0.9rc2__py3-none-any.whl → 1.2.0.10__py3-none-any.whl

datahub/ingestion/source/looker/lookml_config.py

@@ -122,7 +122,7 @@ class LookMLSourceConfig(
         description="List of regex patterns for LookML views to include in the extraction.",
     )
     parse_table_names_from_sql: bool = Field(True, description="See note below.")
-    api: Optional[LookerAPIConfig]
+    api: Optional[LookerAPIConfig] = None
     project_name: Optional[str] = Field(
         None,
         description="Required if you don't specify the `api` section. The project name within which all the model "

datahub/ingestion/source/looker/lookml_source.py

@@ -4,7 +4,7 @@ import tempfile
 from collections import OrderedDict
 from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Dict, Iterable, List, Optional, Set, Tuple
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
 
 import lkml
 import lkml.simple
@@ -12,8 +12,7 @@ from looker_sdk.error import SDKError
 
 from datahub.configuration.git import GitInfo
 from datahub.emitter.mce_builder import make_schema_field_urn
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
-from datahub.emitter.mcp_builder import gen_containers
+from datahub.emitter.mcp_builder import mcps_from_mce
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -77,7 +76,7 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.common import BrowsePaths, Status
+from datahub.metadata.com.linkedin.pegasus2avro.common import Status
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageTypeClass,
     FineGrainedLineageDownstreamType,
@@ -85,18 +84,15 @@ from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     UpstreamLineage,
     ViewProperties,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
-from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
 from datahub.metadata.schema_classes import (
     AuditStampClass,
-    BrowsePathEntryClass,
-    BrowsePathsV2Class,
-    ContainerClass,
     DatasetPropertiesClass,
     FineGrainedLineageClass,
     FineGrainedLineageUpstreamTypeClass,
-    SubTypesClass,
 )
+from datahub.sdk.container import Container
+from datahub.sdk.dataset import Dataset
+from datahub.sdk.entity import Entity
 from datahub.sql_parsing.sqlglot_lineage import ColumnRef
 
 VIEW_LANGUAGE_LOOKML: str = "lookml"
@@ -428,69 +424,40 @@ class LookMLSource(StatefulIngestionSourceBase):
 
         return dataset_props
 
-    def _build_dataset_mcps(
-        self, looker_view: LookerView
-    ) -> List[MetadataChangeProposalWrapper]:
-        view_urn = looker_view.id.get_urn(self.source_config)
-
-        subTypeEvent = MetadataChangeProposalWrapper(
-            entityUrn=view_urn,
-            aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW]),
-        )
-        events = [subTypeEvent]
+    def _build_dataset_entities(self, looker_view: LookerView) -> Iterable[Dataset]:
+        dataset_extra_aspects: List[Union[ViewProperties, Status]] = [
+            Status(removed=False)
+        ]
         if looker_view.view_details is not None:
-            viewEvent = MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.view_details,
-            )
-            events.append(viewEvent)
-
-        project_key = gen_project_key(self.source_config, looker_view.id.project_name)
-
-        container = ContainerClass(container=project_key.as_urn())
-        events.append(
-            MetadataChangeProposalWrapper(entityUrn=view_urn, aspect=container)
-        )
-
-        events.append(
-            MetadataChangeProposalWrapper(
-                entityUrn=view_urn,
-                aspect=looker_view.id.get_browse_path_v2(self.source_config),
-            )
-        )
-
-        return events
-
-    def _build_dataset_mce(self, looker_view: LookerView) -> MetadataChangeEvent:
-        """
-        Creates MetadataChangeEvent for the dataset, creating upstream lineage links
-        """
-        logger.debug(f"looker_view = {looker_view.id}")
+            dataset_extra_aspects.append(looker_view.view_details)
 
-        dataset_snapshot = DatasetSnapshot(
-            urn=looker_view.id.get_urn(self.source_config),
-            aspects=[],  # we append to this list later on
-        )
-        browse_paths = BrowsePaths(
-            paths=[looker_view.id.get_browse_path(self.source_config)]
-        )
-
-        dataset_snapshot.aspects.append(browse_paths)
-        dataset_snapshot.aspects.append(Status(removed=False))
-        upstream_lineage = self._get_upstream_lineage(looker_view)
-        if upstream_lineage is not None:
-            dataset_snapshot.aspects.append(upstream_lineage)
         schema_metadata = LookerUtil._get_schema(
             self.source_config.platform_name,
             looker_view.id.view_name,
             looker_view.fields,
             self.reporter,
         )
-        if schema_metadata is not None:
-            dataset_snapshot.aspects.append(schema_metadata)
-        dataset_snapshot.aspects.append(self._get_custom_properties(looker_view))
 
-        return MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
+        custom_properties: DatasetPropertiesClass = self._get_custom_properties(
+            looker_view
+        )
+
+        yield Dataset(
+            platform=self.source_config.platform_name,
+            name=looker_view.id.get_view_dataset_name(self.source_config),
+            display_name=looker_view.id.view_name,
+            platform_instance=self.source_config.platform_instance,
+            env=self.source_config.env,
+            subtype=DatasetSubTypes.VIEW,
+            parent_container=looker_view.id.get_view_dataset_parent_container(
+                self.source_config
+            ),
+            schema=schema_metadata,
+            custom_properties=custom_properties.customProperties,
+            external_url=custom_properties.externalUrl,
+            upstreams=self._get_upstream_lineage(looker_view),
+            extra_aspects=dataset_extra_aspects,
+        )
 
     def get_project_name(self, model_name: str) -> str:
         if self.source_config.project_name is not None:
@@ -554,7 +521,7 @@ class LookMLSource(StatefulIngestionSourceBase):
             ).workunit_processor,
         ]
 
-    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+    def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         with tempfile.TemporaryDirectory("lookml_tmp") as tmp_dir:
             # Clone the base_folder if necessary.
             if not self.source_config.base_folder:
@@ -715,7 +682,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                 tmp_dir, project, project_visited, manifest_constants
             )
 
-    def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+    def get_internal_workunits(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:  # noqa: C901
         assert self.source_config.base_folder
         viewfile_loader = LookerViewFileLoader(
             self.source_config.project_name,
@@ -949,7 +916,7 @@ class LookMLSource(StatefulIngestionSourceBase):
                                         maybe_looker_view.id.project_name
                                         not in self.processed_projects
                                     ):
-                                        yield from self.gen_project_workunits(
+                                        yield from self.gen_project_containers(
                                             maybe_looker_view.id.project_name
                                         )
 
@@ -957,15 +924,10 @@ class LookMLSource(StatefulIngestionSourceBase):
                                             maybe_looker_view.id.project_name
                                         )
 
-                                    for mcp in self._build_dataset_mcps(
+                                    yield from self._build_dataset_entities(
                                         maybe_looker_view
-                                    ):
-                                        yield mcp.as_workunit()
-                                    mce = self._build_dataset_mce(maybe_looker_view)
-                                    yield MetadataWorkUnit(
-                                        id=f"lookml-view-{maybe_looker_view.id}",
-                                        mce=mce,
                                     )
+
                                     processed_view_files.add(include.include)
                                 else:
                                     (
@@ -994,28 +956,24 @@ class LookMLSource(StatefulIngestionSourceBase):
             self.source_config.tag_measures_and_dimensions
             and self.reporter.events_produced != 0
         ):
-            # Emit tag MCEs for measures and dimensions:
+            # Emit tag MCEs for measures and dimensions if we produced any explores:
             for tag_mce in LookerUtil.get_tag_mces():
-                yield MetadataWorkUnit(
-                    id=f"tag-{tag_mce.proposedSnapshot.urn}", mce=tag_mce
-                )
+                # Convert MCE to MCPs
+                for mcp in mcps_from_mce(tag_mce):
+                    yield mcp.as_workunit()
 
-    def gen_project_workunits(self, project_name: str) -> Iterable[MetadataWorkUnit]:
+    def gen_project_containers(self, project_name: str) -> Iterable[Container]:
         project_key = gen_project_key(
             self.source_config,
             project_name,
         )
-        yield from gen_containers(
+
+        yield Container(
             container_key=project_key,
-            name=project_name,
-            sub_types=[BIContainerSubTypes.LOOKML_PROJECT],
+            display_name=project_name,
+            subtype=BIContainerSubTypes.LOOKML_PROJECT,
+            parent_container=["Folders"],
         )
-        yield MetadataChangeProposalWrapper(
-            entityUrn=project_key.as_urn(),
-            aspect=BrowsePathsV2Class(
-                path=[BrowsePathEntryClass("Folders")],
-            ),
-        ).as_workunit()
 
     def report_skipped_unreachable_views(
         self,

datahub/ingestion/source/metabase.py

@@ -13,7 +13,10 @@ from pydantic import Field, root_validator, validator
 from requests.models import HTTPError
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.source_common import DatasetLineageProviderConfigBase
+from datahub.configuration.source_common import (
+    DatasetLineageProviderConfigBase,
+    LowerCaseDatasetUrnConfigMixin,
+)
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SourceCapability,
@@ -61,7 +64,11 @@ logger = logging.getLogger(__name__)
 DATASOURCE_URN_RECURSION_LIMIT = 5
 
 
-class MetabaseConfig(DatasetLineageProviderConfigBase, StatefulIngestionConfigBase):
+class MetabaseConfig(
+    DatasetLineageProviderConfigBase,
+    StatefulIngestionConfigBase,
+    LowerCaseDatasetUrnConfigMixin,
+):
     # See the Metabase /api/session endpoint for details
     # https://www.metabase.com/docs/latest/api-documentation.html#post-apisession
     connect_uri: str = Field(default="localhost:3000", description="Metabase host URL.")

datahub/ingestion/source/metadata/business_glossary.py

@@ -5,11 +5,11 @@ import time
 from dataclasses import dataclass, field
 from typing import Any, Dict, Iterable, List, Optional, TypeVar, Union
 
-from pydantic import validator
+import pydantic
 from pydantic.fields import Field
 
 import datahub.metadata.schema_classes as models
-from datahub.configuration.common import ConfigModel
+from datahub.configuration.common import ConfigModel, LaxStr
 from datahub.configuration.config_loader import load_config_file
 from datahub.emitter.mce_builder import (
     datahub_guid,
@@ -66,7 +66,7 @@ class GlossaryTermConfig(ConfigModel):
     contains: Optional[List[str]] = None
     values: Optional[List[str]] = None
     related_terms: Optional[List[str]] = None
-    custom_properties: Optional[Dict[str, str]] = None
+    custom_properties: Optional[Dict[str, LaxStr]] = None
     knowledge_links: Optional[List[KnowledgeCard]] = None
     domain: Optional[str] = None
 
@@ -82,7 +82,7 @@ class GlossaryNodeConfig(ConfigModel):
     terms: Optional[List["GlossaryTermConfig"]] = None
     nodes: Optional[List["GlossaryNodeConfig"]] = None
     knowledge_links: Optional[List[KnowledgeCard]] = None
-    custom_properties: Optional[Dict[str, str]] = None
+    custom_properties: Optional[Dict[str, LaxStr]] = None
 
     # Private fields.
     _urn: str
@@ -108,12 +108,12 @@ class BusinessGlossarySourceConfig(ConfigModel):
 
 
 class BusinessGlossaryConfig(DefaultConfig):
-    version: str
+    version: LaxStr
     terms: Optional[List["GlossaryTermConfig"]] = None
     nodes: Optional[List["GlossaryNodeConfig"]] = None
 
-    @validator("version")
-    def version_must_be_1(cls, v):
+    @pydantic.field_validator("version", mode="after")
+    def version_must_be_1(cls, v: str) -> str:
         if v != "1":
             raise ValueError("Only version 1 is supported")
         return v

datahub/ingestion/source/metadata/lineage.py

@@ -49,7 +49,7 @@ class EntityConfig(EnvConfigMixin):
     name: str
     type: str
     platform: str
-    platform_instance: Optional[str]
+    platform_instance: Optional[str] = None
 
     @validator("type")
     def type_must_be_supported(cls, v: str) -> str:

datahub/ingestion/source/mode.py

@@ -7,7 +7,16 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from functools import lru_cache
 from json import JSONDecodeError
-from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union
+from typing import (
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 import dateutil.parser as dp
 import psutil
@@ -24,7 +33,7 @@ from requests.models import HTTPBasicAuth, HTTPError
 from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, HiddenFromDocs
 from datahub.configuration.source_common import (
     DatasetLineageProviderConfigBase,
 )
@@ -200,10 +209,9 @@ class ModeConfig(
         default=True, description="Tag measures and dimensions in the schema"
     )
 
-    items_per_page: int = Field(
-        default=DEFAULT_API_ITEMS_PER_PAGE,
+    items_per_page: HiddenFromDocs[int] = Field(
+        DEFAULT_API_ITEMS_PER_PAGE,
         description="Number of items per page for paginated API requests.",
-        hidden_from_docs=True,
     )
 
     @validator("connect_uri")

datahub/ingestion/source/nifi.py

@@ -166,7 +166,7 @@ class NifiSourceConfig(StatefulIngestionConfigBase, EnvConfigMixin):
     )
 
     @root_validator(skip_on_failure=True)
-    def validate_auth_params(cla, values):
+    def validate_auth_params(cls, values):
         if values.get("auth") is NifiAuthType.CLIENT_CERT and not values.get(
             "client_cert_file"
         ):

datahub/ingestion/source/powerbi/config.py

@@ -4,11 +4,10 @@ from enum import Enum
 from typing import Dict, List, Literal, Optional, Union
 
 import pydantic
-from pydantic import validator
-from pydantic.class_validators import root_validator
+from pydantic import root_validator, validator
 
 import datahub.emitter.mce_builder as builder
-from datahub.configuration.common import AllowDenyPattern, ConfigModel
+from datahub.configuration.common import AllowDenyPattern, ConfigModel, HiddenFromDocs
 from datahub.configuration.source_common import DatasetSourceConfigMixin, PlatformDetail
 from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated
 from datahub.ingestion.api.incremental_lineage_helper import (
@@ -291,22 +290,18 @@ class PowerBiProfilingConfig(ConfigModel):
 class PowerBiDashboardSourceConfig(
     StatefulIngestionConfigBase, DatasetSourceConfigMixin, IncrementalLineageConfigMixin
 ):
-    platform_name: str = pydantic.Field(
-        default=Constant.PLATFORM_NAME, hidden_from_docs=True
-    )
+    platform_name: HiddenFromDocs[str] = pydantic.Field(default=Constant.PLATFORM_NAME)
 
-    platform_urn: str = pydantic.Field(
+    platform_urn: HiddenFromDocs[str] = pydantic.Field(
         default=builder.make_data_platform_urn(platform=Constant.PLATFORM_NAME),
-        hidden_from_docs=True,
     )
 
     # Organization Identifier
     tenant_id: str = pydantic.Field(description="PowerBI tenant identifier")
     # PowerBi workspace identifier
-    workspace_id: Optional[str] = pydantic.Field(
+    workspace_id: HiddenFromDocs[Optional[str]] = pydantic.Field(
         default=None,
         description="[deprecated] Use workspace_id_pattern instead",
-        hidden_from_docs=True,
     )
     # PowerBi workspace identifier
     workspace_id_pattern: AllowDenyPattern = pydantic.Field(
@@ -326,15 +321,14 @@ class PowerBiDashboardSourceConfig(
     # Dataset type mapping PowerBI support many type of data-sources. Here user needs to define what type of PowerBI
     # DataSource needs to be mapped to corresponding DataHub Platform DataSource. For example, PowerBI `Snowflake` is
     # mapped to DataHub `snowflake` PowerBI `PostgreSQL` is mapped to DataHub `postgres` and so on.
-    dataset_type_mapping: Union[Dict[str, str], Dict[str, PlatformDetail]] = (
-        pydantic.Field(
-            default_factory=default_for_dataset_type_mapping,
-            description="[deprecated] Use server_to_platform_instance instead. Mapping of PowerBI datasource type to "
-            "DataHub supported datasources."
-            "You can configured platform instance for dataset lineage. "
-            "See Quickstart Recipe for mapping",
-            hidden_from_docs=True,
-        )
+    dataset_type_mapping: HiddenFromDocs[
+        Union[Dict[str, str], Dict[str, PlatformDetail]]
+    ] = pydantic.Field(
+        default_factory=default_for_dataset_type_mapping,
+        description="[deprecated] Use server_to_platform_instance instead. Mapping of PowerBI datasource type to "
+        "DataHub supported datasources."
+        "You can configured platform instance for dataset lineage. "
+        "See Quickstart Recipe for mapping",
     )
     # PowerBI datasource's server to platform instance mapping
     server_to_platform_instance: Dict[
@@ -541,10 +535,9 @@ class PowerBiDashboardSourceConfig(
         "Increase this value if you encounter the 'M-Query Parsing Timeout' message in the connector report.",
     )
 
-    metadata_api_timeout: int = pydantic.Field(
+    metadata_api_timeout: HiddenFromDocs[int] = pydantic.Field(
         default=30,
         description="timeout in seconds for Metadata Rest Api.",
-        hidden_from_docs=True,
     )
 
     @root_validator(skip_on_failure=True)

datahub/ingestion/source/preset.py

@@ -2,7 +2,7 @@ import logging
 from typing import Dict, Optional
 
 import requests
-from pydantic.class_validators import root_validator, validator
+from pydantic import root_validator, validator
 from pydantic.fields import Field
 
 from datahub.emitter.mce_builder import DEFAULT_ENV

datahub/ingestion/source/qlik_sense/data_classes.py

@@ -1,8 +1,9 @@
+from copy import deepcopy
 from datetime import datetime
 from enum import Enum
 from typing import Dict, List, Optional, Type, Union
 
-from pydantic import BaseModel, Field, root_validator
+from pydantic import BaseModel, ConfigDict, Field, root_validator
 
 from datahub.emitter.mcp_builder import ContainerKey
 from datahub.ingestion.source.qlik_sense.config import QLIK_DATETIME_FORMAT, Constant
@@ -78,7 +79,11 @@ PERSONAL_SPACE_DICT = {
 }
 
 
-class Space(BaseModel):
+class _QlikBaseModel(BaseModel):
+    model_config = ConfigDict(coerce_numbers_to_str=True)
+
+
+class Space(_QlikBaseModel):
     id: str
     name: str
     description: str
@@ -89,6 +94,9 @@ class Space(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
+
         values[Constant.CREATEDAT] = datetime.strptime(
             values[Constant.CREATEDAT], QLIK_DATETIME_FORMAT
         )
@@ -98,7 +106,7 @@ class Space(BaseModel):
         return values
 
 
-class Item(BaseModel):
+class Item(_QlikBaseModel):
     id: str
     description: str = ""
     ownerId: str
@@ -107,7 +115,7 @@ class Item(BaseModel):
     updatedAt: datetime
 
 
-class SchemaField(BaseModel):
+class SchemaField(_QlikBaseModel):
     name: str
     dataType: Optional[str] = None
     primaryKey: Optional[bool] = None
@@ -115,6 +123,8 @@ class SchemaField(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         values[Constant.DATATYPE] = values.get(Constant.DATATYPE, {}).get(Constant.TYPE)
         return values
 
@@ -130,6 +140,8 @@ class QlikDataset(Item):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         # Update str time to datetime
         values[Constant.CREATEDAT] = datetime.strptime(
             values[Constant.CREATEDTIME], QLIK_DATETIME_FORMAT
@@ -148,13 +160,13 @@ class QlikDataset(Item):
         return values
 
 
-class AxisProperty(BaseModel):
+class AxisProperty(_QlikBaseModel):
     Title: str = Field(alias="qFallbackTitle")
     Min: str = Field(alias="qMin")
     Max: str = Field(alias="qMax")
 
 
-class Chart(BaseModel):
+class Chart(_QlikBaseModel):
     qId: str
     visualization: str
     title: str
@@ -164,13 +176,15 @@ class Chart(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         values[Constant.QID] = values[Constant.QINFO][Constant.QID]
         values["qDimension"] = values[Constant.HYPERCUBE]["qDimensionInfo"]
         values["qMeasure"] = values[Constant.HYPERCUBE]["qMeasureInfo"]
         return values
 
 
-class Sheet(BaseModel):
+class Sheet(_QlikBaseModel):
     id: str
     title: str
     description: str
@@ -181,6 +195,8 @@ class Sheet(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         values[Constant.CREATEDAT] = datetime.strptime(
             values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT
         )
@@ -190,7 +206,7 @@ class Sheet(BaseModel):
         return values
 
 
-class QlikTable(BaseModel):
+class QlikTable(_QlikBaseModel):
     tableName: str
     type: BoxType = Field(alias="boxType")
     tableAlias: str
@@ -206,6 +222,8 @@ class QlikTable(BaseModel):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         values[Constant.DATACONNECTORID] = values[Constant.CONNECTIONINFO][Constant.ID]
         values[Constant.DATACONNECTORPLATFORM] = values[Constant.CONNECTIONINFO][
             Constant.SOURCECONNECTORID
@@ -223,6 +241,8 @@ class App(Item):
 
     @root_validator(pre=True)
     def update_values(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
         values[Constant.CREATEDAT] = datetime.strptime(
             values[Constant.CREATEDDATE], QLIK_DATETIME_FORMAT
         )

datahub/ingestion/source/redash.py

@@ -447,7 +447,7 @@ class RedashSource(StatefulIngestionSourceBase):
                 dataset_urns = sql_parser_in_tables.in_tables
                 if sql_parser_in_tables.debug_info.table_error:
                     self.report.queries_problem_parsing.add(str(query_id))
-                    self.error(
+                    self.warn(
                         logger,
                         "sql-parsing",
                         f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}",

datahub/ingestion/source/redshift/config.py

@@ -1,4 +1,5 @@
 import logging
+from copy import deepcopy
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
@@ -6,7 +7,7 @@ from pydantic import root_validator
 from pydantic.fields import Field
 
 from datahub.configuration import ConfigModel
-from datahub.configuration.common import AllowDenyPattern
+from datahub.configuration.common import AllowDenyPattern, HiddenFromDocs
 from datahub.configuration.source_common import DatasetLineageProviderConfigBase
 from datahub.configuration.validate_field_removal import pydantic_removed_field
 from datahub.configuration.validate_field_rename import pydantic_renamed_field
@@ -95,10 +96,9 @@ class RedshiftConfig(
     # Because of this behavior, it uses dramatically fewer round trips for
     # large Redshift warehouses. As an example, see this query for the columns:
     # https://github.com/sqlalchemy-redshift/sqlalchemy-redshift/blob/60b4db04c1d26071c291aeea52f1dcb5dd8b0eb0/sqlalchemy_redshift/dialect.py#L745.
-    scheme: str = Field(
+    scheme: HiddenFromDocs[str] = Field(
         default="redshift+redshift_connector",
         description="",
-        hidden_from_docs=True,
     )
 
     _database_alias_removed = pydantic_removed_field("database_alias")
@@ -216,6 +216,9 @@ class RedshiftConfig(
 
     @root_validator(skip_on_failure=True)
     def connection_config_compatibility_set(cls, values: Dict) -> Dict:
+        # Create a copy to avoid modifying the input dictionary, preventing state contamination in tests
+        values = deepcopy(values)
+
         if (
             ("options" in values and "connect_args" in values["options"])
             and "extra_client_options" in values