acryl-datahub 0.15.0.6rc2__py3-none-any.whl → 1.0.0__py3-none-any.whl

datahub/ingestion/source/cassandra/cassandra.py

@@ -1,19 +1,14 @@
 import dataclasses
 import json
 import logging
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 from datahub.emitter.mce_builder import (
-    make_data_platform_urn,
-    make_dataplatform_instance_urn,
     make_dataset_urn_with_platform_instance,
     make_schema_field_urn,
 )
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.emitter.mcp_builder import (
     ContainerKey,
-    add_dataset_to_container,
-    gen_containers,
 )
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
@@ -31,6 +26,7 @@ from datahub.ingestion.source.cassandra.cassandra_api import (
     CassandraColumn,
     CassandraEntities,
     CassandraKeyspace,
+    CassandraSharedDatasetFields,
     CassandraTable,
     CassandraView,
 )
@@ -51,24 +47,21 @@ from datahub.ingestion.source.state.stale_entity_removal_handler import (
 from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionSourceBase,
 )
-from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass
 from datahub.metadata.com.linkedin.pegasus2avro.schema import (
     SchemaField,
-    SchemaMetadata,
 )
 from datahub.metadata.schema_classes import (
-    DataPlatformInstanceClass,
     DatasetLineageTypeClass,
-    DatasetPropertiesClass,
     FineGrainedLineageClass,
     FineGrainedLineageDownstreamTypeClass,
     FineGrainedLineageUpstreamTypeClass,
-    OtherSchemaClass,
-    SubTypesClass,
     UpstreamClass,
     UpstreamLineageClass,
     ViewPropertiesClass,
 )
+from datahub.sdk.container import Container
+from datahub.sdk.dataset import Dataset
+from datahub.sdk.entity import Entity
 
 logger = logging.getLogger(__name__)
 
@@ -133,6 +126,13 @@ class CassandraSource(StatefulIngestionSourceBase):
     def get_workunits_internal(
         self,
     ) -> Iterable[MetadataWorkUnit]:
+        for metadata in self._get_metadata():
+            if isinstance(metadata, MetadataWorkUnit):
+                yield metadata
+            else:
+                yield from metadata.as_workunits()
+
+    def _get_metadata(self) -> Iterable[Union[MetadataWorkUnit, Entity]]:
         if not self.cassandra_api.authenticate():
             return
         keyspaces: List[CassandraKeyspace] = self.cassandra_api.get_keyspaces()
@@ -145,7 +145,7 @@ class CassandraSource(StatefulIngestionSourceBase):
                 self.report.report_dropped(keyspace_name)
                 continue
 
-            yield from self._generate_keyspace_container(keyspace)
+            yield self._generate_keyspace_container(keyspace)
 
             try:
                 yield from self._extract_tables_from_keyspace(keyspace_name)
@@ -170,21 +170,20 @@ class CassandraSource(StatefulIngestionSourceBase):
         if self.config.is_profiling_enabled():
             yield from self.profiler.get_workunits(self.cassandra_data)
 
-    def _generate_keyspace_container(
-        self, keyspace: CassandraKeyspace
-    ) -> Iterable[MetadataWorkUnit]:
+    def _generate_keyspace_container(self, keyspace: CassandraKeyspace) -> Container:
         keyspace_container_key = self._generate_keyspace_container_key(
             keyspace.keyspace_name
         )
-        yield from gen_containers(
-            container_key=keyspace_container_key,
-            name=keyspace.keyspace_name,
+
+        return Container(
+            keyspace_container_key,
+            display_name=keyspace.keyspace_name,
             qualified_name=keyspace.keyspace_name,
+            subtype=DatasetContainerSubTypes.KEYSPACE,
             extra_properties={
                 "durable_writes": str(keyspace.durable_writes),
                 "replication": json.dumps(keyspace.replication),
             },
-            sub_types=[DatasetContainerSubTypes.KEYSPACE],
         )
 
     def _generate_keyspace_container_key(self, keyspace_name: str) -> ContainerKey:
@@ -196,105 +195,55 @@ class CassandraSource(StatefulIngestionSourceBase):
         )
 
     # get all tables for a given keyspace, iterate over them to extract column metadata
-    def _extract_tables_from_keyspace(
-        self, keyspace_name: str
-    ) -> Iterable[MetadataWorkUnit]:
+    def _extract_tables_from_keyspace(self, keyspace_name: str) -> Iterable[Dataset]:
         self.cassandra_data.keyspaces.append(keyspace_name)
         tables: List[CassandraTable] = self.cassandra_api.get_tables(keyspace_name)
         for table in tables:
-            # define the dataset urn for this table to be used downstream
-            table_name: str = table.table_name
-            dataset_name: str = f"{keyspace_name}.{table_name}"
-
-            if not self.config.table_pattern.allowed(dataset_name):
-                self.report.report_dropped(dataset_name)
-                continue
-
-            self.cassandra_data.tables.setdefault(keyspace_name, []).append(table_name)
-            self.report.report_entity_scanned(dataset_name, ent_type="Table")
-
-            dataset_urn = make_dataset_urn_with_platform_instance(
-                platform=self.platform,
-                name=dataset_name,
-                env=self.config.env,
-                platform_instance=self.config.platform_instance,
+            dataset = self._generate_table(keyspace_name, table)
+            if dataset:
+                yield dataset
+
+    def _generate_table(
+        self, keyspace_name: str, table: CassandraTable
+    ) -> Optional[Dataset]:
+        table_name: str = table.table_name
+        dataset_name: str = f"{keyspace_name}.{table_name}"
+
+        self.report.report_entity_scanned(dataset_name, ent_type="Table")
+        if not self.config.table_pattern.allowed(dataset_name):
+            self.report.report_dropped(dataset_name)
+            return None
+
+        self.cassandra_data.tables.setdefault(keyspace_name, []).append(table_name)
+
+        schema_fields = None
+        try:
+            schema_fields = self._extract_columns_from_table(keyspace_name, table_name)
+        except Exception as e:
+            self.report.failure(
+                message="Failed to extract columns from table",
+                context=dataset_name,
+                exc=e,
             )
 
-            # 1. Extract columns from table, then construct and emit the schemaMetadata aspect.
-            try:
-                yield from self._extract_columns_from_table(
-                    keyspace_name, table_name, dataset_urn
-                )
-            except Exception as e:
-                self.report.failure(
-                    message="Failed to extract columns from table",
-                    context=table_name,
-                    exc=e,
-                )
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=StatusClass(removed=False),
-            ).as_workunit()
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=SubTypesClass(
-                    typeNames=[
-                        DatasetSubTypes.TABLE,
-                    ]
-                ),
-            ).as_workunit()
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=DatasetPropertiesClass(
-                    name=table_name,
-                    qualifiedName=f"{keyspace_name}.{table_name}",
-                    description=table.comment,
-                    customProperties={
-                        "bloom_filter_fp_chance": str(table.bloom_filter_fp_chance),
-                        "caching": json.dumps(table.caching),
-                        "compaction": json.dumps(table.compaction),
-                        "compression": json.dumps(table.compression),
-                        "crc_check_chance": str(table.crc_check_chance),
-                        "dclocal_read_repair_chance": str(
-                            table.dclocal_read_repair_chance
-                        ),
-                        "default_time_to_live": str(table.default_time_to_live),
-                        "extensions": json.dumps(table.extensions),
-                        "gc_grace_seconds": str(table.gc_grace_seconds),
-                        "max_index_interval": str(table.max_index_interval),
-                        "min_index_interval": str(table.min_index_interval),
-                        "memtable_flush_period_in_ms": str(
-                            table.memtable_flush_period_in_ms
-                        ),
-                        "read_repair_chance": str(table.read_repair_chance),
-                        "speculative_retry": str(table.speculative_retry),
-                    },
-                ),
-            ).as_workunit()
-
-            yield from add_dataset_to_container(
-                container_key=self._generate_keyspace_container_key(keyspace_name),
-                dataset_urn=dataset_urn,
-            )
-
-            if self.config.platform_instance:
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=dataset_urn,
-                    aspect=DataPlatformInstanceClass(
-                        platform=make_data_platform_urn(self.platform),
-                        instance=make_dataplatform_instance_urn(
-                            self.platform, self.config.platform_instance
-                        ),
-                    ),
-                ).as_workunit()
+        return Dataset(
+            platform=self.platform,
+            name=dataset_name,
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+            subtype=DatasetSubTypes.TABLE,
+            parent_container=self._generate_keyspace_container_key(keyspace_name),
+            schema=schema_fields,
+            display_name=table_name,
+            qualified_name=dataset_name,
+            description=table.comment,
+            custom_properties=self._get_dataset_custom_props(table),
+        )
 
     # get all columns for a given table, iterate over them to extract column metadata
     def _extract_columns_from_table(
-        self, keyspace_name: str, table_name: str, dataset_urn: str
-    ) -> Iterable[MetadataWorkUnit]:
+        self, keyspace_name: str, table_name: str
+    ) -> Optional[List[SchemaField]]:
         column_infos: List[CassandraColumn] = self.cassandra_api.get_columns(
             keyspace_name, table_name
         )
@@ -305,147 +254,117 @@ class CassandraSource(StatefulIngestionSourceBase):
             self.report.report_warning(
                 message="Table has no columns, skipping", context=table_name
             )
-            return
+            return None
 
+        # Tricky: we also save the column info to a global store.
         jsonable_column_infos: List[Dict[str, Any]] = []
         for column in column_infos:
             self.cassandra_data.columns.setdefault(table_name, []).append(column)
             jsonable_column_infos.append(dataclasses.asdict(column))
 
-        schema_metadata: SchemaMetadata = SchemaMetadata(
-            schemaName=table_name,
-            platform=make_data_platform_urn(self.platform),
-            version=0,
-            hash="",
-            platformSchema=OtherSchemaClass(
-                rawSchema=json.dumps(jsonable_column_infos)
-            ),
-            fields=schema_fields,
-        )
-
-        yield MetadataChangeProposalWrapper(
-            entityUrn=dataset_urn,
-            aspect=schema_metadata,
-        ).as_workunit()
+        return schema_fields
 
-    def _extract_views_from_keyspace(
-        self, keyspace_name: str
-    ) -> Iterable[MetadataWorkUnit]:
+    def _extract_views_from_keyspace(self, keyspace_name: str) -> Iterable[Dataset]:
         views: List[CassandraView] = self.cassandra_api.get_views(keyspace_name)
         for view in views:
-            view_name: str = view.view_name
-            dataset_name: str = f"{keyspace_name}.{view_name}"
-            self.report.report_entity_scanned(dataset_name)
-            dataset_urn: str = make_dataset_urn_with_platform_instance(
-                platform=self.platform,
-                name=dataset_name,
-                env=self.config.env,
-                platform_instance=self.config.platform_instance,
+            dataset = self._generate_view(keyspace_name, view)
+            if dataset:
+                yield dataset
+
+    def _generate_view(
+        self, keyspace_name: str, view: CassandraView
+    ) -> Optional[Dataset]:
+        view_name: str = view.view_name
+        dataset_name: str = f"{keyspace_name}.{view_name}"
+
+        self.report.report_entity_scanned(dataset_name, ent_type="View")
+        if not self.config.table_pattern.allowed(dataset_name):
+            # TODO: Maybe add a view_pattern instead of reusing table_pattern?
+            self.report.report_dropped(dataset_name)
+            return None
+
+        schema_fields = None
+        try:
+            schema_fields = self._extract_columns_from_table(keyspace_name, view_name)
+        except Exception as e:
+            self.report.failure(
+                message="Failed to extract columns from views",
+                context=view_name,
+                exc=e,
             )
 
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=StatusClass(removed=False),
-            ).as_workunit()
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=SubTypesClass(
-                    typeNames=[
-                        DatasetSubTypes.VIEW,
-                    ]
-                ),
-            ).as_workunit()
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=ViewPropertiesClass(
+        dataset = Dataset(
+            platform=self.platform,
+            name=dataset_name,
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+            subtype=DatasetSubTypes.VIEW,
+            parent_container=self._generate_keyspace_container_key(keyspace_name),
+            schema=schema_fields,
+            display_name=view_name,
+            qualified_name=dataset_name,
+            description=view.comment,
+            custom_properties=self._get_dataset_custom_props(view),
+            extra_aspects=[
+                ViewPropertiesClass(
                     materialized=True,
                     viewLogic=view.where_clause,  # Use the WHERE clause as view logic
                     viewLanguage="CQL",  # Use "CQL" as the language
                 ),
-            ).as_workunit()
-
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=DatasetPropertiesClass(
-                    name=view_name,
-                    qualifiedName=f"{keyspace_name}.{view_name}",
-                    description=view.comment,
-                    customProperties={
-                        "bloom_filter_fp_chance": str(view.bloom_filter_fp_chance),
-                        "caching": json.dumps(view.caching),
-                        "compaction": json.dumps(view.compaction),
-                        "compression": json.dumps(view.compression),
-                        "crc_check_chance": str(view.crc_check_chance),
-                        "include_all_columns": str(view.include_all_columns),
-                        "dclocal_read_repair_chance": str(
-                            view.dclocal_read_repair_chance
-                        ),
-                        "default_time_to_live": str(view.default_time_to_live),
-                        "extensions": json.dumps(view.extensions),
-                        "gc_grace_seconds": str(view.gc_grace_seconds),
-                        "max_index_interval": str(view.max_index_interval),
-                        "min_index_interval": str(view.min_index_interval),
-                        "memtable_flush_period_in_ms": str(
-                            view.memtable_flush_period_in_ms
-                        ),
-                        "read_repair_chance": str(view.read_repair_chance),
-                        "speculative_retry": str(view.speculative_retry),
-                    },
-                ),
-            ).as_workunit()
+            ],
+        )
 
-            try:
-                yield from self._extract_columns_from_table(
-                    keyspace_name, view_name, dataset_urn
-                )
-            except Exception as e:
-                self.report.failure(
-                    message="Failed to extract columns from views",
-                    context=view_name,
-                    exc=e,
+        # Construct and emit lineage off of 'base_table_name'
+        # NOTE: we don't need to use 'base_table_id' since table is always in same keyspace, see https://docs.datastax.com/en/cql-oss/3.3/cql/cql_reference/cqlCreateMaterializedView.html#cqlCreateMaterializedView__keyspace-name
+        upstream_urn: str = make_dataset_urn_with_platform_instance(
+            platform=self.platform,
+            name=f"{keyspace_name}.{view.base_table_name}",
+            env=self.config.env,
+            platform_instance=self.config.platform_instance,
+        )
+        fineGrainedLineages = self.get_upstream_fields_of_field_in_datasource(
+            view_name, str(dataset.urn), upstream_urn
+        )
+        upstream_lineage = UpstreamLineageClass(
+            upstreams=[
+                UpstreamClass(
+                    dataset=upstream_urn,
+                    type=DatasetLineageTypeClass.VIEW,
                 )
+            ],
+            fineGrainedLineages=fineGrainedLineages,
+        )
 
-            # Construct and emit lineage off of 'base_table_name'
-            # NOTE: we don't need to use 'base_table_id' since table is always in same keyspace, see https://docs.datastax.com/en/cql-oss/3.3/cql/cql_reference/cqlCreateMaterializedView.html#cqlCreateMaterializedView__keyspace-name
-            upstream_urn: str = make_dataset_urn_with_platform_instance(
-                platform=self.platform,
-                name=f"{keyspace_name}.{view.table_name}",
-                env=self.config.env,
-                platform_instance=self.config.platform_instance,
-            )
-            fineGrainedLineages = self.get_upstream_fields_of_field_in_datasource(
-                view_name, dataset_urn, upstream_urn
-            )
-            yield MetadataChangeProposalWrapper(
-                entityUrn=dataset_urn,
-                aspect=UpstreamLineageClass(
-                    upstreams=[
-                        UpstreamClass(
-                            dataset=upstream_urn,
-                            type=DatasetLineageTypeClass.VIEW,
-                        )
-                    ],
-                    fineGrainedLineages=fineGrainedLineages,
-                ),
-            ).as_workunit()
-
-            yield from add_dataset_to_container(
-                container_key=self._generate_keyspace_container_key(keyspace_name),
-                dataset_urn=dataset_urn,
+        dataset.set_upstreams(upstream_lineage)
+
+        return dataset
+
+    def _get_dataset_custom_props(
+        self, dataset: CassandraSharedDatasetFields
+    ) -> Dict[str, str]:
+        props = {
+            "bloom_filter_fp_chance": str(dataset.bloom_filter_fp_chance),
+            "caching": json.dumps(dataset.caching),
+            "compaction": json.dumps(dataset.compaction),
+            "compression": json.dumps(dataset.compression),
+            "crc_check_chance": str(dataset.crc_check_chance),
+            "dclocal_read_repair_chance": str(dataset.dclocal_read_repair_chance),
+            "default_time_to_live": str(dataset.default_time_to_live),
+            "extensions": json.dumps(dataset.extensions),
+            "gc_grace_seconds": str(dataset.gc_grace_seconds),
+            "max_index_interval": str(dataset.max_index_interval),
+            "min_index_interval": str(dataset.min_index_interval),
+            "memtable_flush_period_in_ms": str(dataset.memtable_flush_period_in_ms),
+            "read_repair_chance": str(dataset.read_repair_chance),
+            "speculative_retry": str(dataset.speculative_retry),
+        }
+        if isinstance(dataset, CassandraView):
+            props.update(
+                {
+                    "include_all_columns": str(dataset.include_all_columns),
+                }
             )
-
-            if self.config.platform_instance:
-                yield MetadataChangeProposalWrapper(
-                    entityUrn=dataset_urn,
-                    aspect=DataPlatformInstanceClass(
-                        platform=make_data_platform_urn(self.platform),
-                        instance=make_dataplatform_instance_urn(
-                            self.platform, self.config.platform_instance
-                        ),
-                    ),
-                ).as_workunit()
+        return props
 
     def get_upstream_fields_of_field_in_datasource(
         self, table_name: str, dataset_urn: str, upstream_urn: str

datahub/ingestion/source/cassandra/cassandra_api.py

@@ -23,9 +23,9 @@ class CassandraKeyspace:
 
 
 @dataclass
-class CassandraTable:
+class CassandraSharedDatasetFields:
     keyspace_name: str
-    table_name: str
+
     bloom_filter_fp_chance: Optional[float]
     caching: Optional[Dict[str, str]]
     comment: Optional[str]
@@ -43,6 +43,11 @@ class CassandraTable:
     speculative_retry: Optional[str]
 
 
+@dataclass
+class CassandraTable(CassandraSharedDatasetFields):
+    table_name: str
+
+
 @dataclass
 class CassandraColumn:
     keyspace_name: str
@@ -55,8 +60,10 @@ class CassandraColumn:
 
 
 @dataclass
-class CassandraView(CassandraTable):
+class CassandraView(CassandraSharedDatasetFields):
     view_name: str
+
+    base_table_name: str
     include_all_columns: Optional[bool]
     where_clause: str = ""
 
@@ -152,7 +159,8 @@ class CassandraAPI:
             self.report.failure(message="Failed to authenticate to Cassandra", exc=e)
             return False
 
-    def get(self, query: str, parameters: Optional[List] = []) -> List:
+    def get(self, query: str, parameters: Optional[List] = None) -> List:
+        parameters = parameters or []
         if not self._cassandra_session:
             return []
 
@@ -261,7 +269,7 @@ class CassandraAPI:
             views = self.get(CassandraQueries.GET_VIEWS_QUERY, [keyspace_name])
             view_list = [
                 CassandraView(
-                    table_name=row.base_table_name,
+                    base_table_name=row.base_table_name,
                     keyspace_name=row.keyspace_name,
                     view_name=row.view_name,
                     bloom_filter_fp_chance=row.bloom_filter_fp_chance,

datahub/ingestion/source/common/gcp_credentials_config.py

@@ -0,0 +1,53 @@
+import json
+import tempfile
+from typing import Any, Dict, Optional
+
+from pydantic import Field, root_validator
+
+from datahub.configuration import ConfigModel
+from datahub.configuration.validate_multiline_string import pydantic_multiline_string
+
+
+class GCPCredential(ConfigModel):
+    project_id: Optional[str] = Field(description="Project id to set the credentials")
+    private_key_id: str = Field(description="Private key id")
+    private_key: str = Field(
+        description="Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n'"
+    )
+    client_email: str = Field(description="Client email")
+    client_id: str = Field(description="Client Id")
+    auth_uri: str = Field(
+        default="https://accounts.google.com/o/oauth2/auth",
+        description="Authentication uri",
+    )
+    token_uri: str = Field(
+        default="https://oauth2.googleapis.com/token", description="Token uri"
+    )
+    auth_provider_x509_cert_url: str = Field(
+        default="https://www.googleapis.com/oauth2/v1/certs",
+        description="Auth provider x509 certificate url",
+    )
+    type: str = Field(default="service_account", description="Authentication type")
+    client_x509_cert_url: Optional[str] = Field(
+        default=None,
+        description="If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email",
+    )
+
+    _fix_private_key_newlines = pydantic_multiline_string("private_key")
+
+    @root_validator(skip_on_failure=True)
+    def validate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        if values.get("client_x509_cert_url") is None:
+            values["client_x509_cert_url"] = (
+                f"https://www.googleapis.com/robot/v1/metadata/x509/{values['client_email']}"
+            )
+        return values
+
+    def create_credential_temp_file(self, project_id: Optional[str] = None) -> str:
+        configs = self.dict()
+        if project_id:
+            configs["project_id"] = project_id
+        with tempfile.NamedTemporaryFile(delete=False) as fp:
+            cred_json = json.dumps(configs, indent=4, separators=(",", ": "))
+            fp.write(cred_json.encode())
+            return fp.name

datahub/ingestion/source/common/subtypes.py

@@ -60,8 +60,15 @@ class BIContainerSubTypes(StrEnum):
     MODE_COLLECTION = "Collection"
 
 
+class FlowContainerSubTypes(StrEnum):
+    MSSQL_JOB = "Job"
+    MSSQL_PROCEDURE_CONTAINER = "Procedures Container"
+
+
 class JobContainerSubTypes(StrEnum):
     NIFI_PROCESS_GROUP = "Process Group"
+    MSSQL_JOBSTEP = "Job Step"
+    MSSQL_STORED_PROCEDURE = "Stored Procedure"
 
 
 class BIAssetSubTypes(StrEnum):
@@ -85,3 +92,8 @@ class BIAssetSubTypes(StrEnum):
     # SAP Analytics Cloud
     SAC_STORY = "Story"
     SAC_APPLICATION = "Application"
+
+
+class MLAssetSubTypes(StrEnum):
+    MLFLOW_TRAINING_RUN = "ML Training Run"
+    MLFLOW_EXPERIMENT = "ML Experiment"

datahub/ingestion/source/csv_enricher.py

@@ -314,7 +314,7 @@ class CSVEnricherSource(Source):
             "datajob": EditableDataJobPropertiesClass,
             "dataflow": EditableDataFlowPropertiesClass,
             "notebook": EditableNotebookPropertiesClass,
-        }.get(entityType, None)
+        }.get(entityType)
 
         if not entityClass:
             raise ValueError(
@@ -640,8 +640,8 @@ class CSVEnricherSource(Source):
                 )
             except Exception as e:
                 raise ConfigurationError(
-                    f"Cannot read remote file {self.config.filename}, error:{e}"
-                )
+                    f"Cannot read remote file {self.config.filename}: {e}"
+                ) from e
         else:
             with open(pathlib.Path(self.config.filename), encoding="utf-8-sig") as f:
                 rows = list(csv.DictReader(f, delimiter=self.config.delimiter))

datahub/ingestion/source/data_lake_common/path_spec.py

@@ -454,10 +454,8 @@ class PathSpec(ConfigModel):
                     return None
                 partition = partition_split[0]
                 # If partition is in the form of /value1/value2/value3 we infer it from the path and assign partition_0, partition_1, partition_2 etc
-                num = 0
-                for partition_value in partition.split("/"):
+                for num, partition_value in enumerate(partition.split("/")):
                     partition_keys.append((f"partition_{num}", partition_value))
-                    num += 1
             return partition_keys
 
         return None