openmetadata-ingestion 1.3.3.0rc1__py3-none-any.whl → 1.4.0.0rc1__py3-none-any.whl

metadata/profiler/processor/metric_filter.py

@@ -0,0 +1,265 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Metric filter class for profiler"""
+
+from typing import List, Optional, Set, Tuple, Type, Union, cast
+
+from sqlalchemy import Column
+
+from metadata.generated.schema.configuration.profilerConfiguration import (
+    MetricConfigurationDefinition,
+    ProfilerConfiguration,
+)
+from metadata.generated.schema.entity.data.table import (
+    ColumnProfilerConfig,
+    TableProfilerConfig,
+)
+from metadata.generated.schema.entity.services import databaseService
+from metadata.profiler.api.models import ThreadPoolMetrics
+from metadata.profiler.metrics.core import (
+    ComposedMetric,
+    CustomMetric,
+    HybridMetric,
+    QueryMetric,
+    StaticMetric,
+    SystemMetric,
+    TMetric,
+)
+from metadata.profiler.metrics.registry import Metrics
+from metadata.profiler.orm.converter.converter_registry import converter_registry
+from metadata.utils.sqa_like_column import SQALikeColumn
+
+
+class MetricFilter:
+    """Metric filter class for profiler"""
+
+    def __init__(
+        self,
+        metrics: Tuple[Type[TMetric]],
+        global_profiler_config: Optional[ProfilerConfiguration] = None,
+        table_profiler_config: Optional[TableProfilerConfig] = None,
+        column_profiler_config: Optional[List[ColumnProfilerConfig]] = None,
+    ):
+        self.metrics = metrics
+        self.global_profiler_config = global_profiler_config
+        self.table_profiler_config = table_profiler_config
+        self.column_profiler_config = column_profiler_config
+
+    @property
+    def static_metrics(self) -> List[Type[StaticMetric]]:
+        """Get static metrics.
+
+        Returns:
+            List[Type[StaticMetric]]:
+        """
+        return self.filter_by_type(StaticMetric)
+
+    @property
+    def composed_metrics(self) -> List[Type[ComposedMetric]]:
+        """Get composed metrics. Composed metrics are computed from other metrics.
+
+        Returns:
+            List[Type[ComposedMetric]]:
+        """
+        return self.filter_by_type(ComposedMetric)
+
+    @property
+    def custom_metrics(self) -> List[Type[CustomMetric]]:
+        """Get custom metrics. Custom metrics are user-defined metrics.
+
+        Returns:
+            List[Type[CustomMetric]]:
+        """
+        return self.filter_by_type(CustomMetric)
+
+    @property
+    def query_metrics(self) -> List[Type[QueryMetric]]:
+        """Get query metrics. Query metrics are computed from a query.
+
+        Returns:
+            List[Type[QueryMetric]]: _description_
+        """
+        return self.filter_by_type(QueryMetric)
+
+    @property
+    def system_metrics(self) -> List[Type[SystemMetric]]:
+        """Get system metrics. System metrics represent system-level metrics.
+
+        Returns:
+            List[Type[SystemMetric]]:
+        """
+        return self.filter_by_type(SystemMetric)
+
+    @property
+    def hybrid_metric(self) -> List[Type[HybridMetric]]:
+        """Get hybrid metrics. Hybrid metrics are a combination of different types of metrics.
+
+        Returns:
+            List[Type[HybridMetric]]:
+        """
+        return self.filter_by_type(HybridMetric)
+
+    @staticmethod
+    def filter_empty_metrics(
+        metric_funcs: List[ThreadPoolMetrics],
+    ) -> List[ThreadPoolMetrics]:
+        """filter thread pool object where metrics attribute is empty
+
+        Args:
+            thread_pool_metrics (List[ThreadPoolMetrics]): list of thread pool metrics to use in `get_all_metrics`
+
+        Returns:
+            List[ThreadPoolMetrics]
+        """
+        return [metric for metric in metric_funcs if metric.metrics]
+
+    def filter_by_type(self, _type: Type[TMetric]) -> List[Type[TMetric]]:
+        """filter a list of metric by type
+
+        Args:
+            _type (Type[TMetric]): metric type
+
+        Returns:
+            List[Type[TMetric]]:
+        """
+        return [metric for metric in self.metrics if issubclass(metric, _type)]
+
+    def filter_column_metrics_from_global_config(
+        self,
+        metrics: List[Type[TMetric]],
+        column: Union[Column, SQALikeColumn],
+        service_type: databaseService.DatabaseServiceType,
+    ) -> List[Optional[Type[TMetric]]]:
+        """Filter metrics based on profiler global configuration. We first check if we have config
+        or if the config has metricConfiguration. If not, we return all metrics. If we have config
+        we'll get the om Dtype from the SQA type (or directly from the SQALikeColumn for non SQA sources).
+        We'll then check if the om Dtype is present in the config. If it is disabled, we return an empty list of metrics
+        otherwise we'll filter the list of metrics set in the config.
+
+        Args:
+            column (Column): sqlalchemy column
+            service_type (DatabaseServiceType): database service type as per OpenMetadata
+                Note: not great to depend on this here, but we load specific data type based
+                sqa client library installed to limit core dependencies
+        Returns:
+            List[Type[TMetric]]
+        """
+        if not self.global_profiler_config or (
+            self.global_profiler_config
+            and not self.global_profiler_config.metricConfiguration
+        ):
+            return [metric for metric in metrics if metric.is_col_metric()]
+
+        self.global_profiler_config.metricConfiguration = cast(
+            List[MetricConfigurationDefinition],
+            self.global_profiler_config.metricConfiguration,
+        )
+
+        # TODO: improve the expected type. Currently Column will have SQA type while SQALikeColumn will have OM type
+        # Column will be returned by SQA sources while SQALikeColumn will be returned by other sources
+        if not isinstance(column, SQALikeColumn):
+            mapper = converter_registry[service_type]
+            sqa_to_om_types = mapper.map_sqa_to_om_types()
+            om_data_types: Optional[Set] = sqa_to_om_types.get(column.type, None)
+        else:
+            om_data_types = {column.type}
+
+        if not om_data_types:
+            return [metric for metric in metrics if metric.is_col_metric()]
+
+        col_dtype_config = next(
+            (
+                metric_config
+                for metric_config in self.global_profiler_config.metricConfiguration
+                if metric_config.dataType in om_data_types
+            ),
+            None,
+        )
+
+        if not col_dtype_config or (
+            not col_dtype_config.disabled and not col_dtype_config.metrics
+        ):
+            return [metric for metric in metrics if metric.is_col_metric()]
+
+        if col_dtype_config.disabled:
+            return []
+
+        metrics = [
+            Metric.value
+            for Metric in Metrics
+            if Metric.value.name() in {mtrc.value for mtrc in col_dtype_config.metrics}
+            and Metric.value in metrics
+        ]
+
+        return metrics
+
+    def filter_column_metrics_from_table_config(
+        self,
+        metrics: List[Type[TMetric]],
+        column: Union[Column, SQALikeColumn],
+    ) -> List[Type[TMetric]]:
+        """Filter column metrics based on table configuration. Table configuration can be source
+        either from the column config or the table config (column config takes precedence over table config)
+
+        Args:
+            column (Union[Column, SQALikeColumn]): _description_
+
+        Returns:
+            List[Type[TMetric]]:
+        """
+        if not self.table_profiler_config and not self.column_profiler_config:
+            return [metric for metric in metrics if metric.is_col_metric()]
+
+        columns_config = (
+            self.column_profiler_config
+            if self.column_profiler_config
+            else self.table_profiler_config.includeColumns
+        )
+        columns_config = cast(List[ColumnProfilerConfig], columns_config)
+        metric_names = next(
+            (
+                include_columns.metrics
+                for include_columns in columns_config
+                if include_columns.columnName == column.name
+            ),
+            None,
+        )
+
+        if not metric_names:
+            return [metric for metric in metrics if metric.is_col_metric()]
+
+        metrics = [
+            Metric.value
+            for Metric in Metrics
+            if Metric.value.name().lower() in {mtrc.lower() for mtrc in metric_names}
+            and Metric.value in metrics
+        ]
+        return [metric for metric in metrics if metric.is_col_metric()]
+
+    def get_column_metrics(
+        self,
+        metric_type: Type[TMetric],
+        column: Column,
+        service_type: Optional[databaseService.DatabaseServiceType],
+    ) -> List[Type[TMetric]]:
+        """Get column metrics. Column metrics are metrics computed for columns.
+
+        Returns:
+            List[Type[TMetric]]:
+        """
+        _metrics = self.filter_by_type(metric_type)
+        metrics = self.filter_column_metrics_from_global_config(
+            _metrics, column, service_type
+        )
+        if metrics:
+            metrics = self.filter_column_metrics_from_table_config(metrics, column)
+
+        return metrics

metadata/profiler/processor/processor.py

@@ -12,7 +12,7 @@
 Profiler Processor Step
 """
 import traceback
-from typing import cast
+from typing import Optional, cast
 
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
@@ -82,7 +82,9 @@ class ProfilerProcessor(Processor):
         return Either()
 
     @classmethod
-    def create(cls, config_dict: dict, _: OpenMetadata) -> "Step":
+    def create(
+        cls, config_dict: dict, _: OpenMetadata, pipeline_name: Optional[str] = None
+    ) -> "Step":
         config = parse_workflow_config_gracefully(config_dict)
         return cls(config=config)
 

metadata/profiler/processor/sampler/nosql/sampler.py

@@ -0,0 +1,72 @@
+from typing import Dict, List, Optional, Tuple
+
+from metadata.generated.schema.entity.data.table import ProfileSampleType, TableData
+from metadata.profiler.adaptors.nosql_adaptor import NoSQLAdaptor
+from metadata.profiler.processor.sampler.sampler_interface import SamplerInterface
+from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
+from metadata.utils.sqa_like_column import SQALikeColumn
+
+
+class NoSQLSampler(SamplerInterface):
+    client: NoSQLAdaptor
+
+    def _rdn_sample_from_user_query(self) -> List[Dict[str, any]]:
+        """
+        Get random sample from user query
+        """
+        limit = self._get_limit()
+        return self.client.query(
+            self.table, self.table.columns, self._profile_sample_query, limit
+        )
+
+    def _fetch_sample_data_from_user_query(self) -> TableData:
+        """
+        Fetch sample data based on a user query. Assuming the enging has one (example: MongoDB)
+        If the engine does not support a custom query, an error will be raised.
+        """
+        records = self._rdn_sample_from_user_query()
+        columns = [
+            SQALikeColumn(name=column.name.__root__, type=column.dataType)
+            for column in self.table.columns
+        ]
+        rows, cols = self.transpose_records(records, columns)
+        return TableData(rows=rows, columns=[c.name for c in cols])
+
+    def random_sample(self):
+        pass
+
+    def fetch_sample_data(self, columns: List[SQALikeColumn]) -> TableData:
+        if self._profile_sample_query:
+            return self._fetch_sample_data_from_user_query()
+        return self._fetch_sample_data(columns)
+
+    def _fetch_sample_data(self, columns: List[SQALikeColumn]):
+        """
+        returns sampled ometa dataframes
+        """
+        limit = self._get_limit()
+        records = self.client.scan(self.table, self.table.columns, limit)
+        rows, cols = self.transpose_records(records, columns)
+        return TableData(rows=rows, columns=[col.name for col in cols])
+
+    def _get_limit(self) -> Optional[int]:
+        num_rows = self.client.item_count(self.table)
+        if self.profile_sample_type == ProfileSampleType.PERCENTAGE:
+            limit = num_rows * (self.profile_sample / 100)
+        elif self.profile_sample_type == ProfileSampleType.ROWS:
+            limit = self.profile_sample
+        else:
+            limit = SAMPLE_DATA_DEFAULT_COUNT
+        return limit
+
+    @staticmethod
+    def transpose_records(
+        records: List[Dict[str, any]], columns: List[SQALikeColumn]
+    ) -> Tuple[List[List[any]], List[SQALikeColumn]]:
+        rows = []
+        for record in records:
+            row = []
+            for column in columns:
+                row.append(record.get(column.name))
+            rows.append(row)
+        return rows, columns

metadata/profiler/processor/sampler/pandas/sampler.py

@@ -20,7 +20,7 @@ from metadata.data_quality.validations.table.pandas.tableRowInsertedCountToBeBet
     TableRowInsertedCountToBeBetweenValidator,
 )
 from metadata.generated.schema.entity.data.table import (
-    PartitionIntervalType,
+    PartitionIntervalTypes,
     PartitionProfilerConfig,
     ProfileSampleType,
     TableData,
@@ -41,7 +41,7 @@ class DatalakeSampler(SamplerInterface):
         partition_field = self._partition_details.partitionColumnName
         if (
             self._partition_details.partitionIntervalType
-            == PartitionIntervalType.COLUMN_VALUE
+            == PartitionIntervalTypes.COLUMN_VALUE
         ):
             return [
                 df[df[partition_field].isin(self._partition_details.partitionValues)]
@@ -49,7 +49,7 @@ class DatalakeSampler(SamplerInterface):
             ]
         if (
             self._partition_details.partitionIntervalType
-            == PartitionIntervalType.INTEGER_RANGE
+            == PartitionIntervalTypes.INTEGER_RANGE
         ):
             return [
                 df[
@@ -136,7 +136,7 @@ class DatalakeSampler(SamplerInterface):
                 break
         return cols, rows
 
-    def random_sample(self):
+    def random_sample(self, is_sampled: bool = False):
         """Generate random sample from the table
 
         Returns:
@@ -148,9 +148,8 @@ class DatalakeSampler(SamplerInterface):
         if self._partition_details:
             self.table = self._partitioned_table()
 
-        if not self.profile_sample:
+        if not self.profile_sample or is_sampled:
             return self.table
-
         return self._get_sampled_dataframe()
 
     def _fetch_rows(self, data_frame):

metadata/profiler/processor/sampler/sampler_factory.py

@@ -21,10 +21,17 @@ from metadata.generated.schema.entity.services.connections.database.bigQueryConn
 from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
     DatalakeConnection,
 )
+from metadata.generated.schema.entity.services.connections.database.dynamoDBConnection import (
+    DynamoDBConnection,
+)
+from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
+    MongoDBConnection,
+)
 from metadata.generated.schema.entity.services.connections.database.trinoConnection import (
     TrinoConnection,
 )
 from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
+from metadata.profiler.processor.sampler.nosql.sampler import NoSQLSampler
 from metadata.profiler.processor.sampler.pandas.sampler import DatalakeSampler
 from metadata.profiler.processor.sampler.sqlalchemy.bigquery.sampler import (
     BigQuerySampler,
@@ -59,3 +66,5 @@ sampler_factory_.register(DatabaseConnection.__name__, SQASampler)
 sampler_factory_.register(BigQueryConnection.__name__, BigQuerySampler)
 sampler_factory_.register(DatalakeConnection.__name__, DatalakeSampler)
 sampler_factory_.register(TrinoConnection.__name__, TrinoSampler)
+sampler_factory_.register(MongoDBConnection.__name__, NoSQLSampler)
+sampler_factory_.register(DynamoDBConnection.__name__, NoSQLSampler)

metadata/profiler/processor/sampler/sampler_interface.py

@@ -17,7 +17,7 @@ from typing import Dict, List, Optional, Union
 
 from sqlalchemy import Column
 
-from metadata.generated.schema.entity.data.table import TableData
+from metadata.generated.schema.entity.data.table import Table, TableData
 from metadata.profiler.api.models import ProfileSampleConfig
 from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
 from metadata.utils.sqa_like_column import SQALikeColumn
@@ -29,7 +29,7 @@ class SamplerInterface(ABC):
     def __init__(
         self,
         client,
-        table,
+        table: Table,
         profile_sample_config: Optional[ProfileSampleConfig] = None,
         partition_details: Optional[Dict] = None,
         profile_sample_query: Optional[str] = None,

metadata/profiler/processor/sampler/sqlalchemy/sampler.py

@@ -21,7 +21,7 @@ from sqlalchemy.orm.util import AliasedClass
 from sqlalchemy.sql.sqltypes import Enum
 
 from metadata.generated.schema.entity.data.table import (
-    PartitionIntervalType,
+    PartitionIntervalTypes,
     PartitionProfilerConfig,
     ProfileSampleType,
     TableData,
@@ -224,7 +224,7 @@ class SQASampler(SamplerInterface):
 
         if (
             self._partition_details.partitionIntervalType
-            == PartitionIntervalType.COLUMN_VALUE
+            == PartitionIntervalTypes.COLUMN_VALUE
         ):
             return aliased(
                 self.table,
@@ -242,7 +242,7 @@ class SQASampler(SamplerInterface):
 
         if (
             self._partition_details.partitionIntervalType
-            == PartitionIntervalType.INTEGER_RANGE
+            == PartitionIntervalTypes.INTEGER_RANGE
         ):
             return aliased(
                 self.table,

metadata/profiler/source/base/profiler_source.py

@@ -18,6 +18,9 @@ from typing import List, Optional, Tuple, cast
 
 from sqlalchemy import MetaData
 
+from metadata.generated.schema.configuration.profilerConfiguration import (
+    ProfilerConfiguration,
+)
 from metadata.generated.schema.entity.data.database import Database
 from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
 from metadata.generated.schema.entity.data.table import ColumnProfilerConfig, Table
@@ -37,9 +40,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.profiler.api.models import ProfilerProcessorConfig, TableConfig
 from metadata.profiler.interface.profiler_interface import ProfilerInterface
-from metadata.profiler.interface.profiler_interface_factory import (
-    profiler_interface_factory,
-)
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.processor.core import Profiler
 from metadata.profiler.processor.default import DefaultProfiler, get_default_metrics
@@ -58,6 +58,7 @@ class ProfilerSource(ProfilerSourceInterface):
         config: OpenMetadataWorkflowConfig,
         database: DatabaseService,
         ometa_client: OpenMetadata,
+        global_profiler_configuration: ProfilerConfiguration,
     ):
         self.service_conn_config = self._copy_service_config(config, database)
         self.source_config = config.source.sourceConfig.config
@@ -71,6 +72,7 @@ class ProfilerSource(ProfilerSourceInterface):
         self.profiler_interface_type: str = self._get_profiler_interface_type(config)
         self.sqa_metadata = self._set_sqa_metadata()
         self._interface = None
+        self.global_profiler_configuration = global_profiler_configuration
 
     @property
     def interface(
@@ -197,6 +199,10 @@ class ProfilerSource(ProfilerSourceInterface):
         db_service: Optional[DatabaseService],
     ) -> ProfilerInterface:
         """Create sqlalchemy profiler interface"""
+        from metadata.profiler.interface.profiler_interface_factory import (  # pylint: disable=import-outside-toplevel
+            profiler_interface_factory,
+        )
+
         profiler_interface: ProfilerInterface = profiler_interface_factory.create(
             self.profiler_interface_type,
             entity,
@@ -273,6 +279,7 @@ class ProfilerSource(ProfilerSourceInterface):
                 profiler_interface=profiler_interface,
                 include_columns=self._get_include_columns(entity, table_config),
                 exclude_columns=self._get_exclude_columns(entity, table_config),
+                global_profiler_configuration=self.global_profiler_configuration,
             )
 
         metrics = (
@@ -290,4 +297,5 @@ class ProfilerSource(ProfilerSourceInterface):
             profiler_interface=profiler_interface,
             include_columns=self._get_include_columns(entity, table_config),
             exclude_columns=self._get_exclude_columns(entity, table_config),
+            global_profiler_configuration=self.global_profiler_configuration,
         )

metadata/profiler/source/databricks/profiler_source.py

@@ -1,5 +1,8 @@
 """Extend the ProfilerSource class to add support for Databricks is_disconnect SQA method"""
 
+from metadata.generated.schema.configuration.profilerConfiguration import (
+    ProfilerConfiguration,
+)
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
@@ -23,8 +26,9 @@ class DataBricksProfilerSource(ProfilerSource):
         config: OpenMetadataWorkflowConfig,
         database: DatabaseService,
         ometa_client: OpenMetadata,
+        global_profiler_config: ProfilerConfiguration,
     ):
-        super().__init__(config, database, ometa_client)
+        super().__init__(config, database, ometa_client, global_profiler_config)
         self.set_is_disconnect()
 
     def set_is_disconnect(self):

metadata/profiler/source/metadata.py

@@ -126,6 +126,7 @@ class OpenMetadataSource(Source):
         self.metadata.health_check()
 
     def _iter(self, *_, **__) -> Iterable[Either[ProfilerSourceAndEntity]]:
+        global_profiler_config = self.metadata.get_profiler_config_settings()
         for database in self.get_database_entities():
             try:
                 profiler_source = profiler_source_factory.create(
@@ -133,6 +134,7 @@ class OpenMetadataSource(Source):
                     self.config,
                     database,
                     self.metadata,
+                    global_profiler_config,
                 )
                 for entity in self.get_table_entities(database=database):
                     yield Either(
@@ -151,7 +153,12 @@ class OpenMetadataSource(Source):
                 )
 
     @classmethod
-    def create(cls, config_dict: dict, metadata: OpenMetadata) -> "Step":
+    def create(
+        cls,
+        config_dict: dict,
+        metadata: OpenMetadata,
+        pipeline_name: Optional[str] = None,
+    ) -> "Step":
         config = parse_workflow_config_gracefully(config_dict)
         return cls(config=config, metadata=metadata)
 

metadata/profiler/source/metadata_ext.py

@@ -117,6 +117,7 @@ class OpenMetadataSourceExt(OpenMetadataSource):
         self._connection = None  # Lazy init as well
 
     def _iter(self, *_, **__) -> Iterable[Either[ProfilerSourceAndEntity]]:
+        global_profiler_config = self.metadata.get_profiler_config_settings()
         for database_name in self.get_database_names():
             try:
                 database_entity = fqn.search_database_from_es(
@@ -150,6 +151,7 @@ class OpenMetadataSourceExt(OpenMetadataSource):
                             self.config,
                             database_entity,
                             self.metadata,
+                            global_profiler_config,
                         )
                         yield Either(
                             right=ProfilerSourceAndEntity(

metadata/readers/dataframe/avro.py

@@ -14,13 +14,8 @@ Avro DataFrame reader
 """
 import io
 
-from avro.datafile import DataFileReader
-from avro.errors import InvalidAvroBinaryEncoding
-from avro.io import DatumReader
-
 from metadata.generated.schema.entity.data.table import Column
 from metadata.generated.schema.type.schema import DataTypeTopic
-from metadata.parsers.avro_parser import parse_avro_schema
 from metadata.readers.dataframe.base import DataFrameReader
 from metadata.readers.dataframe.common import dataframe_to_chunks
 from metadata.readers.dataframe.models import DatalakeColumnWrapper
@@ -51,8 +46,13 @@ class AvroDataFrameReader(DataFrameReader):
         Method to parse the avro data from storage sources
         """
         # pylint: disable=import-outside-toplevel
+        from avro.datafile import DataFileReader
+        from avro.errors import InvalidAvroBinaryEncoding
+        from avro.io import DatumReader
         from pandas import DataFrame, Series
 
+        from metadata.parsers.avro_parser import parse_avro_schema
+
         try:
             elements = DataFileReader(io.BytesIO(avro_text), DatumReader())
             if elements.meta.get(AVRO_SCHEMA):

metadata/readers/dataframe/json.py

@@ -33,7 +33,7 @@ def _get_json_text(key: str, text: bytes, decode: bool) -> Union[str, bytes]:
         processed_text = gzip.decompress(text)
     if key.endswith(".zip"):
         with zipfile.ZipFile(io.BytesIO(text)) as zip_file:
-            processed_text = zip_file.read(zip_file.infolist()[0]).decode(UTF_8)
+            processed_text = zip_file.read(zip_file.infolist()[0])
     if decode:
         return processed_text.decode(UTF_8) if isinstance(text, bytes) else text
     return processed_text

metadata/readers/file/adls.py

@@ -68,3 +68,23 @@ class ADLSReader(Reader):
         to traverse any directories.
         """
         raise NotImplementedError("Not implemented")
+
+    def download(
+        self,
+        path: str,
+        local_file_path: str,
+        *,
+        bucket_name: str = None,
+        verbose: bool = True,
+        **__,
+    ):
+        try:
+            container_client = self.client.get_container_client(bucket_name)
+            with open(local_file_path, "wb") as download_file:
+                download_file.write(
+                    container_client.get_blob_client(path).download_blob().readall()
+                )
+        except Exception as err:
+            if verbose:
+                logger.debug(traceback.format_exc())
+            raise ReadException(f"Error downloading file [{path}] from ADLS: {err}")

metadata/readers/file/base.py

@@ -55,3 +55,9 @@ class Reader(ABC):
             logger.debug(traceback.format_exc())
             logger.error(f"Error getting file tree [{err}]")
         return None
+
+    def download(self, path: str, local_file_path: str, **kwargs):
+        """
+        Given a path, download the file
+        """
+        # To be implemented by required readers