sibi-dst 0.3.64__py3-none-any.whl → 2025.1.2__py3-none-any.whl

sibi_dst/df_helper/backends/django/_io_dask.py

@@ -1,450 +0,0 @@
-import itertools
-
-import dask.dataframe as dd
-import django
-import pandas as pd
-from django.core.cache import cache
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-from django.db.models import Field
-from django.utils.encoding import force_str as force_text
-
-
-class ReadFrameDask:
-    """
-    Handles Django ORM QuerySet to Dask DataFrame conversion with support for field
-    type inference, chunked data retrieval, and verbose updates.
-
-    This class provides methods to efficiently convert a Django QuerySet into a
-    Dask DataFrame while preserving field types and incorporating additional
-    capabilities such as replacing fields with verbose choices or related object
-    information. The class design leverages static and class methods to maintain
-    flexibility and reusability for handling Django model fields and their data
-    types.
-
-    :ivar qs: The Django QuerySet to be converted into a Dask DataFrame.
-    :type qs: django.db.models.query.QuerySet
-    :ivar coerce_float: Whether to attempt to coerce numeric values to floats.
-    :type coerce_float: bool
-    :ivar chunk_size: The number of records to fetch and process per chunk from
-        the QuerySet.
-    :type chunk_size: int
-    :ivar verbose: If True, provides verbose updates during DataFrame creation
-        by replacing fields with readable representations (e.g., verbose names).
-    :type verbose: bool
-    """
-    FieldDoesNotExist = (
-        django.core.exceptions.FieldDoesNotExist
-        if django.VERSION < (1, 8)
-        else django.core.exceptions.FieldDoesNotExist
-    )
-
-    def __init__(
-            self,
-            qs,
-            **kwargs,
-    ):
-        """
-        An initialization method for a class that sets class attributes based on provided
-        arguments or default values using the keyword arguments. The method allows
-        customization of behaviors like coercing data types, handling chunked operations,
-        and verbosity level during execution.
-
-        :param qs: A data source or query set for processing; its type is dependent
-            on the expected data being handled.
-        :param kwargs: Additional keyword arguments that may include:
-            - coerce_float: A boolean indicating whether floats should be coerced
-              during handling. Default is False.
-            - chunk_size: An integer value representing the size of chunks for
-              data processing. Default is 1000.
-            - verbose: A boolean to specify if verbose logging or output
-              should occur during execution. Default is True.
-        """
-        self.qs = qs
-        self.coerce_float = kwargs.setdefault("coerce_float", False)
-        self.chunk_size = kwargs.setdefault("chunk_size", 1000)
-        self.verbose = kwargs.setdefault("verbose", True)
-
-    @staticmethod
-    def replace_from_choices(choices):
-        """
-        Provides a method to replace elements in a list of values based on a mapping of choices.
-
-        This static method generates a closure function that replaces items in a list by
-        looking up their corresponding values in a provided dictionary of choices. If an
-        item cannot be found in the dictionary, it is left unchanged.
-
-        :param choices:
-            Dictionary where keys are original values and values are their replacements.
-        :return:
-            A function that takes a list of values and replaces elements using the
-            provided choices dictionary.
-        """
-        def inner(values):
-            return [choices.get(v, v) for v in values]
-
-        return inner
-
-    @staticmethod
-    def get_model_name(model):
-        """
-        Retrieves the model name from a given Django model instance.
-
-        This method accesses the `_meta.model_name` attribute of the provided
-        model object to extract and return the model's name.
-
-        :param model: A Django model instance from which the model name is
-            derived.
-        :type model: object
-        :return: The name of the model as a string.
-        :rtype: str
-        """
-        return model._meta.model_name
-
-    @staticmethod
-    def get_related_model(field):
-        """
-        Retrieve the related model from the provided field.
-
-        This function determines the related model associated with the given field.
-        It checks various attributes commonly used to indicate relations in models and
-        retrieves the related model if present.
-
-        :param field: The field from which the related model is to be extracted.
-                      It must be an object that potentially contains attributes like
-                      `related_model` or `rel`.
-        :return: The related model associated with the provided field, or None if
-                 no such model is found.
-        """
-        model = None
-        if hasattr(field, "related_model") and field.related_model:
-            model = field.related_model
-        elif hasattr(field, "rel") and field.rel:
-            model = field.rel.to
-        return model
-
-    @classmethod
-    def get_base_cache_key(cls, model):
-        """
-        Generates a base cache key for caching purposes.
-
-        This method constructs a base cache key that can be used in conjunction with
-        Django models to uniquely identify cache entries. The key is formatted to
-        include the app label and model name, ensuring that cache entries are
-        namespaced accordingly.
-
-        :param model: A Django model instance for which the base cache key is generated.
-        :type model: Model
-        :return: The string template for the base cache key, where `%s` can be replaced
-                 with specific identifiers to create unique keys.
-        :rtype: str
-        """
-        return (
-            f"dask_{model._meta.app_label}_{cls.get_model_name(model)}_%s_rendering"
-        )
-
-    @classmethod
-    def replace_pk(cls, model):
-        """
-        Generates a function that replaces primary keys in a pandas Series with their
-        corresponding cached values or database-retrieved representations.
-
-        The function uses a cache mechanism to retrieve pre-stored values for primary
-        keys in the series. If some primary keys are not found in the cache, it queries
-        the database for their representations, updates the cache, and replaces the
-        primary keys in the series accordingly.
-
-        :param model: The Django model class associated with the primary keys to be
-            processed.
-        :type model: Type[Model]
-
-        :return: A function that takes a pandas Series of primary keys as input and
-            returns a Series with replaced values based on cache or database retrieval.
-        :rtype: callable
-        """
-        base_cache_key = cls.get_base_cache_key(model)
-
-        def get_cache_key_from_pk(pk):
-            return None if pk is None else base_cache_key % str(pk)
-
-        def inner(pk_series):
-            pk_series = pk_series.astype(object).where(pk_series.notnull(), None)
-            cache_keys = pk_series.apply(get_cache_key_from_pk, convert_dtype=False)
-            unique_cache_keys = list(filter(None, cache_keys.unique()))
-            if not unique_cache_keys:
-                return pk_series
-
-            out_dict = cache.get_many(unique_cache_keys)
-            if len(out_dict) < len(unique_cache_keys):
-                out_dict = dict(
-                    [
-                        (base_cache_key % obj.pk, force_text(obj))
-                        for obj in model.objects.filter(
-                        pk__in=list(filter(None, pk_series.unique()))
-                    )
-                    ]
-                )
-                cache.set_many(out_dict)
-            return list(map(out_dict.get, cache_keys))
-
-        return inner
-
-    @classmethod
-    def build_update_functions(cls, fieldnames, fields):
-        """
-        This method is responsible for building update functions based on the provided
-        fieldnames and fields. It performs validation for the field type, checks for
-        specific conditions such as `choices` or `ForeignKey` field types, and generates
-        a generator of update functions for the given fieldnames and fields.
-
-        :param fieldnames: A list of field names to be processed.
-        :type fieldnames: list[str]
-        :param fields: A list of field objects corresponding to the fieldnames.
-        :type fields: list[Field]
-        :return: A generator yielding tuples where the first element is a fieldname,
-                 and the second element is the corresponding update function or None.
-        :rtype: generator[tuple[str, Callable | None]]
-        """
-        for fieldname, field in zip(fieldnames, fields):
-            if not isinstance(field, Field):
-                yield fieldname, None
-            else:
-                if field.choices:
-                    choices = dict([(k, force_text(v)) for k, v in field.flatchoices])
-                    yield fieldname, cls.replace_from_choices(choices)
-                elif field.get_internal_type() == "ForeignKey":
-                    yield fieldname, cls.replace_pk(cls.get_related_model(field))
-
-    @classmethod
-    def update_with_verbose(cls, df, fieldnames, fields):
-        """
-        Updates the provided dataframe by applying transformation functions to specified fields.
-        The method iterates over the provided field names and their corresponding functions, applying
-        each transformation function to its related column in the dataframe.
-
-        :param df: The input dataframe to be updated.
-        :param fieldnames: A list of field names in the dataframe that need to be updated.
-        :param fields: A list of transformation functions or mappings corresponding to the field names.
-        :return: The dataframe with updated fields.
-        """
-        for fieldname, function in cls.build_update_functions(fieldnames, fields):
-            if function is not None:
-                df[fieldname] = df[fieldname].map_partitions(lambda x: function(x))
-
-    @classmethod
-    def to_fields(cls, qs, fieldnames):
-        """
-        Converts field names from a queryset into corresponding field objects, resolving relationships
-        and related objects if necessary. This method is typically used to yield fully-resolved field
-        objects for further interaction.
-
-        :param qs: A QuerySet object from which the fields are resolved. This object provides access
-                   to the model and its metadata from which the fields are retrieved.
-        :type qs: QuerySet
-
-        :param fieldnames: A list of field name strings. These can include nested fields separated by
-                           double underscores (__) to denote relationships or subfields.
-        :type fieldnames: List[str]
-
-        :return: A generator that yields resolved field objects corresponding to the provided field names.
-        :rtype: Generator[Field, None, None]
-        """
-        for fieldname in fieldnames:
-            model = qs.model
-            for fieldname_part in fieldname.split("__"):
-                try:
-                    field = model._meta.get_field(fieldname_part)
-                except cls.FieldDoesNotExist:
-                    try:
-                        rels = model._meta.get_all_related_objects_with_model()
-                    except AttributeError:
-                        field = fieldname
-                    else:
-                        for relobj, _ in rels:
-                            if relobj.get_accessor_name() == fieldname_part:
-                                field = relobj.field
-                                model = field.model
-                                break
-                else:
-                    model = cls.get_related_model(field)
-            yield field
-
-    @staticmethod
-    def is_values_queryset(qs):
-        """
-        Determines whether the provided queryset is a values queryset.
-
-        This method checks if the `_iterable_class` attribute of the queryset corresponds
-        to `django.db.models.query.ValuesIterable`. If an exception occurs during the check,
-        the method returns `False`.
-
-        :param qs: The queryset to be checked.
-        :type qs: django.db.models.query.QuerySet
-        :return: A boolean indicating whether the queryset is a values queryset.
-        :rtype: bool
-        """
-        try:
-            return qs._iterable_class == django.db.models.query.ValuesIterable
-        except:
-            return False
-
-    @staticmethod
-    def object_to_dict(obj, fields=None):
-        """
-        Converts an object to a dictionary representation.
-
-        This static method transforms an object's attributes into a dictionary.
-        If no specific fields are provided, all attribute key-value pairs are
-        included. The "_state" attribute, if present, is safely removed in this
-        case. When specific fields are supplied, only those fields are included
-        in the resulting dictionary.
-
-        :param obj: The object to be serialized into a dictionary. This object
-            must have the `__dict__` attribute available.
-        :param fields: A list of strings representing the attribute names to
-            include in the dictionary. If None or not provided, all attributes
-            are included except for "_state".
-        :return: A dictionary representation of the object's attributes. If the
-            provided object is None, an empty dictionary is returned.
-        :rtype: dict
-        """
-        if obj is None:
-            return {}  # Return an empty dictionary if obj is None
-        if not fields:
-            obj.__dict__.pop("_state", None)  # Remove _state safely
-            return obj.__dict__
-        return {field: obj.__dict__.get(field) for field in fields if field is not None}
-
-    @staticmethod
-    def infer_dtypes_from_django(qs):
-        """
-        Infer dtypes from a Django QuerySet model and annotated fields.
-
-        This method infers the appropriate data types (dtypes) for a given
-        Django QuerySet (`qs`) based on the fields defined in its model and
-        any annotated fields included in the QuerySet. The function maps
-        Django model field types to corresponding dtypes compatible with
-        Dask or Pandas dataframes.
-
-        - Fields in the model are identified through their metadata.
-        - Reverse relationships and non-concrete fields are ignored.
-        - Annotated fields are processed separately and default to object
-          dtype if their type cannot be determined.
-
-        :param qs: Django QuerySet whose model is used to infer dtypes.
-        :type qs: QuerySet
-        :return: A mapping of field names to inferred dtypes.
-        :rtype: dict
-        """
-        django_to_dask_dtype = {
-            'AutoField': 'Int64',  # Use nullable integer
-            'BigAutoField': 'Int64',
-            'BigIntegerField': 'Int64',
-            'BooleanField': 'bool',
-            'CharField': 'object',
-            'DateField': 'datetime64[ns]',
-            'DateTimeField': 'datetime64[ns]',
-            'DecimalField': 'float64',
-            'FloatField': 'float64',
-            'IntegerField': 'Int64',  # Use nullable integer
-            'PositiveIntegerField': 'Int64',
-            'SmallIntegerField': 'Int64',
-            'TextField': 'object',
-            'TimeField': 'object',
-            'UUIDField': 'object',
-            'ForeignKey': 'Int64',  # Use nullable integer for FK fields
-        }
-
-        dtypes = {}
-        # Handle model fields
-        for field in qs.model._meta.get_fields():
-            # Skip reverse relationships and non-concrete fields
-            if not getattr(field, 'concrete', False):
-                continue
-
-            # Check for AutoField or BigAutoField explicitly
-            if isinstance(field, (models.AutoField, models.BigAutoField)):
-                dtypes[field.name] = 'Int64'  # Nullable integer for autoincremented fields
-            else:
-                # Use field type to infer dtype
-                field_type = field.get_internal_type()
-                dtypes[field.name] = django_to_dask_dtype.get(field_type, 'object')
-
-        # Handle annotated fields
-        for annotation_name, annotation in qs.query.annotation_select.items():
-            if hasattr(annotation, 'output_field'):
-                field_type = annotation.output_field.get_internal_type()
-                dtype = django_to_dask_dtype.get(field_type, 'object')
-            else:
-                dtype = 'object'  # Default to object for untyped annotations
-            dtypes[annotation_name] = dtype
-
-        return dtypes
-
-    def read_frame(self, fillna_value=None):
-        """
-        Reads a Django QuerySet and returns a dask DataFrame by iterating over the QuerySet in chunks. It
-        handles data type inference, missing values, timezone awareness, and creates partitions to form a
-        single dask DataFrame efficiently.
-
-        This method includes functionality for managing missing values, inferring data types from Django fields,
-        and handling timezone-aware datetime objects. It processes data in chunks to optimize memory usage and
-        supports converting chunks into pandas DataFrames before combining them into a unified dask DataFrame.
-
-        :param fillna_value: The value to fill NaN values in the DataFrame. If None, NaNs are not filled.
-        :type fillna_value: Any
-        :return: A dask DataFrame constructed from the QuerySet after processing and combining all
-                 its partitions.
-        :rtype: dask.dataframe.DataFrame
-        """
-        qs = self.qs
-        coerce_float = self.coerce_float
-        verbose = self.verbose
-        chunk_size = self.chunk_size
-
-        fields = qs.model._meta.fields
-        fieldnames = [f.name for f in fields]
-        fieldnames += list(qs.query.annotation_select.keys())
-        fieldnames = tuple(fieldnames)
-        # Infer dtypes from Django fields
-        dtypes = self.infer_dtypes_from_django(qs)
-        if fieldnames:
-            dtypes = {field: dtype for field, dtype in dtypes.items() if field in fieldnames}
-
-        # Create partitions for Dask by iterating through chunks
-        partitions = []
-        iterator = iter(qs.iterator(chunk_size=chunk_size))
-
-        while True:
-            chunk = list(itertools.islice(iterator, chunk_size))
-            if not chunk:
-                break
-
-            # Convert chunk to DataFrame with inferred dtypes
-            df = pd.DataFrame.from_records(
-                [self.object_to_dict(obj, fieldnames) for obj in chunk],
-                columns=fieldnames,
-                coerce_float=coerce_float,
-            )
-            # Handle NaN values before casting, if specified
-            if fillna_value is not None:
-                df = df.fillna(fillna_value)
-
-            # Convert timezone-aware columns to timezone-naive if needed
-            for col in df.columns:
-                if isinstance(df[col].dtype, pd.DatetimeTZDtype):
-                    df[col] = df[col].dt.tz_localize(None)
-
-            # Convert to the appropriate data types
-            df = df.astype(dtypes)
-            partitions.append(dd.from_pandas(df, npartitions=1))
-
-        # Concatenate partitions into a single Dask DataFrame
-        # Ensure all partitions have the same columns
-
-        dask_df = dd.concat(partitions, axis=0, ignore_index=True)
-
-        if verbose:
-            self.update_with_verbose(dask_df, fieldnames, fields)
-
-        return dask_df

sibi_dst/df_helper/backends/django/_load_from_db.py

@@ -1,227 +0,0 @@
-import warnings
-
-import dask.dataframe as dd
-import pandas as pd
-from django.db.models import Q
-
-from sibi_dst.df_helper.backends.django import ReadFrameDask
-from sibi_dst.df_helper.core import django_field_conversion_map_dask
-from sibi_dst.utils import Logger
-
-
-class DjangoLoadFromDb:
-    """
-    Handles loading data from a Django database into a Dask DataFrame, with support for filtering
-    and column type conversion.
-
-    This class is designed to interface with Django ORM models, allowing data querying and mapping
-    Django model fields to Dask DataFrame columns. It accommodates filtering logic provided via
-    parameters and ensures that excessive data is not accidentally loaded when no filters are applied.
-
-    :ivar connection_config: Configuration for the database connection, including the Django model
-        and connection details.
-    :type connection_config: Any
-    :ivar query_config: Configuration for the query, including the number of records to retrieve.
-    :type query_config: Any
-    :ivar params_config: Configuration for query parameters, including filters and DataFrame options.
-    :type params_config: Any
-    :ivar logger: Logger instance used for debugging and reporting runtime information.
-    :type logger: Logger
-    :ivar debug: Indicates whether debug mode is active for verbose logging.
-    :type debug: bool
-    :ivar df: Dask DataFrame to hold the loaded query results.
-    :type df: dd.DataFrame
-    """
-    df: dd.DataFrame
-
-    def __init__(self, db_connection, db_query, db_params, logger, **kwargs):
-        """
-        This class initializes and configures a database connection along with the
-        specified query and parameters. It ensures the required model is defined
-        and sets up logging. Additional configurations can be provided via keyword
-        arguments.
-
-        :param db_connection: The configuration object representing the database
-            connection details.
-        :type db_connection: Any
-        :param db_query: The configuration or object for defining the database
-            query.
-        :type db_query: Any
-        :param db_params: The configuration or object for defining parameters
-            to be passed to the query.
-        :type db_params: Any
-        :param logger: An instance of a logging class used to log debug or
-            error messages, defaults to the class's default logger if not
-            specified.
-        :type logger: Any, optional
-        :param kwargs: Additional keyword arguments for custom configurations
-            like `debug`. These can include optional parameters to be parsed by
-            `params_config`.
-        :type kwargs: dict
-        :raises ValueError: If no model is specified in the given database
-            connection configuration.
-        """
-        self.connection_config = db_connection
-        self.debug = kwargs.pop('debug', False)
-        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-        if self.connection_config.model is None:
-            if self.debug:
-                self.logger.debug('Model must be specified')
-
-            raise ValueError('Model must be specified')
-
-        self.query_config = db_query
-        self.params_config = db_params
-        self.params_config.parse_params(kwargs)
-
-    def build_and_load(self):
-        """
-        Builds and loads data into a DataFrame by invoking the `_build_and_load` method.
-        This is a utility method designed to perform specific operations for constructing
-        and preparing the data. The loaded data will then be assigned to the instance
-        attribute `df`.
-
-        :param self: Reference to the current instance of the class.
-        :type self: object
-
-        :return: DataFrame containing the built and loaded data.
-        """
-        self.df = self._build_and_load()
-        # self.df = self._convert_columns(self.df)
-        return self.df
-
-    def _build_and_load(self) -> dd.DataFrame:
-        """
-        Builds and loads a Dask DataFrame based on the provided query and configuration. This method queries the data
-        model using the specified connection, applies filters if provided, and converts the query result into a
-        Dask DataFrame. If filters are not provided, only the first `n_records` entries are processed to avoid
-        unintentionally loading the entire table.
-
-        :raises Exception: If an error occurs while loading the query, it logs the error and initializes an
-            empty Dask DataFrame.
-
-        :return: A Dask DataFrame containing the queried data. If no filters or valid results are provided,
-            an empty Dask DataFrame is returned.
-        :rtype: dd.DataFrame
-        """
-        query = self.connection_config.model.objects.using(self.connection_config.connection_name)
-        if not self.params_config.filters:
-            # IMPORTANT: if no filters are provided show only the first n_records
-            # this is to prevent loading the entire table by mistake
-            n_records = self.query_config.n_records if self.query_config.n_records else 100
-            queryset = query.all()[:n_records]
-        else:
-            q_objects = self.__build_query_objects(self.params_config.filters, self.query_config.use_exclude)
-            queryset = query.filter(q_objects)
-        if queryset is not None:
-            try:
-                self.df = ReadFrameDask(queryset, **self.params_config.df_params).read_frame()
-            except Exception as e:
-                self.logger.debug(f'Error loading query: {str(queryset.query)}, error message: {e}')
-                self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-        else:
-            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-
-        return self.df
-
-    @staticmethod
-    def __build_query_objects(filters: dict, use_exclude: bool):
-        """
-        Constructs and returns a composite Q object based on the provided `filters` dictionary.
-        The function determines whether to include or exclude the filter conditions in the final
-        query based on the `use_exclude` parameter. If `use_exclude` is False, the filters are
-        directly added to the composite Q object. If `use_exclude` is True, the negation of
-        the filters is added instead.
-
-        :param filters: A dictionary containing filter conditions where keys represent field names
-            and values represent the conditions to be applied.
-        :type filters: dict
-        :param use_exclude: A boolean flag determining whether to exclude (`True`) or include
-            (`False`) the provided filter conditions.
-        :type use_exclude: bool
-        :return: A composite Q object that aggregates the filters based on the given conditions.
-        :rtype: Q
-        """
-        q_objects = Q()
-        for key, value in filters.items():
-            if not use_exclude:
-                q_objects.add(Q(**{key: value}), Q.AND)
-            else:
-                q_objects.add(~Q(**{key: value}), Q.AND)
-        return q_objects
-
-    def _convert_columns(self, df: dd.DataFrame) -> dd.DataFrame:
-        """
-        [DEPRECATED] Convert the data types of columns in a Dask DataFrame based on the field type in the Django model.
-
-        This function is deprecated and will be removed in a future release. The method converts the data
-        types of columns in a Dask DataFrame to match their corresponding field types defined in a Django model.
-        It emits warnings and logs deprecation notes. The conversions are applied lazily and partition-wise
-        to support distributed computation.
-
-        :param df: Dask DataFrame whose columns' data types are to be converted.
-        :type df: dd.DataFrame
-        :return: Dask DataFrame with converted column data types.
-        :rtype: dd.DataFrame
-        """
-        """
-            [DEPRECATED] Convert the data types of columns in a Dask DataFrame based on the field type in the Django model.
-
-            :param df: Dask DataFrame whose columns' data types are to be converted.
-            :return: Dask DataFrame with converted column data types.
-            """
-        # Emit deprecation warning
-        warnings.warn(
-            "_convert_columns is deprecated and will be removed in a future release. "
-            "Consider using <new_method_name> instead.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-
-        # Log deprecation message if debug mode is enabled
-        if self.debug:
-            self.logger.warning(
-                "[DEPRECATION NOTICE] The `_convert_columns` method is deprecated and will be removed in a future release. "
-                "Consider using <new_method_name> instead."
-            )
-
-        self.logger.debug(f'Converting columns: {list(df.columns)}')
-
-        # Get field information from the Django model
-        model_fields = self.connection_config.model._meta.get_fields()
-        field_type_map = {field.name: type(field).__name__ for field in model_fields}
-        # Simplified loop to apply conversions partition-wise
-        for field_name, field_type in field_type_map.items():
-            if field_name not in df.columns:
-                self.logger.debug(f"Column '{field_name}' not found in DataFrame columns.")
-                continue
-
-            conversion_func = django_field_conversion_map_dask.get(field_type)
-            if not conversion_func:
-                message = f"Field type '{field_type}' not found in conversion_map."
-                self.logger.debug(message)
-                continue
-
-            def apply_conversion(partition):
-                """
-                Apply the conversion function to a single partition for the given column.
-                """
-                try:
-                    if field_name in partition.columns:
-                        partition[field_name] = conversion_func(partition[field_name])
-                except Exception as e:
-                    self.logger.debug(f"Error converting column '{field_name}' in partition: {str(e)}")
-                return partition
-
-            try:
-                # Apply conversion lazily to each partition
-                df = df.map_partitions(
-                    apply_conversion,
-                    meta=df,
-                )
-                self.logger.debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.")
-            except Exception as e:
-                self.logger.debug(f"Failed to queue conversion for column '{field_name}': {str(e)}")
-
-        return df