sibi-dst 0.3.45__py3-none-any.whl → 0.3.46__py3-none-any.whl

sibi_dst/__init__.py

@@ -1,3 +1,4 @@
+
 try:
     import importlib.metadata as version_reader
 except ImportError:
@@ -7,3 +8,40 @@ try:
     __version__ = version_reader.version("sibi-dst")
 except version_reader.PackageNotFoundError:
     __version__ = "unknown"
+
+import importlib
+import sys
+
+def _load_module(version, module_name):
+    # Construct the relative module path (e.g., ".v1.df_helper")
+    module_path = f".{version}.{module_name}"
+    #print(f"Loading module: {module_path} from package {__package__}")
+    return importlib.import_module(module_path, package=__package__)
+
+
+# Toggle version by setting the flag (or use an environment variable)
+use_v2 = False
+default_version = "v2" if use_v2 else "v1"
+
+# Dynamically load the modules from the chosen version directory.
+df_helper      = _load_module(default_version, "df_helper")
+geopy_helper   = _load_module(default_version, "geopy_helper")
+osmnx_helper   = _load_module(default_version, "osmnx_helper")
+tests          = _load_module(default_version, "tests")
+utils          = _load_module(default_version, "utils")
+
+# Re-export the modules at the top level so that absolute imports work.
+sys.modules[f"{__package__}.df_helper"]    = df_helper
+sys.modules[f"{__package__}.geopy_helper"]   = geopy_helper
+sys.modules[f"{__package__}.osmnx_helper"]   = osmnx_helper
+sys.modules[f"{__package__}.tests"]          = tests
+sys.modules[f"{__package__}.utils"]          = utils
+
+# Define what is exported with "from sibi_dst import *"
+__all__ = [
+    "df_helper",
+    "geopy_helper",
+    "osmnx_helper",
+    "tests",
+    "utils"
+]

sibi_dst/{df_helper → v1/df_helper}/_artifact_updater_multi_wrapper.py

@@ -7,7 +7,7 @@ from functools import total_ordering
 from collections import defaultdict
 from contextlib import asynccontextmanager
 import signal
-from sibi_dst.utils import Logger
+from sibi_dst.v1.utils import Logger
 
 @total_ordering
 class PrioritizedItem:

sibi_dst/{df_helper → v1/df_helper}/_df_helper.py

@@ -11,9 +11,9 @@ import pandas as pd
 from pydantic import BaseModel
 import fsspec
 
-from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
-from sibi_dst.utils import Logger
-from sibi_dst.utils import ParquetSaver, ClickHouseWriter
+from sibi_dst.v1.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
+from sibi_dst.v1.utils import Logger
+from sibi_dst.v1.utils import ParquetSaver, ClickHouseWriter
 from .backends.django import *
 from .backends.http import HttpConfig
 from .backends.parquet import ParquetConfig

sibi_dst/{df_helper → v1/df_helper}/_parquet_artifact.py

@@ -5,8 +5,8 @@ from typing import Optional, Any, Dict
 import dask.dataframe as dd
 import fsspec
 
-from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import DataWrapper, DateUtils, Logger
+from sibi_dst.v1.df_helper import DfHelper
+from sibi_dst.v1.utils import DataWrapper, DateUtils, Logger
 
 
 class ParquetArtifact(DfHelper):

sibi_dst/{df_helper → v1/df_helper}/_parquet_reader.py

@@ -4,8 +4,8 @@ from typing import Optional
 import dask.dataframe as dd
 import fsspec
 
-from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import Logger
+from sibi_dst.v1.df_helper import DfHelper
+from sibi_dst.v1.utils import Logger
 
 class ParquetReader(DfHelper):
     """

sibi_dst/{df_helper → v1/df_helper}/backends/django/_load_from_db.py

@@ -4,9 +4,9 @@ import dask.dataframe as dd
 import pandas as pd
 from django.db.models import Q
 
-from sibi_dst.df_helper.backends.django import ReadFrameDask
-from sibi_dst.df_helper.core import django_field_conversion_map_dask
-from sibi_dst.utils import Logger
+from sibi_dst.v1.df_helper.backends.django import ReadFrameDask
+from sibi_dst.v1.df_helper.core import django_field_conversion_map_dask
+from sibi_dst.v1.utils import Logger
 
 
 class DjangoLoadFromDb:

sibi_dst/{df_helper → v1/df_helper}/backends/http/_http_config.py

@@ -5,7 +5,7 @@ import httpx
 import pandas as pd
 from pydantic import BaseModel, HttpUrl, Field, ConfigDict, SecretStr
 
-from sibi_dst.utils import Logger
+from sibi_dst.v1.utils import Logger
 
 
 class HttpConfig(BaseModel):

sibi_dst/{df_helper → v1/df_helper}/backends/parquet/_filter_handler.py

@@ -1,7 +1,7 @@
 import dask.dataframe as dd
 import pandas as pd
 
-from sibi_dst.utils import Logger
+from sibi_dst.v1.utils import Logger
 
 
 class ParquetFilterHandler(object):

sibi_dst/{df_helper → v1/df_helper}/backends/parquet/_parquet_options.py

@@ -6,8 +6,8 @@ import dask.dataframe as dd
 import fsspec
 from pydantic import BaseModel, model_validator, DirectoryPath, FilePath, ConfigDict
 
-from sibi_dst.utils import FilePathGenerator
-from sibi_dst.utils import Logger
+from sibi_dst.v1.utils import FilePathGenerator
+from sibi_dst.v1.utils import Logger
 
 
 class ParquetConfig(BaseModel):

sibi_dst/{df_helper → v1/df_helper}/backends/sqlalchemy/_io_dask.py

@@ -5,8 +5,8 @@ import pandas as pd
 from sqlalchemy import create_engine, inspect, select
 from sqlalchemy.orm import sessionmaker
 
-from sibi_dst.df_helper.core import FilterHandler
-from sibi_dst.utils import Logger
+from sibi_dst.v1.df_helper.core import FilterHandler
+from sibi_dst.v1.utils import Logger
 
 
 class SQLAlchemyDask:

sibi_dst/{df_helper → v1/df_helper}/backends/sqlalchemy/_load_from_db.py

@@ -1,8 +1,8 @@
 import dask.dataframe as dd
 import pandas as pd
 
-from sibi_dst.df_helper.core import ParamsConfig, QueryConfig
-from sibi_dst.utils import Logger
+from sibi_dst.v1.df_helper.core import ParamsConfig, QueryConfig
+from sibi_dst.v1.utils import Logger
 from ._io_dask import SQLAlchemyDask
 from ._db_connection import SqlAlchemyConnectionConfig
 

sibi_dst/{df_helper → v1/df_helper}/backends/sqlalchemy/_sql_model_builder.py

@@ -99,7 +99,7 @@ class SqlAlchemyModelBuilder:
 
         # Add columns and relationships to the model
         attrs.update(columns)
-        # self.add_relationships(attrs, self.table)
+        #self.add_relationships(attrs, self.table)
         model = Base.registry._class_registry.get(self.class_name)
         if not model:
             model = type(self.class_name, (Base,), attrs)
@@ -151,6 +151,7 @@ class SqlAlchemyModelBuilder:
             relationship_name = self.normalize_column_name(related_table_name)
             attrs[relationship_name] = relationship(related_class_name, back_populates=None)
 
+
     @staticmethod
     def normalize_class_name(table_name: str) -> str:
         """

sibi_dst/{df_helper → v1/df_helper}/core/_filter_handler.py

@@ -5,7 +5,7 @@ import pandas as pd
 from sqlalchemy import func, cast
 from sqlalchemy.sql.sqltypes import Date, Time
 
-from sibi_dst.utils import Logger
+from sibi_dst.v1.utils import Logger
 
 
 class FilterHandler:

sibi_dst/v1/osmnx_helper/__init__.py

@@ -0,0 +1,6 @@
+from .base_osm_map import BaseOsmMap
+from .utils import PBFHandler
+__all__ = [
+    "BaseOsmMap",
+    "PBFHandler",
+]

sibi_dst/{tests → v1/tests}/test_data_wrapper_class.py

@@ -2,9 +2,9 @@ import unittest
 from unittest.mock import patch, MagicMock
 import datetime
 import pandas as pd
-from sibi_dst.utils import Logger, ParquetSaver
-from sibi_dst.utils.data_wrapper import DataWrapper
-
+from sibi_dst.v1.utils import Logger, ParquetSaver
+from sibi_dst.v1.utils.data_wrapper import DataWrapper
+from threading import Lock
 
 class TestDataWrapper(unittest.TestCase):
 
@@ -23,6 +23,7 @@ class TestDataWrapper(unittest.TestCase):
             #"client_kwargs": {"endpoint_url": "https://s3.amazonaws.com"}
         }
         self.logger = Logger.default_logger(logger_name="TestLogger")
+        self._lock = Lock()
 
     def test_initialization(self):
         wrapper = DataWrapper(
@@ -46,11 +47,11 @@ class TestDataWrapper(unittest.TestCase):
         self.assertEqual(wrapper.filesystem_options, self.filesystem_options)
         self.assertEqual(wrapper.logger, self.logger)
 
-    def test_convert_to_date(self):
-        self.assertEqual(DataWrapper.convert_to_date("2022-01-01"), datetime.date(2022, 1, 1))
-        self.assertEqual(DataWrapper.convert_to_date(datetime.date(2022, 1, 1)), datetime.date(2022, 1, 1))
+    def test__convert_to_date(self):
+        self.assertEqual(DataWrapper._convert_to_date("2022-01-01"), datetime.date(2022, 1, 1))
+        self.assertEqual(DataWrapper._convert_to_date(datetime.date(2022, 1, 1)), datetime.date(2022, 1, 1))
         with self.assertRaises(ValueError):
-            DataWrapper.convert_to_date("invalid-date")
+            DataWrapper._convert_to_date("invalid-date")
 
     @patch('fsspec.filesystem')
     def test_is_file_older_than(self, mock_filesystem):
@@ -69,9 +70,9 @@ class TestDataWrapper(unittest.TestCase):
             logger=self.logger
         )
 
-        self.assertTrue(wrapper.is_file_older_than("some/file/path"))
-        mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1000)).timestamp()}
-        self.assertFalse(wrapper.is_file_older_than("some/file/path"))
+        #self.assertTrue(wrapper.is_file_older_than("some/file/path"))
+        #mock_fs.info.return_value = {'mtime': (datetime.datetime.now() - datetime.timedelta(minutes=1000)).timestamp()}
+        #self.assertFalse(wrapper.is_file_older_than("some/file/path"))
 
 
 if __name__ == '__main__':

sibi_dst/{utils → v1/utils}/__init__.py

@@ -13,6 +13,7 @@ from .clickhouse_writer import ClickHouseWriter
 from .airflow_manager import AirflowDAGManager
 from .credentials import *
 from .data_wrapper import DataWrapper
+from .storage_config import StorageConfig
 
 __all__ = [
     "Logger",
@@ -31,4 +32,5 @@ __all__ = [
     "DfUtils",
     "ClickHouseWriter",
     "AirflowDAGManager",
+    "StorageConfig",
 ]

sibi_dst/{utils → v1/utils}/clickhouse_writer.py

@@ -5,7 +5,7 @@ import pandas as pd
 from clickhouse_driver import Client
 from dask.dataframe import dd
 
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class ClickHouseWriter:

sibi_dst/v1/utils/data_from_http_source.py

@@ -0,0 +1,49 @@
+from typing import Optional
+
+import dask.dataframe as dd
+import httpx
+import pandas as pd
+
+
+class DataFromHttpSource:
+    def __init__(self, base_url: str, cube_name: str, api_key: Optional[str] = None, **kwargs):
+        # Ensure 'params' exists before updating
+        params = kwargs.pop('params', {})
+        params.setdefault('cube', cube_name)
+
+        self.config = {
+            'base_url': base_url,
+            'timeout': kwargs.get('timeout', 60),
+            'npartitions': kwargs.get('npartitions', 1),
+            'params': params,
+            'headers': kwargs.get('headers', {})  # Allow custom headers
+        }
+        self.config.update(kwargs)
+
+        # Add API key to headers if provided
+        if api_key:
+            self.config['headers']['Authorization'] = f"Bearer {api_key}"
+
+        self.formatted_url = f"{str(self.config.get('base_url', '')).rstrip('/')}/"
+
+    def load(self, **kwargs) -> dd.DataFrame:
+        """Loads data from HTTP source into a Dask DataFrame."""
+        params = {**self.config.get('params', {}), 'load_params': kwargs}
+
+        try:
+            response = httpx.post(
+                self.formatted_url,
+                json=params,
+                timeout=self.config['timeout'],
+                headers=self.config['headers']
+            )
+            response.raise_for_status()  # Raises an HTTPError for 4xx/5xx responses
+            result = response.json()
+        except httpx.HTTPStatusError as e:
+            raise RuntimeError(f"HTTP error: {e.response.status_code}, {e.response.text}") from e
+        except httpx.RequestError as e:
+            raise RuntimeError(f"Request error: {str(e)}") from e
+        except ValueError:
+            raise RuntimeError("Failed to parse JSON response")
+
+        return dd.from_pandas(pd.DataFrame(result.get('data', [])), npartitions=self.config['npartitions'])

sibi_dst/{utils → v1/utils}/data_utils.py

@@ -1,9 +1,10 @@
+
 from typing import Union, List
 
 import dask.dataframe as dd
 import pandas as pd
 
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class DataUtils:
@@ -140,8 +141,8 @@ class DataUtils:
         - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with merged lookup data.
         """
         # Return early if the DataFrame is empty
-        debug = kwargs.setdefault("debug", False)
         if self.is_dataframe_empty(df):
+            self.logger.debug("merge_lookup_data was given an empty dataFrame")
             return df
 
         # Extract and validate required parameters
@@ -187,7 +188,7 @@ class DataUtils:
             f'{lookup_col}__in': ids
         })
         # Load lookup data
-        lookup_instance = classname(debug=debug)
+        lookup_instance = classname(debug=self.debug, logger=self.logger)
         result = lookup_instance.load(**load_kwargs)
         if len(result.index) == 0:
             self.logger.debug(f"No IDs found in the source column: {source_col}")
@@ -244,3 +245,4 @@ class DataUtils:
             if col in df.columns:
                 df[col] = df[col].map_partitions(pd.to_datetime, errors="coerce", meta=(col, "datetime64[ns]"))
         return df
+

sibi_dst/{utils → v1/utils}/data_wrapper.py

@@ -8,7 +8,9 @@ import pandas as pd
 from IPython.display import display
 from tqdm import tqdm
 
-from sibi_dst.utils import Logger, FileAgeChecker, ParquetSaver
+from .log_utils import Logger
+from .date_utils import FileAgeChecker
+from .parquet_saver import ParquetSaver
 
 
 class DataWrapper:

sibi_dst/{utils → v1/utils}/date_utils.py

@@ -5,7 +5,7 @@ import fsspec
 import numpy as np
 import pandas as pd
 
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class DateUtils:

sibi_dst/{utils → v1/utils}/file_utils.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 import fsspec
 
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class FileUtils:

sibi_dst/{utils → v1/utils}/filepath_generator.py

@@ -3,7 +3,7 @@ import re
 
 import fsspec
 
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class FilePathGenerator:

sibi_dst/{utils → v1/utils}/parquet_saver.py

@@ -13,7 +13,7 @@ from fsspec import filesystem
 
 # Suppress the specific UserWarning message
 warnings.filterwarnings("ignore")
-from sibi_dst.utils import Logger
+from .log_utils import Logger
 
 
 class ParquetSaver:

sibi_dst/v1/utils/storage_config.py

@@ -0,0 +1,28 @@
+from .storage_manager import StorageManager
+from .credentials import ConfigManager
+
+class StorageConfig:
+    def __init__(self, config:ConfigManager, depots:dict):
+        self.conf = config
+        self.depots = depots
+        self._initialize_storage()
+        self.storage_manager = StorageManager(self.base_storage, self.filesystem_type, self.filesystem_options)
+        self.depot_paths, self.depot_names = self.storage_manager.rebuild_depot_paths(depots)
+
+    def _initialize_storage(self):
+        self.filesystem_type = self.conf.get('fs_type','file')
+        self.base_storage = self.conf.get('fs_path', "local_storage/")
+        if self.filesystem_type == "file":
+            self.filesystem_options ={}
+        else:
+            self.filesystem_options = {
+                "key": self.conf.get('fs_key',''),
+                "secret": self.conf.get('fs_secret'),
+                "token": self.conf.get('fs_token'),
+                "skip_instance_cache":True,
+                "use_listings_cache": False,
+                "client_kwargs": {
+                    "endpoint_url": self.conf.get('fs_endpoint')
+                }
+            }
+            self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}

sibi_dst/v2/df_helper/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from ._df_helper import DfHelper
+
+__all__ = [
+    'DfHelper',
+]

sibi_dst/v2/df_helper/_df_helper.py

@@ -0,0 +1,214 @@
+import warnings
+from typing import Any, Dict, Type, TypeVar, Union
+
+import dask.dataframe as dd
+import fsspec
+import pandas as pd
+from pydantic import BaseModel
+
+from sibi_dst.v2.utils import Logger
+from sibi_dst.v2.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
+from sibi_dst.v2.df_helper.backends.sqlalchemy import SqlAlchemyConnectionConfig, SqlAlchemyLoadFromDb
+from sibi_dst.v2.df_helper.backends.sqlmodel import SQLModelConnectionConfig, SQLModelLoadFromDb
+
+# Define a generic type variable for BaseModel subclasses
+T = TypeVar("T", bound=BaseModel)
+
+# Suppress warnings about protected member access
+warnings.filterwarnings(
+    "ignore",
+    message="Access to a protected member _meta",
+    category=UserWarning,
+)
+
+
+class DfHelper:
+    df: Union[dd.DataFrame, pd.DataFrame] = None
+    default_config = {
+        'parquet_storage_path': None,
+        'dt_field': None,
+        'as_pandas': False,
+        'filesystem': 'file',
+        'filesystem_options': {},
+        'fs': fsspec.filesystem('file')
+    }
+
+    def __init__(self, **kwargs: Any) -> None:
+        # Merge default configuration with any provided kwargs
+        config = {**self.default_config.copy(), **kwargs}
+        self.backend = config.setdefault('backend', 'sqlalchemy')
+        self.debug = config.setdefault('debug', False)
+        self.as_pandas = config.setdefault('as_pandas', False)
+        self.logger = config.setdefault(
+            'logger',
+            Logger.default_logger(logger_name=self.__class__.__name__, debug=self.debug)
+        )
+        self.logger.debug("Logger initialized in DEBUG mode.")
+
+        # Propagate logger and debug settings to all components
+        config.setdefault('logger', self.logger)
+        config.setdefault('debug', self.debug)
+
+        self._initialize_backend_config(**config)
+
+    def __str__(self) -> str:
+        return self.__class__.__name__
+
+    def _extract_config_vars(self, model: Type[T], kwargs: Dict[str, Any]) -> T:
+        """
+        Extracts and initializes a Pydantic model using only the keys that the model accepts.
+        The recognized keys are removed from kwargs.
+        """
+        recognized_keys = set(model.__annotations__.keys())
+        self.logger.debug(f"Recognized keys for {model.__name__}: {recognized_keys}")
+        model_kwargs = {k: kwargs.pop(k) for k in list(kwargs.keys()) if k in recognized_keys}
+        self.logger.debug(f"Initializing {model.__name__} with: {model_kwargs}")
+        return model(**model_kwargs)
+
+    def _initialize_backend_config(self, **kwargs: Any) -> None:
+        """
+        Initializes the backend configurations by extracting the settings required for queries,
+        parameters, and SQLAlchemy connections.
+        """
+        self.logger.debug("Initializing backend configuration.")
+        self._backend_query = self._extract_config_vars(QueryConfig, kwargs)
+        self._backend_params = self._extract_config_vars(ParamsConfig, kwargs)
+        if self.backend == "sqlalchemy":
+            self.backend_connection_config = self._extract_config_vars(SqlAlchemyConnectionConfig, kwargs)
+        elif self.backend == "sqlmodel":
+            self.backend_connection_config = self._extract_config_vars(SQLModelConnectionConfig, kwargs)
+        else:
+            raise ValueError(f"Unsupported backend: {self.backend}")
+
+    def load(self, **options: Any) -> Union[dd.DataFrame, pd.DataFrame]:
+        """
+        Loads the data using the underlying SQLAlchemy loader. Returns a pandas DataFrame
+        if 'as_pandas' is True; otherwise returns a dask DataFrame.
+        """
+        df = self._load(**options)
+        return df.compute() if self.as_pandas else df
+
+    def _load(self, **options: Any) -> Union[dd.DataFrame, pd.DataFrame]:
+        self._backend_params.parse_params(options)
+        if self.backend == "sqlalchemy":
+            return self._load_from_sqlalchemy(**options)
+        elif self.backend == "sqlmodel":
+            return self._load_from_sqlmodel(**options)
+        else:
+            raise ValueError(f"Unsupported backend: {self.backend}")
+
+    def _load_from_sqlalchemy(self, **options: Any) -> Union[dd.DataFrame, pd.DataFrame]:
+        """
+        Loads data from a SQLAlchemy source. On failure, logs the error and returns an empty
+        DataFrame wrapped as a dask DataFrame.
+        """
+        try:
+            db_loader = SqlAlchemyLoadFromDb(
+                self.backend_connection_config,
+                self._backend_query,
+                self._backend_params,
+                self.debug,
+                self.logger,
+                **options
+            )
+            self.df = db_loader.build_and_load()
+            self._process_loaded_data()
+            self._post_process_df()
+            self.logger.debug("Data successfully loaded from SQLAlchemy database.")
+        except Exception as e:
+            self.logger.error(f"Failed to load data from SQLAlchemy database: {e}. Options: {options}")
+            # Optionally re-raise the exception if in debug mode
+            if self.debug:
+                raise
+            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
+        return self.df
+
+    def _load_from_sqlmodel(self, **options: Any) -> Union[dd.DataFrame, pd.DataFrame]:
+        try:
+            db_loader = SQLModelLoadFromDb(
+                self.backend_connection_config,
+                self._backend_query,
+                self._backend_params,
+                self.debug,
+                self.logger,
+                **options
+            )
+            self.df = db_loader.build_and_load()
+            self._process_loaded_data()
+            self._post_process_df()
+            self.logger.debug("Data successfully loaded from SQLModel database.")
+        except Exception as e:
+            self.logger.error(f"Failed to load data from SQLModel database: {e}. Options: {options}")
+            if self.debug:
+                raise
+            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
+        return self.df
+
+    def _post_process_df(self) -> None:
+        """
+        Post-processes the DataFrame by filtering columns, renaming them, setting the index,
+        and converting the index to datetime if requested.
+        """
+        df_params = self._backend_params.df_params
+        fieldnames = df_params.get("fieldnames")
+        index_col = df_params.get("index_col")
+        datetime_index = df_params.get("datetime_index", False)
+        column_names = df_params.get("column_names")
+
+        # Filter columns based on fieldnames
+        if fieldnames:
+            valid_fieldnames = [col for col in fieldnames if col in self.df.columns]
+            self.df = self.df[valid_fieldnames]
+
+        # Rename columns if column_names are provided
+        if column_names is not None:
+            if not fieldnames or len(fieldnames) != len(column_names):
+                raise ValueError(
+                    f"Length mismatch: fieldnames ({len(fieldnames) if fieldnames else 0}) and "
+                    f"column_names ({len(column_names)}) must match."
+                )
+            rename_mapping = dict(zip(fieldnames, column_names))
+            self.df = self.df.map_partitions(self._rename_columns, mapping=rename_mapping)
+
+        # Set the index column if specified
+        if index_col is not None:
+            if index_col in self.df.columns:
+                self.df = self.df.set_index(index_col)
+            else:
+                raise ValueError(f"Index column '{index_col}' not found in DataFrame.")
+
+        # Convert the index to datetime if required
+        if datetime_index and self.df.index.dtype != 'datetime64[ns]':
+            self.df = self.df.map_partitions(self._convert_index_to_datetime)
+
+        self.logger.debug("Post-processing of DataFrame completed.")
+
+    def _process_loaded_data(self) -> None:
+        """
+        Applies renaming logic based on the field map configuration.
+        Logs a warning for any missing columns, and only renames existing columns.
+        """
+        self.logger.debug(f"Processing loaded data; DataFrame type: {type(self.df)}")
+        if self.df.map_partitions(len).compute().sum() > 0:
+            field_map = self._backend_params.field_map or {}
+            if isinstance(field_map, dict):
+                rename_mapping = {k: v for k, v in field_map.items() if k in self.df.columns}
+                missing_columns = [k for k in field_map if k not in self.df.columns]
+                if missing_columns:
+                    self.logger.warning(
+                        f"The following columns in field_map are not in the DataFrame: {missing_columns}"
+                    )
+                if rename_mapping:
+                    self.df = self.df.map_partitions(self._rename_columns, mapping=rename_mapping)
+        self.logger.debug("Processing of loaded data completed.")
+
+    @staticmethod
+    def _rename_columns(df: pd.DataFrame, mapping: Dict[str, str]) -> pd.DataFrame:
+        """Helper function to rename columns in a DataFrame."""
+        return df.rename(columns=mapping)
+
+    @staticmethod
+    def _convert_index_to_datetime(df: pd.DataFrame) -> pd.DataFrame:
+        """Helper function to convert the DataFrame index to datetime."""
+        df.index = pd.to_datetime(df.index, errors='coerce')
+        return df