sibi-dst 0.3.49__py3-none-any.whl → 0.3.52__py3-none-any.whl

sibi_dst/utils/__init__.py

@@ -15,6 +15,7 @@ from .credentials import *
 from .data_wrapper import DataWrapper
 from .storage_config import StorageConfig
 from .data_from_http_source import DataFromHttpSource
+from .webdav_client import WebDAVClient
 
 __all__ = [
     "Logger",
@@ -34,5 +35,6 @@ __all__ = [
     "ClickHouseWriter",
     "AirflowDAGManager",
     "StorageConfig",
-    "DataFromHttpSource"
+    "DataFromHttpSource",
+    "WebDAVClient"
 ]

sibi_dst/utils/clickhouse_writer.py

@@ -3,7 +3,7 @@ from concurrent.futures import ThreadPoolExecutor
 import clickhouse_connect
 import pandas as pd
 from clickhouse_driver import Client
-from dask.dataframe import dd
+import dask.dataframe as dd
 
 from .log_utils import Logger
 

sibi_dst/utils/date_utils.py

@@ -51,6 +51,9 @@ class DateUtils:
                     continue
         raise ValueError(f"Unsupported date format: {value}")
 
+    # Public alias to access _ensure_date from other classes
+    ensure_date = _ensure_date
+
     @classmethod
     def calc_week_range(cls, reference_date: Union[str, datetime.date, datetime.datetime, pd.Timestamp]) -> Tuple[
         datetime.date, datetime.date]:
@@ -367,6 +370,8 @@ class BusinessDays:
         def calculate_business_days(row, holidays, weekmask):
             begin_date = pd.to_datetime(row[begin_date_col])
             end_date = pd.to_datetime(row[end_date_col])
+            if pd.isna(begin_date) or pd.isna(end_date):
+                return np.nan
             busdaycal = np.busdaycalendar(holidays=holidays, weekmask=weekmask)
             return np.busday_count(
                 begin_date.strftime("%Y-%m-%d"),

sibi_dst/utils/storage_config.py

@@ -2,19 +2,23 @@ from .storage_manager import StorageManager
 from .credentials import ConfigManager
 
 class StorageConfig:
-    def __init__(self, config:ConfigManager, depots:dict):
+    def __init__(self, config:ConfigManager, depots:dict=None):
         self.conf = config
         self.depots = depots
         self._initialize_storage()
         self.storage_manager = StorageManager(self.base_storage, self.filesystem_type, self.filesystem_options)
-        self.depot_paths, self.depot_names = self.storage_manager.rebuild_depot_paths(depots)
+        if self.depots is not None:
+            self.depot_paths, self.depot_names = self.storage_manager.rebuild_depot_paths(depots)
+        else:
+            self.depot_paths = None
+            self.depot_names = None
 
     def _initialize_storage(self):
         self.filesystem_type = self.conf.get('fs_type','file')
         self.base_storage = self.conf.get('fs_path', "local_storage/")
         if self.filesystem_type == "file":
             self.filesystem_options ={}
-        else:
+        elif self.filesystem_type == "s3":
             self.filesystem_options = {
                 "key": self.conf.get('fs_key',''),
                 "secret": self.conf.get('fs_secret'),
@@ -25,4 +29,21 @@ class StorageConfig:
                     "endpoint_url": self.conf.get('fs_endpoint')
                 }
             }
-            self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}
+        elif self.filesystem_type == "webdav":
+            verify_ssl = self.conf.get('fs_verify_ssl', True)
+            # Convert string 'false' to boolean False
+            if isinstance(verify_ssl, str) and verify_ssl.lower() == 'false':
+                verify_ssl = False
+            self.filesystem_options = {
+                "base_url": self.conf.get('fs_endpoint', ''),
+                "username": self.conf.get('fs_key', ''),
+                "password": self.conf.get('fs_secret', ''),
+                "token": self.conf.get('fs_token', ''),
+                "verify": verify_ssl
+            }
+        else:
+            # unsupported filesystem type
+            # defaulting to local filesystem
+            self.filesystem_type = 'file'
+            self.filesystem_options = {}
+        self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}

sibi_dst/utils/storage_manager.py

@@ -1,7 +1,7 @@
 from types import SimpleNamespace
 
 import fsspec
-
+from .webdav_client import WebDAVClient
 
 class StorageManager:
     def __init__(self, storage_path, fs_type="file", fs_options=None, debug=False):
@@ -16,11 +16,37 @@ class StorageManager:
         self.storage_path = storage_path.rstrip("/")
         self.fs_type = fs_type
         self.fs_options = fs_options or {}
-
-        self.fs = fsspec.filesystem(fs_type, **self.fs_options)
+        if fs_type == "webdav":
+            self._initialize_webdav()
+        else:
+            self.fs = fsspec.filesystem(fs_type, **self.fs_options)
         self.depot_paths = {}
         self.depot_name = None
 
+    def _initialize_webdav(self):
+        """
+        Initialize WebDAV filesystem using the WebDAVClient.
+        """
+        base_url = self.fs_options.get("base_url", "")
+        username = self.fs_options.get("username", "")
+        password = self.fs_options.get("password", "")
+        token = self.fs_options.get("token", "")
+        # Convert string 'false' to boolean False
+        verify = self.fs_options.get("verify", True)
+        if isinstance(verify, str) and verify.lower() == 'false':
+            verify = False
+
+        # Create WebDAV client
+        self.webdav_client = WebDAVClient(
+            base_url=base_url,
+            username=username,
+            password=password,
+            token=token,
+            verify=verify
+        )
+
+        # Use the fsspec-compatible filesystem
+        self.fs = self.webdav_client.get_fs()
     @staticmethod
     def join_paths(*parts):
         """
@@ -101,4 +127,38 @@ class StorageManager:
             print("Rebuild complete.")
 
     def get_fs_instance(self):
-        return fsspec.filesystem(self.fs_type, **self.fs_options)
+        """
+        Returns the filesystem instance.
+        """
+        if self.fs_type == "webdav":
+            return self.fs
+        else:
+            return fsspec.filesystem(self.fs_type, **self.fs_options)
+
+    def upload_file(self, local_path, remote_path):
+        """
+        Upload a file to the storage.
+
+        :param local_path: Local file path
+        :param remote_path: Remote file path
+        """
+        if self.fs_type == "webdav":
+            # Use the WebDAV client's upload method for WebDAV
+            self.webdav_client.upload_file(local_path, remote_path)
+        else:
+            # Use fsspec's put method for other filesystems
+            self.fs.put(local_path, remote_path)
+
+    def download_file(self, remote_path, local_path):
+        """
+        Download a file from the storage.
+
+        :param remote_path: Remote file path
+        :param local_path: Local file path
+        """
+        if self.fs_type == "webdav":
+            # Use the WebDAV client's download method for WebDAV
+            self.webdav_client.download_file(remote_path, local_path)
+        else:
+            # Use fsspec's get method for other filesystems
+            self.fs.get(remote_path, local_path)

sibi_dst/utils/webdav_client.py

@@ -0,0 +1,220 @@
+"""
+WebDAV client implementation for storage manager.
+"""
+from webdav4.client import Client, ResourceNotFound
+from webdav4.fsspec import WebdavFileSystem
+
+
+class WebDAVClient:
+    """
+    WebDAV client wrapper for interacting with WebDAV servers like NextCloud.
+    """
+
+    def __init__(self, base_url, username=None, password=None, token=None, verify=True):
+        """
+        Initialize the WebDAV client.
+
+        :param base_url: Base URL of the WebDAV server (e.g., "https://nextcloud.example.com/remote.php/dav/files/username/")
+        :param username: Username for authentication
+        :param password: Password for authentication
+        :param token: Authentication token (alternative to username/password)
+        :param verify: Whether to verify SSL certificates
+        """
+        self.base_url = base_url
+        self.username = username
+        self.password = password
+        self.token = token
+        # Set up verification
+        if isinstance(verify, str) and verify.lower() == 'false':
+            self.verify = False
+        else:
+            self.verify = verify
+
+        # Set up authentication
+        if token:
+            self.auth = token
+        elif username and password:
+            self.auth = (username, password)
+        else:
+            self.auth = None
+
+        # Initialize the client
+        self.client = Client(
+            base_url,
+            auth=self.auth,
+            verify=self.verify
+        )
+
+        # Initialize the fsspec filesystem
+        self.fs = WebdavFileSystem(
+            base_url,
+            auth=self.auth,
+            verify=self.verify
+        )
+
+    def exists(self, path):
+        """
+        Check if a path exists on the WebDAV server.
+
+        :param path: Path to check
+        :return: True if exists, False otherwise
+        """
+        try:
+            return self.client.exists(path)
+        except Exception:
+            return False
+
+    def list_directory(self, path="", detail=False):
+        """
+        List contents of a directory.
+
+        :param path: Directory path to list
+        :param detail: Whether to return detailed information
+        :return: List of files/directories or detailed information
+        """
+        try:
+            return self.client.ls(path, detail=detail)
+        except ResourceNotFound:
+            # Return empty list if directory doesn't exist
+            return [] if not detail else {}
+
+    def ensure_directory_exists(self, path):
+        """
+        Ensure a directory exists, creating parent directories if needed.
+
+        :param path: Directory path to ensure exists
+        """
+        if not path or path == "/" or path == ".":
+            return
+
+        # Check if the directory already exists
+        if self.exists(path):
+            return
+
+        # Split the path into components
+        parts = path.strip('/').split('/')
+        current_path = ""
+
+        # Create each directory in the path if it doesn't exist
+        for part in parts:
+            if not part:
+                continue
+
+            current_path = f"{current_path}/{part}" if current_path else part
+
+            if not self.exists(current_path):
+                try:
+                    self.client.mkdir(current_path)
+                except ResourceNotFound:
+                    # If parent directory doesn't exist, create it first
+                    parent_path = '/'.join(current_path.split('/')[:-1])
+                    self.ensure_directory_exists(parent_path)
+                    # Then try to create the directory again
+                    self.client.mkdir(current_path)
+                except Exception as e:
+                    # If directory already exists or other error, log and continue
+                    if self.exists(current_path):
+                        pass  # Directory exists, which is fine
+                    else:
+                        raise e  # Re-raise if it's a different error
+
+    def create_directory(self, path):
+        """
+        Create a directory on the WebDAV server.
+
+        :param path: Directory path to create
+        """
+        self.ensure_directory_exists(path)
+
+    def upload_file(self, local_path, remote_path):
+        """
+        Upload a file to the WebDAV server.
+
+        :param local_path: Local file path
+        :param remote_path: Remote file path
+        """
+        # Ensure parent directory exists
+        parent_dir = '/'.join(remote_path.split('/')[:-1])
+        if parent_dir:
+            self.ensure_directory_exists(parent_dir)
+
+        # Upload the file
+        self.client.upload_file(local_path, remote_path)
+
+    def download_file(self, remote_path, local_path):
+        """
+        Download a file from the WebDAV server.
+
+        :param remote_path: Remote file path
+        :param local_path: Local file path
+        """
+        self.client.download_file(remote_path, local_path)
+
+    def delete(self, path, recursive=False):
+        """
+        Delete a file or directory on the WebDAV server.
+
+        :param path: Path to delete
+        :param recursive: Whether to delete recursively (for directories)
+        """
+        try:
+            self.client.remove(path, recursive=recursive)
+        except ResourceNotFound:
+            # If the resource doesn't exist, that's fine - it's already gone
+            pass
+        except Exception as e:
+            # Re-raise other exceptions
+            raise e
+
+    def get_fs(self):
+        """
+        Get the fsspec filesystem instance.
+
+        :return: WebdavFileSystem instance
+        """
+        return self.fs
+
+    # Add fsspec-compatible method aliases
+    def mkdir(self, path, create_parents=True):
+        """
+        Create a directory (fsspec-compatible method).
+
+        :param path: Directory path to create
+        :param create_parents: Whether to create parent directories
+        """
+        if create_parents:
+            self.ensure_directory_exists(path)
+        else:
+            try:
+                self.client.mkdir(path)
+            except ResourceNotFound:
+                # If parent directory doesn't exist and create_parents is False, raise error
+                raise
+
+    def ls(self, path="", detail=False):
+        """
+        List contents of a directory (fsspec-compatible method).
+
+        :param path: Directory path to list
+        :param detail: Whether to return detailed information
+        :return: List of files/directories or detailed information
+        """
+        return self.list_directory(path, detail=detail)
+
+    def rm(self, path, recursive=False):
+        """
+        Delete a file or directory (fsspec-compatible method).
+
+        :param path: Path to delete
+        :param recursive: Whether to delete recursively (for directories)
+        """
+        self.delete(path, recursive=recursive)
+
+    def makedirs(self, path, exist_ok=True):
+        """
+        Create a directory and all parent directories (fsspec-compatible method).
+
+        :param path: Directory path to create
+        :param exist_ok: Whether it's okay if the directory already exists
+        """
+        self.ensure_directory_exists(path)

{sibi_dst-0.3.49.dist-info → sibi_dst-0.3.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.49
+Version: 0.3.52
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -10,42 +10,46 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: complete
+Provides-Extra: df-helper
+Provides-Extra: geopy-helper
+Provides-Extra: osmnx-helper
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
-Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
-Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "complete"
-Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "complete"
-Requires-Dist: folium (>=0.19.4,<0.20.0)
-Requires-Dist: geopandas (>=1.0.1,<2.0.0)
-Requires-Dist: geopy (>=2.4.1,<3.0.0)
+Requires-Dist: dask[complete] (>=2025.3.0,<2026.0.0)
+Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "osmnx-helper" or extra == "complete"
+Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "osmnx-helper" or extra == "complete"
+Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geopy-helper" or extra == "complete"
 Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
-Requires-Dist: mysqlclient (>=2.2.6,<3.0.0)
+Requires-Dist: mysqlclient (>=2.2.6,<3.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: nltk (>=3.9.1,<4.0.0)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
-Requires-Dist: osmnx (>=2.0.1,<3.0.0)
+Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "osmnx-helper" or extra == "complete"
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: psutil (>=6.1.0,<7.0.0)
-Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
-Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: pymysql (>=1.1.1,<2.0.0) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
-Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
-Requires-Dist: sqlmodel (>=0.0.22,<0.0.23)
+Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: sqlmodel (>=0.0.22,<0.0.23) ; extra == "df-helper" or extra == "complete"
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
 Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
 Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
+Requires-Dist: webdav4[fsspec] (>=0.10.0,<0.11.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst

{sibi_dst-0.3.49.dist-info → sibi_dst-0.3.52.dist-info}/RECORD

@@ -38,22 +38,23 @@ sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqT
 sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEFg,20577
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=ecBkVoWRqzljPRpmufl0FFWqTJNsVa56e4ahgKwRojM,1017
+sibi_dst/utils/__init__.py,sha256=_4kuTzjCfbRF9927ywXqi-JKoHNec8wf05LYh4DMbPI,1077
 sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
-sibi_dst/utils/clickhouse_writer.py,sha256=JE_WMsnLTlxoABIDLuvrDUVHkTidlev8gSC4krDag3M,9879
+sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcWxIQJ60,9877
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
 sibi_dst/utils/data_wrapper.py,sha256=pIIQxeHknUeQd0YbISkAhL-xYBK4OdijoATBY-oBznw,12114
-sibi_dst/utils/date_utils.py,sha256=7cqgC6WEcfkh6BKTgq-kyig4H9rf_0VzpySPYElSo_0,18359
+sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
 sibi_dst/utils/log_utils.py,sha256=eSAbi_jmMpJ8RpycakzT4S4zNkqVZDj3FY8WwnxpdXc,4623
 sibi_dst/utils/parquet_saver.py,sha256=Tucxv9jRX66VuLQZn0dPQBN7JOttBou6SF8FxqufeGE,8169
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
-sibi_dst/utils/storage_config.py,sha256=ugM70OHo63dN7LPukl0FZTWwXKBuoCILFh3RdNEeMgY,1239
-sibi_dst/utils/storage_manager.py,sha256=H_itUFJv9nP0BfXYYQDsw4RzB0YWfgVOAHNWAiMpZ_w,4443
+sibi_dst/utils/storage_config.py,sha256=Cg8EOGLZ_5v9sunaQHZLYHdp5FDkgPrCVVNHF-ys5sQ,2181
+sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
+sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
 sibi_dst/v2/df_helper/_df_helper.py,sha256=9pED3bjQ2Z81zqzJrZ9e7SguoO4-hBmNTJK4WOKrr4M,9297
@@ -74,6 +75,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.49.dist-info/METADATA,sha256=JJQStRGmtUBhkPW2SeEPFfbJ4vBVYsEjIr_AHZVpTnw,6614
-sibi_dst-0.3.49.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.49.dist-info/RECORD,,
+sibi_dst-0.3.52.dist-info/METADATA,sha256=VIiTzkdO-FsYBkie1cfuwFKEwFFSE5TL5Ory-VzPa0g,7221
+sibi_dst-0.3.52.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.52.dist-info/RECORD,,