FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/cli/utils.py

@@ -1,15 +1,20 @@
 import ast
+import importlib
 import json
-from pathlib import Path
+import posixpath
 import re
-import importlib
+import sys
+from pathlib import Path
 from typing import Callable
+
 from loguru import logger
-import sys
-import posixpath
 
 from flowerpower.pipeline import PipelineManager
 
+from ..utils.logging import setup_logging
+
+setup_logging()
+
 
 # Parse additional parameters
 def parse_param_dict(param_str: str | None) -> dict:
@@ -99,12 +104,13 @@ def parse_dict_or_list_param(
             logger.warning(f"Could not parse {param_type} parameter: {value}")
             return None
 
+
 def load_hook(
-        pipeline_name: str,
-        function_path: str,
-        base_dir = None,
-        storage_options: str | None = None,
-        ) -> Callable:
+    pipeline_name: str,
+    function_path: str,
+    base_dir=None,
+    storage_options: str | None = None,
+) -> Callable:
     """
     Load a hook function from a specified path.
     This function dynamically imports the module and retrieves the function
@@ -118,21 +124,26 @@ def load_hook(
     Returns:
         Callable: The loaded hook function
     """
-    with PipelineManager(
-        storage_options=storage_options, base_dir=base_dir
-    ) as pm:
-        path_segments = function_path.rsplit('.', 2)
+    with PipelineManager(storage_options=storage_options, base_dir=base_dir) as pm:
+        path_segments = function_path.rsplit(".", 2)
         if len(path_segments) == 2:
             # If the function path is in the format 'module_name.function_name'
             module_name, function_name = path_segments
-            module_path = ''
+            module_path = ""
         elif len(path_segments) == 3:
             # If the function path is in the format 'package.[subpackage.]module_name.function_name'
             module_path, module_name, function_name = path_segments
 
-
-        logger.debug(posixpath.join(pm._fs.path,"hooks", pipeline_name , module_path.replace('.', '/')))
-        sys.path.append(posixpath.join(pm._fs.path,"hooks", pipeline_name ,module_path.replace('.', '/')))
+        logger.debug(
+            posixpath.join(
+                pm._fs.path, "hooks", pipeline_name, module_path.replace(".", "/")
+            )
+        )
+        sys.path.append(
+            posixpath.join(
+                pm._fs.path, "hooks", pipeline_name, module_path.replace(".", "/")
+            )
+        )
         hook_module = importlib.import_module(module_name)
         hook_function = getattr(hook_module, function_name)
-        return hook_function
+        return hook_function

flowerpower/flowerpower.py

@@ -6,15 +6,19 @@ from pathlib import Path
 import rich
 from fsspec.spec import AbstractFileSystem
 
-from .cfg import Config
+from .cfg import ProjectConfig
 from .fs import get_filesystem
-
+from . import settings
 
 def init(
     name: str | None = None,
     base_dir: str | None = None,
     storage_options: dict = {},
     fs: AbstractFileSystem | None = None,
+    job_queue_type: str = settings.DEFAULT_JOB_QUEUE,
+    cfg_dir: str = settings.CONFIG_DIR,
+    pipelines_dir: str = settings.PIPELINES_DIR,
+    hooks_dir: str = settings.HOOKS_DIR,
 ):
     if name is None:
         name = str(Path.cwd().name)
@@ -25,11 +29,11 @@ def init(
 
     fs = get_filesystem(posixpath.join(base_dir, name), **storage_options)
 
-    fs.makedirs("conf/pipelines", exist_ok=True)
-    fs.makedirs("pipelines", exist_ok=True)
-    fs.makedirs("hooks", exist_ok=True)
+    fs.makedirs(f"{cfg_dir}/pipelines", exist_ok=True)
+    fs.makedirs(pipelines_dir, exist_ok=True)
+    fs.makedirs(hooks_dir, exist_ok=True)
 
-    cfg = Config.load(base_dir=posixpath.join(base_dir, name), name=name)
+    cfg = ProjectConfig.load(base_dir=posixpath.join(base_dir, name), name=name, job_queue_type=job_queue_type)
 
     with open(posixpath.join(base_dir, name, "README.md"), "w") as f:
         f.write(
@@ -55,9 +59,9 @@ def init(
             [dim]More options:[/dim]
                 [blue underline]https://docs.astral.sh/uv/getting-started/installation/[/blue underline]
 
-    🚀  Initialize your project:
+    🚀  Initialize uv in your flowerpower project:
             [dim]Run the following in your project directory:[/dim]
-            [bold white]uv init --app --no-readme --vcs git[/bold white]
+            [bold white]uv init --bare --no-readme[/bold white]
     """
     )
 

flowerpower/fs/__init__.py

@@ -1,10 +1,28 @@
 import importlib
 
 has_orjson = importlib.util.find_spec("orjson") is not None
+has_polars = importlib.util.find_spec("polars") is not None
 
-if has_orjson:
+if has_orjson and has_polars:
     from .ext import AbstractFileSystem
 else:
     from fsspec import AbstractFileSystem
 
-from .base import get_filesystem
+from .base import get_filesystem  # noqa: E402
+from .storage_options import AwsStorageOptions  # noqa: E402
+from .storage_options import AzureStorageOptions  # noqa: E402
+from .storage_options import (BaseStorageOptions, GcsStorageOptions,
+                              GitHubStorageOptions, GitLabStorageOptions,
+                              StorageOptions)
+
+__all__ = [
+    "get_filesystem",
+    "AbstractFileSystem",
+    "StorageOptions",
+    "AwsStorageOptions",
+    "AzureStorageOptions",
+    "GcsStorageOptions",
+    "GitHubStorageOptions",
+    "GitLabStorageOptions",
+    "BaseStorageOptions",
+]

flowerpower/fs/base.py

@@ -4,6 +4,7 @@ import os
 import posixpath
 import urllib
 from pathlib import Path
+from typing import Any
 
 import fsspec
 import requests
@@ -15,16 +16,65 @@ from fsspec.implementations.memory import MemoryFile
 from fsspec.utils import infer_storage_options
 from loguru import logger
 
-from .ext import AbstractFileSystem
+from ..utils.logging import setup_logging
+from . import has_orjson, has_polars
+
+if has_orjson and has_polars:
+    from .ext import AbstractFileSystem
+else:
+    from fsspec import AbstractFileSystem
+
 from .storage_options import BaseStorageOptions
 from .storage_options import from_dict as storage_options_from_dict
 
+setup_logging()
+
 
 class FileNameCacheMapper(AbstractCacheMapper):
-    def __init__(self, directory):
+    """Maps remote file paths to local cache paths while preserving directory structure.
+
+    This cache mapper maintains the original file path structure in the cache directory,
+    creating necessary subdirectories as needed.
+
+    Attributes:
+        directory (str): Base directory for cached files
+
+    Example:
+        >>> # Create cache mapper for S3 files
+        >>> mapper = FileNameCacheMapper("/tmp/cache")
+        >>>
+        >>> # Map remote path to cache path
+        >>> cache_path = mapper("bucket/data/file.csv")
+        >>> print(cache_path)  # Preserves structure
+        'bucket/data/file.csv'
+    """
+
+    def __init__(self, directory: str):
+        """Initialize cache mapper with base directory.
+
+        Args:
+            directory: Base directory where cached files will be stored
+        """
         self.directory = directory
 
     def __call__(self, path: str) -> str:
+        """Map remote file path to cache file path.
+
+        Creates necessary subdirectories in the cache directory to maintain
+        the original path structure.
+
+        Args:
+            path: Original file path from remote filesystem
+
+        Returns:
+            str: Cache file path that preserves original structure
+
+        Example:
+            >>> mapper = FileNameCacheMapper("/tmp/cache")
+            >>> # Maps maintain directory structure
+            >>> print(mapper("data/nested/file.txt"))
+            'data/nested/file.txt'
+        """
         os.makedirs(
             posixpath.dirname(posixpath.join(self.directory, path)), exist_ok=True
         )
@@ -32,15 +82,81 @@ class FileNameCacheMapper(AbstractCacheMapper):
 
 
 class MonitoredSimpleCacheFileSystem(SimpleCacheFileSystem):
-    def __init__(self, **kwargs):
-        # kwargs["cache_storage"] = posixpath.join(
-        #    kwargs.get("cache_storage"), kwargs.get("fs").protocol[0]
-        # )
+    """Enhanced caching filesystem with monitoring and improved path handling.
+
+    This filesystem extends SimpleCacheFileSystem to provide:
+    - Verbose logging of cache operations
+    - Improved path mapping for cache files
+    - Enhanced synchronization capabilities
+    - Better handling of parallel operations
+
+    Attributes:
+        _verbose (bool): Whether to print verbose cache operations
+        _mapper (FileNameCacheMapper): Maps remote paths to cache paths
+        storage (list[str]): List of cache storage locations
+        fs (AbstractFileSystem): Underlying filesystem being cached
+
+    Example:
+        >>> from fsspec import filesystem
+        >>> # Create monitored cache for S3
+        >>> s3 = filesystem("s3", key="ACCESS_KEY", secret="SECRET_KEY")
+        >>> cached_fs = MonitoredSimpleCacheFileSystem(
+        ...     fs=s3,
+        ...     cache_storage="/tmp/s3_cache",
+        ...     verbose=True
+        ... )
+        >>>
+        >>> # Read file (downloads and caches)
+        >>> with cached_fs.open("bucket/data.csv") as f:
+        ...     data = f.read()
+        Downloading s3://bucket/data.csv
+        >>>
+        >>> # Second read uses cache
+        >>> with cached_fs.open("bucket/data.csv") as f:
+        ...     data = f.read()  # No download message
+    """
+
+    def __init__(self, **kwargs: Any):
+        """Initialize monitored cache filesystem.
+
+        Args:
+            **kwargs: Configuration options including:
+                fs (AbstractFileSystem): Filesystem to cache
+                cache_storage (str): Cache directory path
+                verbose (bool): Enable verbose logging
+                And any other SimpleCacheFileSystem options
+
+        Example:
+            >>> # Cache with custom settings
+            >>> cached_fs = MonitoredSimpleCacheFileSystem(
+            ...     fs=remote_fs,
+            ...     cache_storage="/tmp/cache",
+            ...     verbose=True,
+            ...     same_names=True  # Use original filenames
+            ... )
+        """
         self._verbose = kwargs.get("verbose", False)
         super().__init__(**kwargs)
         self._mapper = FileNameCacheMapper(kwargs.get("cache_storage"))
 
-    def _check_file(self, path):
+    def _check_file(self, path: str) -> str | None:
+        """Check if file exists in cache and download if needed.
+
+        Args:
+            path: Path to file in the remote filesystem
+
+        Returns:
+            str | None: Path to cached file if found/downloaded, None otherwise
+
+        Example:
+            >>> fs = MonitoredSimpleCacheFileSystem(
+            ...     fs=remote_fs,
+            ...     cache_storage="/tmp/cache"
+            ... )
+            >>> cached_path = fs._check_file("data.csv")
+            >>> print(cached_path)
+            '/tmp/cache/data.csv'
+        """
         self._check_cache()
         cache_path = self._mapper(path)
         for storage in self.storage:
@@ -50,17 +166,50 @@ class MonitoredSimpleCacheFileSystem(SimpleCacheFileSystem):
             if self._verbose:
                 logger.info(f"Downloading {self.protocol[0]}://{path}")
 
-    # def glob(self, path):
-    #    return [self._strip_protocol(path)]
+    def size(self, path: str) -> int:
+        """Get size of file in bytes.
+
+        Checks cache first, falls back to remote filesystem.
+
+        Args:
+            path: Path to file
 
-    def size(self, path):
+        Returns:
+            int: Size of file in bytes
+
+        Example:
+            >>> fs = MonitoredSimpleCacheFileSystem(
+            ...     fs=remote_fs,
+            ...     cache_storage="/tmp/cache"
+            ... )
+            >>> size = fs.size("large_file.dat")
+            >>> print(f"File size: {size} bytes")
+        """
         cached_file = self._check_file(self._strip_protocol(path))
         if cached_file is None:
             return self.fs.size(path)
         else:
             return posixpath.getsize(cached_file)
 
-    def sync(self, reload: bool = False):
+    def sync_cache(self, reload: bool = False) -> None:
+        """Synchronize cache with remote filesystem.
+
+        Downloads all files in remote path to cache if not present.
+
+        Args:
+            reload: Whether to force reload all files, ignoring existing cache
+
+        Example:
+            >>> fs = MonitoredSimpleCacheFileSystem(
+            ...     fs=remote_fs,
+            ...     cache_storage="/tmp/cache"
+            ... )
+            >>> # Initial sync
+            >>> fs.sync_cache()
+            >>>
+            >>> # Force reload all files
+            >>> fs.sync_cache(reload=True)
+        """
         if reload:
             self.clear_cache()
         content = self.glob("**/*")
@@ -154,6 +303,41 @@ class MonitoredSimpleCacheFileSystem(SimpleCacheFileSystem):
 
 
 class GitLabFileSystem(AbstractFileSystem):
+    """FSSpec-compatible filesystem interface for GitLab repositories.
+
+    Provides access to files in GitLab repositories through the GitLab API,
+    supporting read operations with authentication.
+
+    Attributes:
+        project_name (str): Name of the GitLab project
+        project_id (str): ID of the GitLab project
+        access_token (str): GitLab personal access token
+        branch (str): Git branch to read from
+        base_url (str): GitLab instance URL
+
+    Example:
+        >>> # Access public project
+        >>> fs = GitLabFileSystem(
+        ...     project_name="my-project",
+        ...     access_token="glpat-xxxx"
+        ... )
+        >>>
+        >>> # Read file contents
+        >>> with fs.open("path/to/file.txt") as f:
+        ...     content = f.read()
+        >>>
+        >>> # List directory
+        >>> files = fs.ls("path/to/dir")
+        >>>
+        >>> # Access enterprise GitLab
+        >>> fs = GitLabFileSystem(
+        ...     project_id="12345",
+        ...     access_token="glpat-xxxx",
+        ...     base_url="https://gitlab.company.com",
+        ...     branch="develop"
+        ... )
+    """
+
     def __init__(
         self,
         project_name: str | None = None,
@@ -163,6 +347,21 @@ class GitLabFileSystem(AbstractFileSystem):
         base_url: str = "https://gitlab.com",
         **kwargs,
     ):
+        """Initialize GitLab filesystem.
+
+        Args:
+            project_name: Name of the GitLab project. Required if project_id not provided.
+            project_id: ID of the GitLab project. Required if project_name not provided.
+            access_token: GitLab personal access token for authentication.
+                Required for private repositories.
+            branch: Git branch to read from. Defaults to "main".
+            base_url: GitLab instance URL. Defaults to "https://gitlab.com".
+            **kwargs: Additional arguments passed to AbstractFileSystem.
+
+        Raises:
+            ValueError: If neither project_name nor project_id is provided
+            requests.RequestException: If GitLab API request fails
+        """
         super().__init__(**kwargs)
         self.project_name = project_name
         self.project_id = project_id
@@ -173,11 +372,29 @@ class GitLabFileSystem(AbstractFileSystem):
         if not self.project_id:
             self.project_id = self._get_project_id()
 
-    def _validate_init(self):
+    def _validate_init(self) -> None:
+        """Validate initialization parameters.
+
+        Ensures that either project_id or project_name is provided.
+
+        Raises:
+            ValueError: If neither project_id nor project_name is provided
+        """
         if not self.project_id and not self.project_name:
             raise ValueError("Either 'project_id' or 'project_name' must be provided")
 
-    def _get_project_id(self):
+    def _get_project_id(self) -> str:
+        """Retrieve project ID from GitLab API using project name.
+
+        Makes an API request to search for projects and find the matching project ID.
+
+        Returns:
+            str: The GitLab project ID
+
+        Raises:
+            ValueError: If project not found
+            requests.RequestException: If API request fails
+        """
         url = f"{self.base_url}/api/v4/projects"
         headers = {"PRIVATE-TOKEN": self.access_token}
         params = {"search": self.project_name}
@@ -192,7 +409,29 @@ class GitLabFileSystem(AbstractFileSystem):
         else:
             response.raise_for_status()
 
-    def _open(self, path, mode="rb", **kwargs):
+    def _open(self, path: str, mode: str = "rb", **kwargs) -> MemoryFile:
+        """Open a file from GitLab repository.
+
+        Retrieves file content from GitLab API and returns it as a memory file.
+
+        Args:
+            path: Path to file within repository
+            mode: File open mode. Only "rb" (read binary) is supported.
+            **kwargs: Additional arguments (unused)
+
+        Returns:
+            MemoryFile: File-like object containing file content
+
+        Raises:
+            NotImplementedError: If mode is not "rb"
+            requests.RequestException: If API request fails
+
+        Example:
+            >>> fs = GitLabFileSystem(project_id="12345", access_token="glpat-xxxx")
+            >>> with fs.open("README.md") as f:
+            ...     content = f.read()
+            ...     print(content.decode())
+        """
         if mode != "rb":
             raise NotImplementedError("Only read mode is supported")
 
@@ -209,7 +448,34 @@ class GitLabFileSystem(AbstractFileSystem):
         else:
             response.raise_for_status()
 
-    def _ls(self, path, detail=False, **kwargs):
+    def _ls(self, path: str, detail: bool = False, **kwargs) -> list[str] | list[dict]:
+        """List contents of a directory in GitLab repository.
+
+        Args:
+            path: Directory path within repository
+            detail: Whether to return detailed information about each entry.
+                If True, returns list of dicts with file metadata.
+                If False, returns list of filenames.
+            **kwargs: Additional arguments (unused)
+
+        Returns:
+            list[str] | list[dict]: List of file/directory names or detailed info
+
+        Raises:
+            requests.RequestException: If API request fails
+
+        Example:
+            >>> fs = GitLabFileSystem(project_id="12345", access_token="glpat-xxxx")
+            >>> # List filenames
+            >>> files = fs.ls("docs")
+            >>> print(files)
+            ['README.md', 'API.md']
+            >>>
+            >>> # List with details
+            >>> details = fs.ls("docs", detail=True)
+            >>> for item in details:
+            ...     print(f"{item['name']}: {item['type']}")
+        """
         url = f"{self.base_url}/api/v4/projects/{self.project_id}/repository/tree?path={path}&ref={self.branch}"
         headers = {"PRIVATE-TOKEN": self.access_token}
         response = requests.get(url, headers=headers)
@@ -258,19 +524,77 @@ def get_filesystem(
     fs: AbstractFileSystem | None = None,
     **storage_options_kwargs,
 ) -> AbstractFileSystem:
-    """
-    Get a filesystem based on the given path.
+    """Get a filesystem instance based on path or configuration.
 
-    Args:
-        path: (str, optional) Path to the filesystem. Defaults to None.
-        storage_options: (AwsStorageOptions | GitHubStorageOptions | GitLabStorageOptions |
-            GcsStorageOptions | AzureStorageOptions | dict[str, str], optional) Storage options.
-            Defaults to None.
-        dirfs: (bool, optional) If True, return a DirFileSystem. Defaults to True.
-        cached: (bool, optional) If True, use a cached filesystem. Defaults to False.
-        cache_storage: (str, optional) Path to the cache storage. Defaults to None.
-        **storage_options_kwargs: Additional keyword arguments for the storage options.
+    This function creates and configures a filesystem instance based on the provided path
+    and options. It supports various filesystem types including local, S3, GCS, Azure,
+    and Git-based filesystems.
 
+    Args:
+        path: URI or path to the filesystem location. Examples:
+            - Local: "/path/to/data"
+            - S3: "s3://bucket/path"
+            - GCS: "gs://bucket/path"
+            - Azure: "abfs://container/path"
+            - GitHub: "github://org/repo/path"
+        storage_options: Configuration options for the filesystem. Can be:
+            - BaseStorageOptions object with protocol-specific settings
+            - Dictionary of key-value pairs for authentication/configuration
+            - None to use environment variables or default credentials
+        dirfs: Whether to wrap filesystem in DirFileSystem for path-based operations.
+            Set to False when you need direct protocol-specific features.
+        cached: Whether to enable local caching of remote files.
+            Useful for frequently accessed remote files.
+        cache_storage: Directory path for cached files. Defaults to path-based location
+            in current directory if not specified.
+        fs: Existing filesystem instance to wrap with caching or dirfs.
+            Use this to customize an existing filesystem instance.
+        **storage_options_kwargs: Additional keyword arguments for storage options.
+            Alternative to passing storage_options dictionary.
+
+    Returns:
+        AbstractFileSystem: Configured filesystem instance with requested features.
+
+    Raises:
+        ValueError: If storage protocol or options are invalid
+        FSSpecError: If filesystem initialization fails
+        ImportError: If required filesystem backend is not installed
+
+    Example:
+        >>> # Local filesystem
+        >>> fs = get_filesystem("/path/to/data")
+        >>>
+        >>> # S3 with credentials
+        >>> fs = get_filesystem(
+        ...     "s3://bucket/data",
+        ...     storage_options={
+        ...         "key": "ACCESS_KEY",
+        ...         "secret": "SECRET_KEY"
+        ...     }
+        ... )
+        >>>
+        >>> # Cached GCS filesystem
+        >>> fs = get_filesystem(
+        ...     "gs://bucket/data",
+        ...     storage_options=GcsStorageOptions(
+        ...         token="service_account.json"
+        ...     ),
+        ...     cached=True,
+        ...     cache_storage="/tmp/gcs_cache"
+        ... )
+        >>>
+        >>> # Azure with environment credentials
+        >>> fs = get_filesystem(
+        ...     "abfs://container/data",
+        ...     storage_options=AzureStorageOptions.from_env()
+        ... )
+        >>>
+        >>> # Wrap existing filesystem
+        >>> base_fs = filesystem("s3", key="ACCESS", secret="SECRET")
+        >>> cached_fs = get_filesystem(
+        ...     fs=base_fs,
+        ...     cached=True
+        ... )
     """
     if fs is not None:
         if cached: