cloud-dog-storage 0.1.4__py3-none-any.whl

cloud_dog_storage/__init__.py

@@ -0,0 +1,68 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: Public API exports for cloud_dog_storage.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from cloud_dog_storage.backends.base import StorageBackend
+from cloud_dog_storage.errors import (
+    BackendConnectionError,
+    ConfigurationError,
+    NotSupportedError,
+    QuotaExceededError,
+    StorageError,
+    StorageFileNotFoundError,
+    StoragePermissionError,
+)
+from cloud_dog_storage.factory import build_storage_backend
+from cloud_dog_storage.models import StorageEntry, StorageStat, StoredFile
+from cloud_dog_storage import path_utils
+from cloud_dog_storage.transfer import (
+    AttachmentDescriptor,
+    decode_base64,
+    detect_content_type,
+    encode_base64,
+    fetch_uri,
+    list_mime_attachments,
+    sanitize_filename,
+    validate_file_size,
+)
+
+__all__ = [
+    "AsyncStorageBackend",
+    "AttachmentDescriptor",
+    "BackendConnectionError",
+    "ConfigurationError",
+    "NotSupportedError",
+    "QuotaExceededError",
+    "StorageBackend",
+    "StorageEntry",
+    "StorageError",
+    "StorageFileNotFoundError",
+    "StoragePermissionError",
+    "StorageStat",
+    "StoredFile",
+    "build_storage_backend",
+    "decode_base64",
+    "detect_content_type",
+    "encode_base64",
+    "fetch_uri",
+    "list_mime_attachments",
+    "path_utils",
+    "sanitize_filename",
+    "validate_file_size",
+]
+
+__version__ = "0.1.4"
+
+
+def __getattr__(name: str):
+    """Lazy import async wrapper to avoid unnecessary asyncio imports."""
+    if name == "AsyncStorageBackend":
+        from cloud_dog_storage.async_wrapper import AsyncStorageBackend
+
+        return AsyncStorageBackend
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

cloud_dog_storage/api/__init__.py

@@ -0,0 +1,8 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: API integration exports for cloud_dog_storage.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""

cloud_dog_storage/api/fastapi/__init__.py

@@ -0,0 +1,12 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: FastAPI dependency exports for storage backends.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from cloud_dog_storage.api.fastapi.deps import get_storage_backend
+
+__all__ = ["get_storage_backend"]

cloud_dog_storage/api/fastapi/deps.py

@@ -0,0 +1,42 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: FastAPI dependency provider for storage backend access.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from typing import TYPE_CHECKING
+
+try:
+    # Optional dependency: only imported when FastAPI integration is used.
+    from starlette.requests import Request
+except Exception:  # pragma: no cover
+    Request = object  # type: ignore[assignment]
+
+from cloud_dog_storage.config.models import StorageConfig
+from cloud_dog_storage.factory import build_storage_backend
+
+if TYPE_CHECKING:
+    from cloud_dog_storage.backends.base import StorageBackend
+
+
+def get_storage_backend(request: Request) -> "StorageBackend":
+    """Return storage backend from app state, creating it from `storage_config` if needed."""
+    app_state = request.app.state
+    backend = getattr(app_state, "storage_backend", None)
+    if backend is not None:
+        return backend
+
+    config = getattr(app_state, "storage_config", None)
+    if config is None:
+        raise RuntimeError("FastAPI app.state.storage_config or app.state.storage_backend is required")
+
+    if isinstance(config, StorageConfig):
+        backend = build_storage_backend(config)
+    else:
+        backend = build_storage_backend(StorageConfig.model_validate(config))
+
+    app_state.storage_backend = backend
+    return backend

cloud_dog_storage/async_wrapper.py

@@ -0,0 +1,133 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: Async wrapper that delegates sync storage operations to a thread pool.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing import TYPE_CHECKING
+
+from cloud_dog_storage.models import StorageEntry, StorageStat, StoredFile
+from cloud_dog_storage.security.path_sanitiser import clean_posix
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from cloud_dog_storage.backends.base import StorageBackend
+
+
+class AsyncStorageBackend:
+    """Wrap a synchronous backend and execute operations in a thread pool.
+
+    This wrapper uses an explicit ThreadPoolExecutor instead of the event loop default
+    executor to avoid environment-specific default executor issues.
+    """
+
+    def __init__(self, backend: StorageBackend, *, max_workers: int | None = None) -> None:
+        self._backend = backend
+        self._executor = ThreadPoolExecutor(
+            max_workers=max_workers,
+            thread_name_prefix="cloud_dog_storage",
+        )
+
+    @property
+    def backend_name(self) -> str:
+        return self._backend.backend_name
+
+    def close(self) -> None:
+        """Shut down the internal executor."""
+        self._executor.shutdown(wait=False, cancel_futures=True)
+
+    async def _run(self, func, /, *args, **kwargs):  # type: ignore[no-untyped-def]
+        # In some environments, the event-loop self-pipe wake mechanism is unreliable, causing
+        # `await loop.run_in_executor(...)` to hang if the thread finishes after the loop blocks.
+        # Submitting directly and polling via `asyncio.wait(..., timeout=...)` guarantees the loop
+        # wakes periodically to process the completion callback.
+        cfut = self._executor.submit(partial(func, *args, **kwargs))
+        afut = asyncio.wrap_future(cfut)
+        while True:
+            done, _ = await asyncio.wait({afut}, timeout=0.05)
+            if done:
+                return await afut
+
+    async def read_bytes(self, path: str) -> bytes:
+        return await self._run(self._backend.read_bytes, path)
+
+    async def write_bytes(self, path: str, data: bytes, *, overwrite: bool = True) -> None:
+        await self._run(self._backend.write_bytes, path, data, overwrite=overwrite)
+
+    async def delete_path(self, path: str, *, missing_ok: bool = False) -> None:
+        await self._run(self._backend.delete_path, path, missing_ok=missing_ok)
+
+    async def list_dir(self, path: str, *, recursive: bool = False) -> list[StorageEntry]:
+        return await self._run(self._backend.list_dir, path, recursive=recursive)
+
+    async def stat(self, path: str) -> StorageStat | None:
+        return await self._run(self._backend.stat, path)
+
+    async def exists(self, path: str) -> bool:
+        return await self._run(self._backend.exists, path)
+
+    async def create_dir(self, path: str, *, parents: bool = True, exist_ok: bool = True) -> None:
+        await self._run(self._backend.create_dir, path, parents=parents, exist_ok=exist_ok)
+
+    async def copy_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        await self._run(self._backend.copy_path, src, dst, overwrite=overwrite)
+
+    async def move_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        await self._run(self._backend.move_path, src, dst, overwrite=overwrite)
+
+    async def rename_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        await self._run(self._backend.rename_path, src, dst, overwrite=overwrite)
+
+    async def chmod_path(self, path: str, mode: int, *, recursive: bool = False) -> None:
+        await self._run(self._backend.chmod_path, path, mode, recursive=recursive)
+
+    async def iter_paths(self, roots: Iterable[str], *, max_depth: int | None = None) -> list[str]:
+        return await self._run(lambda: list(self._backend.iter_paths(roots, max_depth=max_depth)))
+
+    async def get_url(self, path: str) -> str:
+        return await self._run(self._backend.get_url, path)
+
+    async def store_file(
+        self,
+        content: bytes,
+        filename: str,
+        content_type: str,
+        metadata: dict | None = None,
+    ) -> StoredFile:
+        logical = clean_posix(filename)
+        await self.write_bytes(logical, content, overwrite=True)
+        extension = os.path.splitext(logical)[1].lstrip(".") or (
+            content_type.split("/")[-1] if "/" in content_type else "bin"
+        )
+        return StoredFile(
+            path=logical,
+            format=extension,
+            size_bytes=len(content),
+            backend_name=self.backend_name,
+            metadata=metadata or {},
+        )
+
+    async def get_file_content(self, path: str) -> bytes:
+        return await self.read_bytes(path)
+
+    async def delete_file(self, path: str) -> bool:
+        if not await self.exists(path):
+            return False
+        await self.delete_path(path, missing_ok=True)
+        return True
+
+    async def file_exists(self, path: str) -> bool:
+        return await self.exists(path)
+
+    async def get_file_url(self, path: str) -> str:
+        return await self.get_url(path)

cloud_dog_storage/backends/__init__.py

@@ -0,0 +1,45 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: Storage backend exports with lazy imports for optional backends.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from cloud_dog_storage.backends.base import StorageBackend
+from cloud_dog_storage.backends.local import LocalStorage
+
+if TYPE_CHECKING:
+    from cloud_dog_storage.backends.ftp import FtpStorage
+    from cloud_dog_storage.backends.s3 import S3Storage
+    from cloud_dog_storage.backends.webdav import WebDavStorage
+
+__all__ = [
+    "FtpStorage",
+    "LocalStorage",
+    "S3Storage",
+    "StorageBackend",
+    "WebDavStorage",
+]
+
+
+def __getattr__(name: str):
+    # Optional backends must not be imported unless explicitly requested.
+    if name == "S3Storage":
+        from cloud_dog_storage.backends.s3 import S3Storage
+
+        return S3Storage
+    if name == "WebDavStorage":
+        from cloud_dog_storage.backends.webdav import WebDavStorage
+
+        return WebDavStorage
+    if name == "FtpStorage":
+        from cloud_dog_storage.backends.ftp import FtpStorage
+
+        return FtpStorage
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

cloud_dog_storage/backends/base.py

@@ -0,0 +1,89 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: Base storage backend interface and default behaviour.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from cloud_dog_storage.errors import NotSupportedError
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from cloud_dog_storage.models import StorageEntry, StorageStat
+
+
+class StorageBackend:
+    """Minimal file-like API over a backing store."""
+
+    backend_name: str = "unknown"
+
+    def read_bytes(self, path: str) -> bytes:
+        raise NotImplementedError
+
+    def write_bytes(self, path: str, data: bytes, *, overwrite: bool = True) -> None:
+        raise NotImplementedError
+
+    def delete_path(self, path: str, *, missing_ok: bool = False) -> None:
+        raise NotImplementedError
+
+    def list_dir(self, path: str, *, recursive: bool = False) -> list[StorageEntry]:
+        raise NotImplementedError
+
+    def stat(self, path: str) -> StorageStat | None:
+        raise NotImplementedError
+
+    def exists(self, path: str) -> bool:
+        """Check path existence via `stat` by default."""
+        return self.stat(path) is not None
+
+    def create_dir(self, path: str, *, parents: bool = True, exist_ok: bool = True) -> None:
+        raise NotSupportedError("create_dir", backend=self.backend_name)
+
+    def copy_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        raise NotSupportedError("copy_path", backend=self.backend_name)
+
+    def move_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        raise NotSupportedError("move_path", backend=self.backend_name)
+
+    def rename_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        self.move_path(src, dst, overwrite=overwrite)
+
+    def chmod_path(self, path: str, mode: int, *, recursive: bool = False) -> None:
+        raise NotSupportedError("chmod_path", backend=self.backend_name)
+
+    def iter_paths(self, roots: Iterable[str], *, max_depth: int | None = None) -> Iterable[str]:
+        """Enumerate file paths under the given roots."""
+        for root in roots:
+            queue: list[tuple[str, int]] = [(root, 0)]
+            while queue:
+                current, depth = queue.pop(0)
+                for entry in self.list_dir(current, recursive=False):
+                    next_depth = depth + 1
+                    if entry.is_dir:
+                        if max_depth is None or next_depth <= max_depth:
+                            queue.append((entry.path, next_depth))
+                        continue
+                    if max_depth is None or next_depth <= max_depth:
+                        yield entry.path
+
+    def append_text(self, path: str, text: str, *, encoding: str = "utf-8") -> None:
+        """Append text to a file."""
+        raise NotSupportedError("append_text", backend=self.backend_name)
+
+    def copy_with_metadata(self, src: str, dst: str) -> None:
+        """Copy a file preserving metadata (timestamps, permissions)."""
+        raise NotSupportedError("copy_with_metadata", backend=self.backend_name)
+
+    def disk_usage(self) -> tuple[int, int, int]:
+        """Return (total, used, free) bytes for the backend's storage."""
+        raise NotSupportedError("disk_usage", backend=self.backend_name)
+
+    def get_url(self, path: str) -> str:
+        raise NotSupportedError("get_url", backend=self.backend_name)

cloud_dog_storage/backends/ftp.py

@@ -0,0 +1,252 @@
+"""
+**************************************************
+License: Apache 2.0
+Ownership: Cloud-Dog, Viewdeck Engineering Ltd.
+Description: FTP/FTPS backend with MLSD fallback and connection-per-operation behaviour.
+Standard: PS-85 (Storage Interfaces)
+**************************************************
+"""
+
+from __future__ import annotations
+
+import io
+import posixpath
+from ftplib import FTP, FTP_TLS, error_perm
+
+from cloud_dog_storage.config.models import FtpConfig, TlsConfig
+from cloud_dog_storage.errors import (
+    BackendConnectionError,
+    ConfigurationError,
+    NotSupportedError,
+    StorageFileNotFoundError,
+)
+from cloud_dog_storage.models import StorageEntry, StorageStat
+from cloud_dog_storage.security.path_sanitiser import clean_posix
+from cloud_dog_storage.security.tls import build_ssl_context
+
+from .base import StorageBackend
+
+
+class FtpStorage(StorageBackend):
+    """FTP/FTPS backend with thread-safe connection-per-operation semantics."""
+
+    backend_name = "ftp"
+
+    def __init__(self, config: FtpConfig, *, tls: TlsConfig | None = None, timeout_s: int = 30) -> None:
+        if not config.host:
+            raise ConfigurationError("FTP storage requires ftp.host", backend_name=self.backend_name)
+        self._host = config.host
+        self._port = int(config.port)
+        self._username = config.username or ""
+        self._password = config.password or ""
+        self._base_dir = clean_posix(config.base_dir or "/")
+        self._use_tls = bool(config.use_tls)
+        self._timeout_s = int(timeout_s)
+        self._ssl_context = build_ssl_context(tls or TlsConfig()) if self._use_tls else None
+
+    def _connect(self) -> FTP:
+        try:
+            ftp = FTP_TLS(context=self._ssl_context) if self._use_tls else FTP()
+            ftp.connect(self._host, self._port, timeout=self._timeout_s)
+            ftp.login(self._username, self._password)
+            if self._use_tls and isinstance(ftp, FTP_TLS):
+                ftp.prot_p()
+            if self._base_dir != "/":
+                ftp.cwd(self._base_dir)
+            return ftp
+        except Exception as exc:
+            raise BackendConnectionError(f"FTP connection failed: {exc}", backend_name=self.backend_name) from exc
+
+    def _remote_path(self, path: str) -> str:
+        relative = clean_posix(path).lstrip("/")
+        if not relative:
+            return self._base_dir
+        if self._base_dir == "/":
+            return "/" + relative
+        return posixpath.join(self._base_dir, relative)
+
+    def read_bytes(self, path: str) -> bytes:
+        ftp = self._connect()
+        try:
+            buffer = io.BytesIO()
+            ftp.retrbinary(f"RETR {self._remote_path(path)}", buffer.write)
+            return buffer.getvalue()
+        except error_perm as exc:
+            if str(exc).startswith("550"):
+                raise StorageFileNotFoundError(clean_posix(path), backend=self.backend_name) from exc
+            raise
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def write_bytes(self, path: str, data: bytes, *, overwrite: bool = True) -> None:
+        ftp = self._connect()
+        remote = self._remote_path(path)
+        try:
+            if not overwrite:
+                try:
+                    ftp.size(remote)
+                    raise FileExistsError(clean_posix(path))
+                except error_perm:
+                    pass
+            ftp.storbinary(f"STOR {remote}", io.BytesIO(data))
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def delete_path(self, path: str, *, missing_ok: bool = False) -> None:
+        ftp = self._connect()
+        remote = self._remote_path(path)
+        try:
+            try:
+                ftp.delete(remote)
+                return
+            except error_perm as exc:
+                if str(exc).startswith("550"):
+                    try:
+                        ftp.rmd(remote)
+                        return
+                    except error_perm as exc_dir:
+                        if str(exc_dir).startswith("550") and missing_ok:
+                            return
+                        if str(exc_dir).startswith("550"):
+                            raise StorageFileNotFoundError(clean_posix(path), backend=self.backend_name) from exc_dir
+                        raise
+                raise
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def stat(self, path: str) -> StorageStat | None:
+        ftp = self._connect()
+        remote = self._remote_path(path)
+        try:
+            try:
+                size = ftp.size(remote)
+                if size is not None:
+                    return StorageStat(path=clean_posix(path), is_dir=False, size=int(size))
+            except error_perm:
+                pass
+
+            current = ftp.pwd()
+            try:
+                ftp.cwd(remote)
+                ftp.cwd(current)
+                return StorageStat(path=clean_posix(path), is_dir=True, size=None)
+            except error_perm:
+                return None
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def list_dir(self, path: str, *, recursive: bool = False) -> list[StorageEntry]:
+        ftp = self._connect()
+        root = clean_posix(path)
+
+        def list_one(directory: str) -> list[tuple[str, bool]]:
+            remote = self._remote_path(directory)
+            items: list[tuple[str, bool]] = []
+            try:
+                for name, facts in ftp.mlsd(remote):
+                    if name in {".", ".."}:
+                        continue
+                    items.append((name, facts.get("type") == "dir"))
+                return items
+            except Exception:
+                for full in ftp.nlst(remote):
+                    name = posixpath.basename(full.rstrip("/"))
+                    if not name or name in {".", ".."}:
+                        continue
+                    is_dir = False
+                    current = ftp.pwd()
+                    try:
+                        ftp.cwd(posixpath.join(remote, name))
+                        ftp.cwd(current)
+                        is_dir = True
+                    except Exception:
+                        is_dir = False
+                    items.append((name, is_dir))
+                return items
+
+        try:
+            out: list[StorageEntry] = []
+            queue: list[str] = [root]
+            while queue:
+                current = queue.pop(0)
+                for name, is_dir in list_one(current):
+                    child = clean_posix(posixpath.join(current, name))
+                    out.append(StorageEntry(path=child, is_dir=is_dir))
+                    if recursive and is_dir:
+                        queue.append(child)
+            return out
+        except error_perm as exc:
+            if str(exc).startswith("550"):
+                raise StorageFileNotFoundError(clean_posix(path), backend=self.backend_name) from exc
+            raise
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def create_dir(self, path: str, *, parents: bool = True, exist_ok: bool = True) -> None:
+        ftp = self._connect()
+        try:
+            target = clean_posix(path)
+            parts = [part for part in target.split("/") if part]
+            current = "/"
+            for part in parts:
+                current = clean_posix(posixpath.join(current, part))
+                remote = self._remote_path(current)
+                try:
+                    ftp.mkd(remote)
+                except error_perm as exc:
+                    if exist_ok and str(exc).startswith("550"):
+                        continue
+                    raise
+                if not parents:
+                    break
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def copy_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        data = self.read_bytes(src)
+        self.write_bytes(dst, data, overwrite=overwrite)
+
+    def move_path(self, src: str, dst: str, *, overwrite: bool = False) -> None:
+        ftp = self._connect()
+        source = self._remote_path(src)
+        target = self._remote_path(dst)
+        try:
+            if not overwrite:
+                try:
+                    ftp.size(target)
+                    raise FileExistsError(clean_posix(dst))
+                except error_perm:
+                    pass
+            ftp.rename(source, target)
+        finally:
+            try:
+                ftp.quit()
+            except Exception:
+                ftp.close()
+
+    def chmod_path(self, path: str, mode: int, *, recursive: bool = False) -> None:
+        _ = path
+        _ = mode
+        _ = recursive
+        raise NotSupportedError("chmod_path", backend=self.backend_name)
+
+    def get_url(self, path: str) -> str:
+        return f"ftp://{self._host}:{self._port}{self._remote_path(path)}"