cloudfs 0.1.0__tar.gz → 0.2.0__tar.gz

cloudfs-0.2.0/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.4
+Name: cloudfs
+Version: 0.2.0
+Summary: An interface to interact with cloud storage as if it's a local filesystem.
+License-Expression: Apache-2.0
+License-File: LICENSE
+Author: Allen Chou
+Author-email: f1470891079@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Provides-Extra: all
+Provides-Extra: azure
+Provides-Extra: google
+Provides-Extra: s3
+Requires-Dist: azure-storage-blob (>=12,<13) ; extra == "all"
+Requires-Dist: azure-storage-blob (>=12,<13) ; extra == "azure"
+Requires-Dist: boto3 (>=1.35,<2) ; extra == "all"
+Requires-Dist: boto3 (>=1.35,<2) ; extra == "s3"
+Requires-Dist: google-cloud-storage (>=3,<4) ; extra == "all"
+Requires-Dist: google-cloud-storage (>=3,<4) ; extra == "google"
+Description-Content-Type: text/markdown
+
+# CloudFS
+
+Cloud storage that works like your local filesystem.
+
+CloudFS gives you a `pathlib.Path`-compatible interface for Google Cloud Storage, AWS S3, and Azure Blob Storage.
+
+## Installation
+
+```bash
+pip install "cloudfs[google]"   # Google Cloud Storage
+pip install "cloudfs[s3]"       # AWS S3
+pip install "cloudfs[azure]"    # Azure Blob Storage
+pip install "cloudfs[all]"      # All backends
+```
+
+## Quick Start
+
+```python
+from cloudfs import Path
+
+# Works the same across all backends
+p = Path("gs://my-bucket/data/report.csv")    # GCS
+p = Path("s3://my-bucket/data/report.csv")    # S3
+p = Path("az://my-container/data/report.csv") # Azure
+
+p.write_text("Hello, CloudFS!")
+print(p.read_text())            # Hello, CloudFS!
+print(p.name)                   # report.csv
+print(p.parent / "other.csv")  # gs://my-bucket/data/other.csv
+
+for child in p.parent.iterdir():
+    print(child)
+```
+
+## Documentation
+
+Full documentation at **[allen2c.github.io/cloudfs](https://allen2c.github.io/cloudfs/)**.
+
+## License
+
+Apache 2.0
+

cloudfs-0.2.0/README.md

@@ -0,0 +1,41 @@
+# CloudFS
+
+Cloud storage that works like your local filesystem.
+
+CloudFS gives you a `pathlib.Path`-compatible interface for Google Cloud Storage, AWS S3, and Azure Blob Storage.
+
+## Installation
+
+```bash
+pip install "cloudfs[google]"   # Google Cloud Storage
+pip install "cloudfs[s3]"       # AWS S3
+pip install "cloudfs[azure]"    # Azure Blob Storage
+pip install "cloudfs[all]"      # All backends
+```
+
+## Quick Start
+
+```python
+from cloudfs import Path
+
+# Works the same across all backends
+p = Path("gs://my-bucket/data/report.csv")    # GCS
+p = Path("s3://my-bucket/data/report.csv")    # S3
+p = Path("az://my-container/data/report.csv") # Azure
+
+p.write_text("Hello, CloudFS!")
+print(p.read_text())            # Hello, CloudFS!
+print(p.name)                   # report.csv
+print(p.parent / "other.csv")  # gs://my-bucket/data/other.csv
+
+for child in p.parent.iterdir():
+    print(child)
+```
+
+## Documentation
+
+Full documentation at **[allen2c.github.io/cloudfs](https://allen2c.github.io/cloudfs/)**.
+
+## License
+
+Apache 2.0

cloudfs-0.2.0/cloudfs/__init__.py

@@ -0,0 +1,5 @@
+from .base import CloudPath as Path
+from .exceptions import CloudOperationError
+from .version import VERSION as __version__
+
+__all__ = ["__version__", "Path", "CloudOperationError"]

cloudfs-0.2.0/cloudfs/backend/azure.py

@@ -0,0 +1,483 @@
+"""Azure Blob Storage backend for CloudFS.
+
+Azure-specific behavior and known differences from local filesystems:
+
+Directories:
+    Azure Blob Storage has no real directories — only blob names containing
+    slashes. Directories are simulated either by placeholder blobs (a zero-byte
+    blob named "prefix/") created by mkdir(), or implicitly by blobs that share
+    a common prefix.
+
+    - mkdir() creates a placeholder blob. If all files under a directory are
+      unlinked but the placeholder remains, is_dir() still returns True.
+    - Writing directly to a sub-path (write_text, write_bytes) never requires
+      a prior mkdir(), unlike local filesystems.
+    - A path can simultaneously satisfy is_file() and is_dir() if a blob named
+      "foo" and another named "foo/bar" both exist. This cannot happen locally.
+
+rmdir():
+    Only removes the placeholder blob created by mkdir(). A "virtual" directory
+    that was never explicitly created will raise FileNotFoundError even if
+    is_dir() returns True.
+
+open():
+    Read modes download the full blob into memory. Write modes buffer in memory
+    and upload on close.
+
+rename():
+    Implemented as copy + delete. Not atomic — a crash between the two steps
+    leaves both source and destination in place.
+
+Consistency:
+    Azure Blob Storage provides strong consistency for all operations.
+
+Performance:
+    Each exists(), is_file(), and is_dir() call makes at least one API request.
+    Avoid calling them in tight loops; prefer bulk listing via iterdir() or walk().
+"""
+
+from __future__ import annotations
+
+import io
+from typing import IO, Any, Generator, Iterator
+
+from cloudfs.base import CloudPath
+
+
+class AzurePath(CloudPath):
+    """pathlib.Path-compatible interface for Azure Blob Storage.
+
+    URI format: az://container/blob
+    Credentials: set AZURE_STORAGE_CONNECTION_STRING in environment.
+    """
+
+    def __init__(self, container: str, key: str = "", _client=None):
+        self._container_name = container
+        self._key = key.strip("/") if key else ""
+        self.__client = _client
+
+    @property
+    def _client(self):
+        if self.__client is None:
+            import os
+
+            from azure.storage.blob import BlobServiceClient
+
+            conn_str = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
+            self.__client = BlobServiceClient.from_connection_string(conn_str)
+        return self.__client
+
+    @property
+    def _container(self):
+        return self._client.get_container_client(self._container_name)
+
+    def _child(self, key: str) -> "AzurePath":
+        return AzurePath(self._container_name, key, _client=self.__client)
+
+    def __str__(self) -> str:
+        if self._key:
+            return f"az://{self._container_name}/{self._key}"
+        return f"az://{self._container_name}"
+
+    def __repr__(self) -> str:
+        return f"AzurePath('{self}')"
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, AzurePath):
+            return (
+                self._container_name == other._container_name
+                and self._key == other._key
+            )
+        return NotImplemented
+
+    def __hash__(self) -> int:
+        return hash((self._container_name, self._key))
+
+    def __lt__(self, other: "CloudPath") -> bool:
+        return str(self) < str(other)
+
+    def __le__(self, other: "CloudPath") -> bool:
+        return str(self) <= str(other)
+
+    def __gt__(self, other: "CloudPath") -> bool:
+        return str(self) > str(other)
+
+    def __ge__(self, other: "CloudPath") -> bool:
+        return str(self) >= str(other)
+
+    @property
+    def drive(self) -> str:
+        return f"az://{self._container_name}"
+
+    @property
+    def root(self) -> str:
+        return "/"
+
+    @property
+    def anchor(self) -> str:
+        return f"az://{self._container_name}/"
+
+    @property
+    def name(self) -> str:
+        return self._key.split("/")[-1] if self._key else ""
+
+    @property
+    def stem(self) -> str:
+        n = self.name
+        idx = n.rfind(".")
+        return n[:idx] if idx > 0 else n
+
+    @property
+    def suffix(self) -> str:
+        n = self.name
+        idx = n.rfind(".")
+        return n[idx:] if idx > 0 else ""
+
+    @property
+    def suffixes(self) -> list[str]:
+        parts = self.name.split(".")
+        return ["." + p for p in parts[1:]] if len(parts) > 1 else []
+
+    @property
+    def parent(self) -> "AzurePath":
+        if "/" in self._key:
+            return self._child("/".join(self._key.split("/")[:-1]))
+        return self._child("")
+
+    @property
+    def parents(self) -> list["AzurePath"]:
+        parts = self._key.split("/") if self._key else []
+        result = []
+        for i in range(len(parts) - 1, -1, -1):
+            result.append(self._child("/".join(parts[:i])))
+        return result
+
+    @property
+    def parts(self) -> tuple[str, ...]:
+        root = f"az://{self._container_name}/"
+        if not self._key:
+            return (root,)
+        return (root,) + tuple(self._key.split("/"))
+
+    def __truediv__(self, other: str) -> "AzurePath":
+        other = str(other).strip("/")
+        new_key = f"{self._key}/{other}" if self._key else other
+        return self._child(new_key)
+
+    def joinpath(self, *others: str) -> "AzurePath":
+        result = self
+        for part in others:
+            result = result / part
+        return result
+
+    def with_name(self, name: str) -> "AzurePath":
+        if not self._key:
+            raise ValueError("AzurePath has no name component")
+        parent_key = "/".join(self._key.split("/")[:-1])
+        new_key = f"{parent_key}/{name}" if parent_key else name
+        return self._child(new_key)
+
+    def with_stem(self, stem: str) -> "AzurePath":
+        return self.with_name(stem + self.suffix)
+
+    def with_suffix(self, suffix: str) -> "AzurePath":
+        if suffix and not suffix.startswith("."):
+            raise ValueError(f"Invalid suffix: {suffix!r}")
+        return self.with_name(self.stem + suffix)
+
+    def is_absolute(self) -> bool:
+        return True
+
+    def resolve(self, strict: bool = False) -> "AzurePath":
+        return self._child(self._key)
+
+    def absolute(self) -> "AzurePath":
+        return self._child(self._key)
+
+    def _blob_exists(self) -> bool:
+        from azure.core.exceptions import ResourceNotFoundError
+
+        try:
+            self._container.get_blob_client(self._key).get_blob_properties()
+            return True
+        except ResourceNotFoundError:
+            return False
+
+    def _has_children(self, prefix: str) -> bool:
+        items = self._container.list_blobs(name_starts_with=prefix)
+        return any(True for _ in items)
+
+    def exists(self) -> bool:
+        if not self._key:
+            try:
+                self._container.get_container_properties()
+                return True
+            except Exception:
+                return False
+        if self._blob_exists():
+            return True
+        return self._has_children(self._key.rstrip("/") + "/")
+
+    def is_file(self) -> bool:
+        return bool(self._key) and self._blob_exists()
+
+    def is_dir(self) -> bool:
+        if not self._key:
+            return self.exists()
+        if self._blob_exists():
+            return False
+        return self._has_children(self._key.rstrip("/") + "/")
+
+    def samefile(self, other: "CloudPath") -> bool:
+        if not isinstance(other, AzurePath):
+            return False
+        return self._container_name == other._container_name and self._key == other._key
+
+    def iterdir(self) -> Iterator["AzurePath"]:
+        from azure.storage.blob import BlobPrefix
+
+        prefix = (self._key.rstrip("/") + "/") if self._key else ""
+        seen: set[str] = set()
+        for item in self._container.walk_blobs(name_starts_with=prefix, delimiter="/"):
+            if isinstance(item, BlobPrefix):
+                rel = item.name[len(prefix) :].rstrip("/")
+            else:
+                rel = item.name[len(prefix) :]
+                rel = rel.split("/")[0]
+            if rel and rel not in seen:
+                seen.add(rel)
+                yield self._child(prefix + rel)
+
+    def glob(self, pattern: str) -> Iterator["AzurePath"]:
+        import fnmatch
+
+        prefix = (self._key.rstrip("/") + "/") if self._key else ""
+        for blob in self._container.list_blobs(name_starts_with=prefix):
+            rel = blob.name[len(prefix) :]
+            if fnmatch.fnmatch(rel, pattern):
+                yield self._child(blob.name)
+
+    def rglob(self, pattern: str) -> Iterator["AzurePath"]:
+        import fnmatch
+
+        prefix = (self._key.rstrip("/") + "/") if self._key else ""
+        for blob in self._container.list_blobs(name_starts_with=prefix):
+            rel = blob.name[len(prefix) :]
+            if fnmatch.fnmatch(rel, "**/" + pattern) or fnmatch.fnmatch(rel, pattern):
+                yield self._child(blob.name)
+
+    def walk(
+        self,
+        top_down: bool = True,
+        on_error: Any = None,
+    ) -> Generator[tuple["AzurePath", list[str], list[str]], None, None]:
+        from collections import defaultdict
+
+        prefix = (self._key.rstrip("/") + "/") if self._key else ""
+        tree: dict[str, tuple[list[str], list[str]]] = defaultdict(lambda: ([], []))
+        tree[self._key]
+
+        try:
+            for blob in self._container.list_blobs(name_starts_with=prefix):
+                rel = blob.name[len(prefix) :]
+                if not rel:
+                    continue
+                parts = rel.split("/")
+                dir_key = self._key
+                for part in parts[:-1]:
+                    parent_key = dir_key
+                    dir_key = f"{dir_key}/{part}" if dir_key else part
+                    if part not in tree[parent_key][0]:
+                        tree[parent_key][0].append(part)
+                    tree[dir_key]
+                tree[dir_key][1].append(parts[-1])
+        except Exception as e:
+            if on_error:
+                on_error(e)
+            return
+
+        def _yield(
+            key: str,
+        ) -> Generator[tuple["AzurePath", list[str], list[str]], None, None]:
+            dirnames, filenames = tree[key]
+            dirpath = self._child(key)
+            if top_down:
+                yield dirpath, list(dirnames), list(filenames)
+                for d in dirnames:
+                    child_key = f"{key}/{d}" if key else d
+                    yield from _yield(child_key)
+            else:
+                for d in dirnames:
+                    child_key = f"{key}/{d}" if key else d
+                    yield from _yield(child_key)
+                yield dirpath, list(dirnames), list(filenames)
+
+        yield from _yield(self._key)
+
+    def open(
+        self,
+        mode: str = "r",
+        buffering: int = -1,
+        encoding: str | None = None,
+        errors: str | None = None,
+        newline: str | None = None,
+    ) -> IO:
+        if mode in ("rb", "r"):
+            data = self.read_bytes()
+            buf = io.BytesIO(data)
+            if mode == "r":
+                return io.TextIOWrapper(
+                    buf,
+                    encoding=encoding or "utf-8",
+                    errors=errors,
+                    newline=newline,
+                )
+            return buf
+        if mode in ("wb", "w"):
+            return _AzureWriteBuffer(
+                self._container,
+                self._key,
+                binary=mode == "wb",
+                encoding=encoding or "utf-8",
+                errors=errors,
+                newline=newline,
+            )
+        raise ValueError(f"Unsupported mode: {mode!r}")
+
+    def read_bytes(self) -> bytes:
+        return self._container.download_blob(self._key).readall()
+
+    def read_text(self, encoding: str = "utf-8") -> str:
+        return self.read_bytes().decode(encoding)
+
+    def write_bytes(self, data: bytes) -> int:
+        self._container.upload_blob(self._key, data, overwrite=True)
+        return len(data)
+
+    def write_text(self, data: str, encoding: str = "utf-8") -> int:
+        encoded = data.encode(encoding)
+        self._container.upload_blob(self._key, encoded, overwrite=True)
+        return len(encoded)
+
+    def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+        if self._blob_exists():
+            if not exist_ok:
+                raise FileExistsError(str(self))
+            return
+        self._container.upload_blob(self._key, b"", overwrite=True)
+
+    def unlink(self, missing_ok: bool = False) -> None:
+        if not self._blob_exists():
+            if missing_ok:
+                return
+            raise FileNotFoundError(str(self))
+        self._container.delete_blob(self._key)
+
+    def rename(self, target: "AzurePath | str") -> "AzurePath":
+        if isinstance(target, str):
+            target = AzurePath.from_uri(target)
+        src_url = self._container.get_blob_client(self._key).url
+        dst_blob = target._container.get_blob_client(target._key)
+        dst_blob.start_copy_from_url(src_url)
+        self._container.delete_blob(self._key)
+        return target
+
+    def replace(self, target: "AzurePath | str") -> "AzurePath":
+        return self.rename(target)
+
+    def mkdir(self, parents: bool = False, exist_ok: bool = False) -> None:
+        if self.exists():
+            if exist_ok:
+                return
+            raise FileExistsError(str(self))
+        placeholder = self._key.rstrip("/") + "/"
+        self._container.upload_blob(placeholder, b"", overwrite=True)
+
+    def rmdir(self) -> None:
+        from azure.core.exceptions import ResourceNotFoundError
+
+        placeholder = self._key.rstrip("/") + "/"
+        blob_client = self._container.get_blob_client(placeholder)
+        try:
+            blob_client.get_blob_properties()
+        except ResourceNotFoundError:
+            raise FileNotFoundError(str(self))
+        children = [
+            b
+            for b in self._container.list_blobs(name_starts_with=placeholder)
+            if b.name != placeholder
+        ]
+        if children:
+            raise OSError(f"Directory not empty: {self}")
+        self._container.delete_blob(placeholder)
+
+    def stat(self, follow_symlinks: bool = True) -> "AzureStatResult":
+        props = self._container.get_blob_client(self._key).get_blob_properties()
+        return AzureStatResult(props)
+
+    @classmethod
+    def from_uri(cls, uri: str, _client=None) -> "AzurePath":
+        if not uri.startswith("az://"):
+            raise ValueError(f"Not an Azure URI: {uri!r}")
+        without_scheme = uri[5:]
+        parts = without_scheme.split("/", 1)
+        container = parts[0]
+        key = parts[1] if len(parts) > 1 else ""
+        return cls(container, key, _client=_client)
+
+
+class _AzureWriteBuffer(io.RawIOBase):
+    def __init__(self, container, key, binary, encoding, errors, newline):
+        self._container = container
+        self._key = key
+        self._binary = binary
+        self._buf = io.BytesIO()
+        self._text_wrapper = None
+        if not binary:
+            self._text_wrapper = io.TextIOWrapper(
+                self._buf, encoding=encoding, errors=errors, newline=newline
+            )
+
+    def write(self, data) -> int:
+        if self._text_wrapper:
+            return self._text_wrapper.write(data)
+        return self._buf.write(data)
+
+    def close(self) -> None:
+        if not self.closed:
+            if self._text_wrapper:
+                self._text_wrapper.flush()
+            self._buf.seek(0)
+            self._container.upload_blob(self._key, self._buf.read(), overwrite=True)
+        super().close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+class AzureStatResult:
+    def __init__(self, props):
+        self._props = props
+
+    @property
+    def st_size(self) -> int:
+        return self._props.get("size", 0) or 0
+
+    @property
+    def st_mtime(self) -> float:
+        dt = self._props.get("last_modified")
+        return dt.timestamp() if dt else 0.0
+
+    @property
+    def st_ctime(self) -> float:
+        dt = self._props.get("creation_time")
+        return dt.timestamp() if dt else self.st_mtime
+
+    def __repr__(self) -> str:
+        return (
+            f"AzureStatResult(st_size={self.st_size}, "
+            f"st_mtime={self.st_mtime}, st_ctime={self.st_ctime})"
+        )