cloudfs 0.2.0__tar.gz → 0.3.0__tar.gz

{cloudfs-0.2.0 → cloudfs-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudfs
-Version: 0.2.0
+Version: 0.3.0
 Summary: An interface to interact with cloud storage as if it's a local filesystem.
 License-Expression: Apache-2.0
 License-File: LICENSE

{cloudfs-0.2.0 → cloudfs-0.3.0}/cloudfs/backend/azure.py

@@ -21,8 +21,10 @@ rmdir():
     is_dir() returns True.
 
 open():
-    Read modes download the full blob into memory. Write modes buffer in memory
-    and upload on close.
+    Read modes stream the blob via a chunked reader; write modes stream the
+    upload by staging blocks and committing them on close. Memory use is bounded
+    by the upload chunk size, not the blob size. read_bytes/write_bytes still
+    load the whole blob, matching pathlib semantics.
 
 rename():
     Implemented as copy + delete. Not atomic — a crash between the two steps
@@ -38,11 +40,16 @@ Performance:
 
 from __future__ import annotations
 
+import base64
 import io
 from typing import IO, Any, Generator, Iterator
 
 from cloudfs.base import CloudPath
 
+# Streaming upload block size. Bounds the memory held during a streaming
+# open("wb"); a chunk is staged as a block and the block list committed on close.
+_UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
+
 
 class AzurePath(CloudPath):
     """pathlib.Path-compatible interface for Azure Blob Storage.
@@ -323,25 +330,23 @@ class AzurePath(CloudPath):
         newline: str | None = None,
     ) -> IO:
         if mode in ("rb", "r"):
-            data = self.read_bytes()
-            buf = io.BytesIO(data)
+            downloader = self._container.download_blob(self._key)
+            buf = io.BufferedReader(_AzureReadBuffer(downloader))
             if mode == "r":
                 return io.TextIOWrapper(
-                    buf,
+                    buf, encoding=encoding or "utf-8", errors=errors, newline=newline
+                )
+            return buf
+        if mode in ("wb", "w"):
+            raw = _AzureWriteBuffer(self._container, self._key)
+            if mode == "w":
+                return io.TextIOWrapper(
+                    io.BufferedWriter(raw),
                     encoding=encoding or "utf-8",
                     errors=errors,
                     newline=newline,
                 )
-            return buf
-        if mode in ("wb", "w"):
-            return _AzureWriteBuffer(
-                self._container,
-                self._key,
-                binary=mode == "wb",
-                encoding=encoding or "utf-8",
-                errors=errors,
-                newline=newline,
-            )
+            return raw
         raise ValueError(f"Unsupported mode: {mode!r}")
 
     def read_bytes(self) -> bytes:
@@ -355,9 +360,7 @@ class AzurePath(CloudPath):
         return len(data)
 
     def write_text(self, data: str, encoding: str = "utf-8") -> int:
-        encoded = data.encode(encoding)
-        self._container.upload_blob(self._key, encoded, overwrite=True)
-        return len(encoded)
+        return self.write_bytes(data.encode(encoding))
 
     def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
         if self._blob_exists():
@@ -426,36 +429,60 @@ class AzurePath(CloudPath):
         return cls(container, key, _client=_client)
 
 
+class _AzureReadBuffer(io.RawIOBase):
+    def __init__(self, downloader):
+        self._chunks = downloader.chunks()
+        self._leftover = b""
+
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, b) -> int:
+        if not self._leftover:
+            try:
+                self._leftover = next(self._chunks)
+            except StopIteration:
+                return 0
+        n = min(len(b), len(self._leftover))
+        b[:n] = self._leftover[:n]
+        self._leftover = self._leftover[n:]
+        return n
+
+
 class _AzureWriteBuffer(io.RawIOBase):
-    def __init__(self, container, key, binary, encoding, errors, newline):
-        self._container = container
-        self._key = key
-        self._binary = binary
-        self._buf = io.BytesIO()
-        self._text_wrapper = None
-        if not binary:
-            self._text_wrapper = io.TextIOWrapper(
-                self._buf, encoding=encoding, errors=errors, newline=newline
-            )
+    def __init__(self, container, key):
+        self._blob = container.get_blob_client(key)
+        self._buf = bytearray()
+        self._block_ids: list[str] = []
+
+    def writable(self) -> bool:
+        return True
 
     def write(self, data) -> int:
-        if self._text_wrapper:
-            return self._text_wrapper.write(data)
-        return self._buf.write(data)
+        self._buf.extend(data)
+        while len(self._buf) >= _UPLOAD_CHUNK_SIZE:
+            self._flush_block(_UPLOAD_CHUNK_SIZE)
+        return len(data)
+
+    def _flush_block(self, size: int) -> None:
+        chunk = bytes(self._buf[:size])
+        del self._buf[:size]
+        block_id = base64.b64encode(f"{len(self._block_ids):032d}".encode()).decode()
+        self._blob.stage_block(block_id, chunk)
+        self._block_ids.append(block_id)
 
     def close(self) -> None:
-        if not self.closed:
-            if self._text_wrapper:
-                self._text_wrapper.flush()
-            self._buf.seek(0)
-            self._container.upload_blob(self._key, self._buf.read(), overwrite=True)
-        super().close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
+        if self.closed:
+            return
+        from azure.storage.blob import BlobBlock
+
+        try:
+            if self._buf:
+                self._flush_block(len(self._buf))
+            block_list = [BlobBlock(block_id=bid) for bid in self._block_ids]
+            self._blob.commit_block_list(block_list)
+        finally:
+            super().close()
 
 
 class AzureStatResult:

{cloudfs-0.2.0 → cloudfs-0.3.0}/cloudfs/backend/s3.py

@@ -20,8 +20,10 @@ rmdir():
     is_dir() returns True.
 
 open():
-    S3 has no native streaming file object. Read modes download the full object
-    into memory. Write modes buffer in memory and upload on close.
+    Read modes stream the object via a chunked reader; write modes stream the
+    upload as an S3 multipart upload, flushing one part at a time. Memory use is
+    bounded by the upload chunk size, not the object size. read_bytes/write_bytes
+    still load the whole object, matching pathlib semantics.
 
 Consistency:
     S3 provides strong read-after-write consistency for all operations since
@@ -39,6 +41,10 @@ from typing import IO, Any, Generator, Iterator
 
 from cloudfs.base import CloudPath
 
+# Streaming upload part size. Must stay >= 5 MiB, S3's minimum non-final
+# multipart part size. Bounds the memory held during a streaming open("wb").
+_UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
+
 
 class S3Path(CloudPath):
     """pathlib.Path-compatible interface for AWS S3."""
@@ -321,26 +327,25 @@ class S3Path(CloudPath):
         newline: str | None = None,
     ) -> IO:
         if mode in ("rb", "r"):
-            data = self.read_bytes()
-            buf = io.BytesIO(data)
+            body = self._client.get_object(Bucket=self._bucket_name, Key=self._key)[
+                "Body"
+            ]
+            buf = io.BufferedReader(_S3ReadBuffer(body))
             if mode == "r":
                 return io.TextIOWrapper(
-                    buf,
+                    buf, encoding=encoding or "utf-8", errors=errors, newline=newline
+                )
+            return buf
+        if mode in ("wb", "w"):
+            raw = _S3WriteBuffer(self._client, self._bucket_name, self._key)
+            if mode == "w":
+                return io.TextIOWrapper(
+                    io.BufferedWriter(raw),
                     encoding=encoding or "utf-8",
                     errors=errors,
                     newline=newline,
                 )
-            return buf
-        if mode in ("wb", "w"):
-            return _S3WriteBuffer(
-                self._client,
-                self._bucket_name,
-                self._key,
-                binary=mode == "wb",
-                encoding=encoding or "utf-8",
-                errors=errors,
-                newline=newline,
-            )
+            return raw
         raise ValueError(f"Unsupported mode: {mode!r}")
 
     def read_bytes(self) -> bytes:
@@ -355,9 +360,7 @@ class S3Path(CloudPath):
         return len(data)
 
     def write_text(self, data: str, encoding: str = "utf-8") -> int:
-        encoded = data.encode(encoding)
-        self._client.put_object(Bucket=self._bucket_name, Key=self._key, Body=encoded)
-        return len(encoded)
+        return self.write_bytes(data.encode(encoding))
 
     def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
         if self._object_exists():
@@ -428,42 +431,91 @@ class S3Path(CloudPath):
         return cls(bucket, key, _client=_client)
 
 
+class _S3ReadBuffer(io.RawIOBase):
+    def __init__(self, body):
+        self._body = body
+
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, b) -> int:
+        chunk = self._body.read(len(b))
+        if not chunk:
+            return 0
+        n = len(chunk)
+        b[:n] = chunk
+        return n
+
+    def close(self) -> None:
+        if not self.closed:
+            try:
+                self._body.close()
+            finally:
+                super().close()
+
+
 class _S3WriteBuffer(io.RawIOBase):
-    def __init__(self, client, bucket, key, binary, encoding, errors, newline):
+    def __init__(self, client, bucket, key):
         self._client = client
         self._bucket = bucket
         self._key = key
-        self._binary = binary
-        self._encoding = encoding
-        self._errors = errors
-        self._newline = newline
-        self._buf = io.BytesIO()
-        self._text_wrapper = None
-        if not binary:
-            self._text_wrapper = io.TextIOWrapper(
-                self._buf, encoding=encoding, errors=errors, newline=newline
-            )
+        self._buf = bytearray()
+        self._parts: list[dict] = []
+        self._part_num = 1
+        resp = client.create_multipart_upload(Bucket=bucket, Key=key)
+        self._upload_id = resp["UploadId"]
+
+    def writable(self) -> bool:
+        return True
 
     def write(self, data) -> int:
-        if self._text_wrapper:
-            return self._text_wrapper.write(data)
-        return self._buf.write(data)
+        self._buf.extend(data)
+        while len(self._buf) >= _UPLOAD_CHUNK_SIZE:
+            self._flush_part(_UPLOAD_CHUNK_SIZE)
+        return len(data)
 
-    def close(self) -> None:
-        if not self.closed:
-            if self._text_wrapper:
-                self._text_wrapper.flush()
-            self._buf.seek(0)
-            self._client.put_object(
-                Bucket=self._bucket, Key=self._key, Body=self._buf.read()
-            )
-        super().close()
+    def _flush_part(self, size: int) -> None:
+        chunk = bytes(self._buf[:size])
+        del self._buf[:size]
+        resp = self._client.upload_part(
+            Bucket=self._bucket,
+            Key=self._key,
+            PartNumber=self._part_num,
+            UploadId=self._upload_id,
+            Body=chunk,
+        )
+        self._parts.append({"PartNumber": self._part_num, "ETag": resp["ETag"]})
+        self._part_num += 1
 
-    def __enter__(self):
-        return self
+    def _safe_abort(self) -> None:
+        try:
+            self._client.abort_multipart_upload(
+                Bucket=self._bucket, Key=self._key, UploadId=self._upload_id
+            )
+        except Exception:
+            pass
 
-    def __exit__(self, *args):
-        self.close()
+    def close(self) -> None:
+        if self.closed:
+            return
+        try:
+            if self._buf:
+                self._flush_part(len(self._buf))
+            if self._parts:
+                self._client.complete_multipart_upload(
+                    Bucket=self._bucket,
+                    Key=self._key,
+                    UploadId=self._upload_id,
+                    MultipartUpload={"Parts": self._parts},
+                )
+            else:
+                self._safe_abort()
+                self._client.put_object(Bucket=self._bucket, Key=self._key, Body=b"")
+        except Exception:
+            self._safe_abort()
+            raise
+        finally:
+            super().close()
 
 
 class S3StatResult:

cloudfs-0.3.0/cloudfs/version.py

@@ -0,0 +1 @@
+VERSION = "0.3.0"

{cloudfs-0.2.0 → cloudfs-0.3.0}/pyproject.toml

@@ -1,22 +1,22 @@
 [project]
-name = "cloudfs"
-version = "0.2.0"
-description = "An interface to interact with cloud storage as if it's a local filesystem."
 authors = [{ name = "Allen Chou", email = "f1470891079@gmail.com" }]
+dependencies = []
+description = "An interface to interact with cloud storage as if it's a local filesystem."
 license = "Apache-2.0"
+name = "cloudfs"
 readme = "README.md"
 requires-python = ">=3.11,<4.0"
-dependencies = []
+version = "0.3.0"
 
 [project.optional-dependencies]
-google = ["google-cloud-storage>=3,<4"]
-s3 = ["boto3>=1.35,<2"]
-azure = ["azure-storage-blob>=12,<13"]
 all = [
-    "google-cloud-storage>=3,<4",
-    "boto3>=1.35,<2",
     "azure-storage-blob>=12,<13",
+    "boto3>=1.35,<2",
+    "google-cloud-storage>=3,<4",
 ]
+azure = ["azure-storage-blob>=12,<13"]
+google = ["google-cloud-storage>=3,<4"]
+s3 = ["boto3>=1.35,<2"]
 
 [dependency-groups]
 dev = [
@@ -29,19 +29,29 @@ dev = [
     "pytest",
     "pytest-xdist",
     "python-dotenv",
+    "ruff",
 ]
 
 [tool.black]
+line-length = 88
 target-version = ["py311"]
 
 [tool.flake8]
-max-line-length = 88
 extend-ignore = ["E203"]
+max-line-length = 88
 
 [tool.isort]
-profile = "black"
 line_length = 88
+profile = "black"
+
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+
+[tool.ruff.lint]
+# E203 (whitespace before ':') conflicts with black; mirror the flake8 ignore.
+extend-ignore = ["E203"]
 
 [build-system]
-requires = ["poetry-core>=2.0.0,<3.0.0"]
 build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=2.0.0,<3.0.0"]

cloudfs-0.2.0/cloudfs/version.py

@@ -1 +0,0 @@
-VERSION = "0.2.0"