flatfs 0.2.0__tar.gz → 0.3.0__tar.gz

{flatfs-0.2.0 → flatfs-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flatfs
-Version: 0.2.0
+Version: 0.3.0
 Summary: Minimal filesystem abstraction with pluggable backends, treating paths as keys and file contents as values.
 License: MIT
 License-File: LICENSE.txt

flatfs-0.3.0/flatfs/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{flatfs-0.2.0 → flatfs-0.3.0}/flatfs/_utils.py

@@ -1,7 +1,8 @@
 import asyncio
 import datetime
 import functools
-from typing import Callable, TypeVar
+import queue
+from typing import AsyncGenerator, Callable, Generator, TypeVar
 
 from flatfs.exc import PathAccessError
 
@@ -56,3 +57,54 @@ async def run_blocking(func: Callable[..., T], *args, **kwargs) -> T:
 def utcnow() -> datetime.datetime:
     """Get the current date and time in UTC."""
     return datetime.datetime.now(datetime.timezone.utc)
+
+
+def split_into_chunks(data: bytes, chunk_size: int) -> Generator[bytes, None, None]:
+    """Generator that splits given binary data into chunks of provided maximum size.
+
+    :param data:
+        The data to split.
+
+    :chunk_size:
+        The maximum size of a single chunk.
+    """
+    while chunk := data[:chunk_size]:
+        yield chunk
+        data = data[chunk_size:]
+
+
+async def async_split_into_chunks(data: bytes, chunk_size: int) -> AsyncGenerator[bytes, None]:
+    """Async variant of the :func:`split_into_chunks` function.
+
+    :param data:
+        The data to split.
+
+    :param chunk_size:
+        The maximum size of a single chunk.
+    """
+    while chunk := data[:chunk_size]:
+        yield chunk
+        data = data[chunk_size:]
+
+
+def generate_chunks_from_queue(chunk_queue: queue.Queue[bytes]) -> Generator[bytes, None, None]:
+    """Generator that yields chunks it receives via provided queue until empty
+    bytes object is received.
+
+    :param chunk_queue:
+        Chunk queue to adapt.
+    """
+    while True:
+        chunk = chunk_queue.get()
+        if not chunk:
+            return
+        yield chunk
+
+
+async def async_generate_chunks_from_queue(chunk_queue: asyncio.Queue[bytes]) -> AsyncGenerator[bytes, None]:
+    """Async variant of the :func:`generate_chunks_from_queue` function."""
+    while True:
+        chunk = await chunk_queue.get()
+        if not chunk:
+            return
+        yield chunk

{flatfs-0.2.0 → flatfs-0.3.0}/flatfs/backends.py

@@ -5,7 +5,7 @@ import fnmatch
 import os
 import pathlib
 from queue import Queue
-from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Iterator, Optional
+from typing import AsyncGenerator, AsyncIterator, Generator, Iterator, Optional
 
 from flatfs.exc import PathAccessError, PathNotFoundError
 from flatfs.interface import FlatFsReaderWriter, Stat
@@ -107,14 +107,12 @@ class LocalFlatFs:
                     break
                 yield chunk
 
-    def write_chunks(self, path: str, chunks: Iterable[bytes]) -> int:
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
         abspath = self.__make_abspath(path)
         abspath.parent.mkdir(parents=True, exist_ok=True)
-        total_bytes = 0
         with abspath.open("wb") as fd:
-            for chunk in chunks:
-                total_bytes += fd.write(chunk)
-        return total_bytes
+            for chunk in chunk_gen:
+                yield fd.write(chunk)
 
     def remove(self, path: str):
         abspath = self.__make_abspath(path)
@@ -165,15 +163,18 @@ class InMemoryFlatFs:
             data_left = data_left[chunk_size:]
             yield chunk
 
-    def write_chunks(self, path: str, chunks: Iterable[bytes]) -> int:
-        payload = b"".join(chunks)
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
         now = _utils.utcnow()
         stat = Stat(
             modified=now,
-            size=len(payload),
+            size=0,
         )
-        self.__storage[self.__make_key(path)] = self._File(payload, stat)
-        return stat.size
+        self.__storage[self.__make_key(path)] = file = self._File(b"", stat)
+        for chunk in chunk_gen:
+            file.payload += chunk
+            write_count = len(chunk)
+            file.stat.size += write_count
+            yield write_count
 
     def remove(self, path: str):
         key = self.__make_key(path)
@@ -221,7 +222,7 @@ class AsyncFlatFsAdapter:
     async def stat(self, path: str) -> Stat:
         return await _utils.run_blocking(self.__target.stat, path)
 
-    async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
+    async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncIterator[bytes]:
 
         def reader():
             try:
@@ -242,21 +243,23 @@ class AsyncFlatFsAdapter:
                 raise chunk_or_exc
             yield chunk_or_exc
 
-    async def write_chunks(self, path: str, chunks: AsyncIterable[bytes]) -> int:
+    async def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
 
-        def gen() -> Generator[bytes, None, None]:
-            while True:
-                chunk = queue.get()
-                if not chunk:
-                    return
-                yield chunk
+        def blocking_writer():
+            for write_count in self.__target.write_chunks(path, _utils.generate_chunks_from_queue(chunk_queue)):
+                loop.call_soon_threadsafe(ack_queue.put_nowait, write_count)
 
-        queue: Queue[bytes] = Queue()
-        task = asyncio.create_task(_utils.run_blocking(self.__target.write_chunks, path, gen()))
-        async for chunk in chunks:
-            queue.put(chunk)
-        queue.put(b"")
-        return await task
+        loop = asyncio.get_running_loop()
+        chunk_queue: Queue[bytes] = Queue()
+        ack_queue: asyncio.Queue[int] = asyncio.Queue()
+        task = asyncio.create_task(_utils.run_blocking(blocking_writer))
+        try:
+            async for chunk in chunk_gen:
+                chunk_queue.put(chunk)
+                yield await ack_queue.get()
+        finally:
+            chunk_queue.put(b"")
+            await task
 
     async def remove(self, path: str):
         await _utils.run_blocking(self.__target.remove, path)

flatfs-0.3.0/flatfs/helpers.py

@@ -0,0 +1,449 @@
+import asyncio
+from contextlib import AbstractAsyncContextManager, AbstractContextManager
+from queue import Queue
+from types import TracebackType
+from typing import Iterator, AsyncIterator
+
+from . import _export, _utils
+from .interface import SupportsAsyncReadChunks, SupportsAsyncWriteChunks, SupportsReadChunks, SupportsWriteChunks
+
+__all__ = export = _export.Export()  # type: ignore
+
+
+@export
+def read_bytes(fs: SupportsReadChunks, path: str, chunk_size: int = 65535) -> bytes:
+    """Read whole file and return as bytes.
+
+    :param fs:
+        FlatFS object to read from.
+
+    :param path:
+        Path to a file to read.
+
+    :param chunk_size:
+        The chunk size to use during read.
+
+        This is the maximum number of bytes to read from a file per single read
+        operation.
+
+        Defaults to 64kB.
+    """
+    out = b""
+    for chunk in fs.read_chunks(path, chunk_size):
+        out += chunk
+    return out
+
+
+@export
+async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
+    """Async variant of the :func:`read_bytes` helper."""
+    out = b""
+    async for chunk in fs.read_chunks(path, chunk_size):
+        out += chunk
+    return out
+
+
+@export
+def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
+    """Write given data to file at given path either creating the file, or
+    replacing with new content.
+
+    Returns the number of bytes written.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        The path to a file to create or replace.
+
+    :param data:
+        The data to write.
+
+    :param chunk_size:
+        The chunk size to use during writing.
+
+        If *data* length is greater, it will be split into chunks and written
+        chunk by chunk, with each chunk having at most *chunk_size* bytes.
+
+        Defaults to 64kB.
+    """
+    total_bytes = 0
+    for byte_count in fs.write_chunks(path, _utils.split_into_chunks(data, chunk_size)):
+        total_bytes += byte_count
+    return total_bytes
+
+
+@export
+async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
+    """Async variant of the :func:`write_bytes` helper."""
+    total_bytes = 0
+    async for byte_count in fs.write_chunks(path, _utils.async_split_into_chunks(data, chunk_size)):
+        total_bytes += byte_count
+    return total_bytes
+
+
+@export
+def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535) -> str:
+    """Read whole text file and return as string.
+
+    Uses :func:`read_bytes` underneath and then just decodes its output using
+    provided *encoding*, which is ``utf-8`` by default.
+
+    :param fs:
+        FlatFS object to read from.
+
+    :param path:
+        The path to a file to read.
+
+    :param encoding:
+        The encoding to use.
+
+    :param chunk_size:
+        The chunk size to use during reading.
+
+        This tells how many bytes can at most be read in single pass.
+    """
+    return read_bytes(fs, path, chunk_size).decode(encoding)
+
+
+@export
+async def async_read_text(
+    fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
+) -> str:
+    """Async variant of the :func:`read_text` helper."""
+    return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
+
+
+@export
+def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535):
+    """Create new text file or replace existing with given text data.
+
+    Uses :func:`write_bytes` underneath and just encodes given *data* with
+    provided *encoding*, which is ``utf-8`` by default.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        The path to a file to create or replace.
+
+    :param encoding:
+        The encoding to use.
+
+    :param chunk_size:
+        The chunk size to use during writing.
+
+        This tells how many bytes can at most be written in a single pass.
+    """
+    return write_bytes(fs, path, data.encode(encoding), chunk_size)
+
+
+@export
+async def async_write_text(
+    fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
+):
+    """Async variant of the :func:`write_text` helper."""
+    await async_write_bytes(fs, path, data.encode(encoding), chunk_size)
+
+
+@export
+class BufferedWriter(AbstractContextManager):
+    """Buffered writer for creating new or replacing existing files.
+
+    It allows efficient writing of arbitrary binary data thanks to the buffer
+    being used. Once the length of buffered data exceeds given buffer size, the
+    data is flushed to the underlying filesystem and buffer is cleared. This
+    allows to limit actual write operations to minimum.
+
+    This helper can be used as a context manager.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        Path to a file to create or replace.
+
+    :param buffer_size:
+        The size of a buffer, in bytes.
+
+        This is the maximum amount of data that can be buffered before it is
+        flushed to the backend. This is equivalent to the maximum size of a
+        single chunk that is written to the underlying filesystem.
+    """
+
+    def __init__(self, fs: SupportsWriteChunks, path: str, buffer_size: int = 65535):
+        self.__chunk_queue: Queue[bytes] = Queue()
+        self.__buffer_size = buffer_size
+        self.__buffer = b""
+        self.__writer = fs.write_chunks(path, _utils.generate_chunks_from_queue(self.__chunk_queue))
+
+    def __enter__(self) -> "BufferedWriter":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def __write_chunk(self, chunk: bytes):
+        self.__chunk_queue.put(chunk)
+        next(self.__writer)
+
+    def flush(self):
+        """Flush the buffer and clear it.
+
+        This is automatically called on close or context manager exit.
+        """
+        if not self.__buffer:
+            return  # Nothing to do
+        self.__write_chunk(self.__buffer)
+        self.__buffer = b""
+
+    def close(self):
+        """Flush remaining data and close underlying file."""
+        self.flush()
+        return self.__writer.close()
+
+    def write(self, data: bytes) -> int:
+        """Write data to the buffer.
+
+        If the buffer is full, it will be flushed and remaining data (if any)
+        will be used to initialize emptied buffer.
+
+        :param data:
+            The data to write.
+        """
+        self.__buffer += data
+        while len(self.__buffer) > self.__buffer_size:
+            chunk = self.__buffer[: self.__buffer_size]
+            self.__write_chunk(chunk)
+            self.__buffer = self.__buffer[self.__buffer_size :]
+        return len(data)
+
+
+@export
+class AsyncBufferedWriter(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedWriter` class."""
+
+    def __init__(self, fs: SupportsAsyncWriteChunks, path: str, buffer_size: int = 65535):
+        self.__chunk_queue: asyncio.Queue[bytes] = asyncio.Queue()
+        self.__buffer_size = buffer_size
+        self.__buffer = b""
+        self.__writer = fs.write_chunks(path, _utils.async_generate_chunks_from_queue(self.__chunk_queue))
+
+    async def __aenter__(self) -> "AsyncBufferedWriter":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def __write_chunk(self, chunk: bytes):
+        self.__chunk_queue.put_nowait(chunk)
+        await anext(self.__writer)
+
+    async def write(self, data: bytes) -> int:
+        self.__buffer += data
+        while len(self.__buffer) > self.__buffer_size:
+            chunk = self.__buffer[: self.__buffer_size]
+            await self.__write_chunk(chunk)
+            self.__buffer = self.__buffer[self.__buffer_size :]
+        return len(data)
+
+    async def flush(self):
+        if not self.__buffer:
+            return  # Nothing to do
+        await self.__write_chunk(self.__buffer)
+        self.__buffer = b""
+
+    async def close(self):
+        await self.flush()
+        await self.__writer.aclose()
+
+
+@export
+class BufferedReader(AbstractContextManager):
+    """Helper for buffered reading from underlying flat filesystem.
+
+    If reads from an underlying filesystem chunks of bytes of given maximum
+    length and stores it locally for reading. When entire buffer is read, the
+    next chunk is fetched from the filesystem. The whole operation is repeated
+    until file is fully read or the reader is closed.
+
+    This helper can be used via context manager.
+
+    :param fs:
+        The filesystem to read from.
+
+    :param path:
+        The path to a file to read.
+
+    :param buffer_size:
+        The size of a read buffer, in bytes.
+
+        This is also the maximum size of a single chunk read from a file.
+    """
+
+    def __init__(self, fs: SupportsReadChunks, path: str, buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__buffer = b""
+
+    def __enter__(self) -> "BufferedReader":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def close(self):
+        self.__buffer = b""
+        self.__reader.close()
+
+    def read(self, count: int) -> bytes:
+        if not self.__buffer:
+            self.__buffer = next(self.__reader, b"")
+        data = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return data
+
+
+@export
+class AsyncBufferedReader(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedReader` class."""
+
+    def __init__(self, fs: SupportsAsyncReadChunks, path: str, buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__buffer = b""
+
+    async def __aenter__(self) -> "AsyncBufferedReader":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def close(self):
+        self.__buffer = b""
+        await self.__reader.aclose()
+
+    async def read(self, count: int) -> bytes:
+        if not self.__buffer:
+            self.__buffer = await anext(self.__reader, b"")
+        data = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return data
+
+
+@export
+class BufferedTextReader(AbstractContextManager):
+    """Equivalent of :class:`BufferedReader` for text files.
+
+    It reads at most *buffer_size* bytes from underlying file, stores in local
+    buffer, and then streams from that buffer until it expires. Once it is
+    expired and more data is needed, the next chunk of data is automatically
+    read and stored in the buffer.
+
+    :param fs:
+        The filesystem reader object.
+
+    :param path:
+        The path to a text file to read.
+
+    :param encoding:
+        The encoding to use.
+
+    :param buffer_size:
+        The maximum amount of data to read from an underlying file in one pass,
+        in bytes.
+    """
+
+    def __init__(self, fs: SupportsReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__encoding = encoding
+        self.__buffer = ""
+
+    def __read_next_chunk(self) -> str:
+        return next(self.__reader, b"").decode(self.__encoding)
+
+    def __enter__(self) -> "BufferedTextReader":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def __iter__(self) -> Iterator[str]:
+        while next_chunk := self.__read_next_chunk():
+            self.__buffer += next_chunk
+            lines = self.__buffer.splitlines()
+            if len(lines) > 1:
+                for line in lines[:-1]:
+                    yield line
+                self.__buffer = lines[-1]
+        yield self.__buffer
+
+    def read(self, count: int) -> str:
+        """Read at most *count* characters.
+
+        :param count:
+            The maximum number of characters to read.
+        """
+        if not self.__buffer:
+            self.__buffer = self.__read_next_chunk()
+        out = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return out
+
+    def close(self):
+        """Close this reader.
+
+        This also closes the underlying file.
+        """
+        self.__buffer = ""
+        self.__reader.close()
+
+
+@export
+class AsyncBufferedTextReader(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedTextReader` class."""
+
+    def __init__(self, fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__encoding = encoding
+        self.__buffer = ""
+
+    async def __read_next_chunk(self) -> str:
+        return (await anext(self.__reader, b"")).decode(self.__encoding)
+
+    async def __aenter__(self) -> "AsyncBufferedTextReader":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def __aiter__(self) -> AsyncIterator[str]:
+        while next_chunk := await self.__read_next_chunk():
+            self.__buffer += next_chunk
+            lines = self.__buffer.splitlines()
+            if len(lines) > 1:
+                for line in lines[:-1]:
+                    yield line
+                self.__buffer = lines[-1]
+        yield self.__buffer
+
+    async def read(self, count: int) -> str:
+        if not self.__buffer:
+            self.__buffer = await self.__read_next_chunk()
+        out = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return out
+
+    async def close(self):
+        self.__buffer = ""
+        await self.__reader.aclose()

{flatfs-0.2.0 → flatfs-0.3.0}/flatfs/interface.py

@@ -1,6 +1,6 @@
 import dataclasses
 import datetime
-from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Iterator, Protocol
+from typing import AsyncGenerator, AsyncIterator, Generator, Iterator, Protocol
 
 from . import _export
 
@@ -24,14 +24,15 @@ class SupportsReadChunks(Protocol):
     """Protocol for objects that support chunked reading."""
 
     def read_chunks(self, path: str, chunk_size: int = 65535) -> Generator[bytes, None, None]:
-        """Read file in chunks and return iterator yielding chunks.
+        """Return generator for reading existing file in chunks of given
+        maximum size.
 
-        This method can be safely used with large files without the risk of
-        getting out of memory.
-
-        If *path* does not point to an existing file,
+        If *path* does not point to an existing file
         :exc:`flatfs.exc.PathNotFoundError` will be raised.
 
+        Closing or expiring the returned generator automatically closes
+        underlying file.
+
         :param path:
             The path to a file to read.
 
@@ -47,18 +48,21 @@ class SupportsReadChunks(Protocol):
 class SupportsWriteChunks(Protocol):
     """Protocol for object that support writing in chunks."""
 
-    def write_chunks(self, path: str, chunks: Iterable[bytes]) -> int:
-        """Create or overwrite file at given *path* and write chunks of data
-        generated by provided *chunks* iterable.
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
+        """Create or replace file at given path.
+
+        Returns a generator that consumes a chunk from provided *chunk_gen*,
+        writes it to the file, and yields number of bytes written for the
+        current chunk.
 
-        Returns the total number of bytes written, which is equal to the size
-        of a resulting file.
+        Closing or expiring returned generator automatically closes underlying
+        file and flushes what was written so far.
 
         :param path:
-            The path to a file to create or modify.
+            The path to a file.
 
-        :param chunks:
-            Iterable providing chunks of bytes to write to a file.
+        :param chunk_gen:
+            Generator producing chunks to be written.
         """
         ...
 
@@ -68,15 +72,13 @@ class FlatFsReader(SupportsReadChunks, Protocol):
     """Read-only interface to access files in the flat filesystem."""
 
     def scan(self) -> Iterator[str]:
-        """Scan filesystem and generate paths to existing files.
-
-        Each yielded path can later be used with other methods of this
-        interface.
-        """
+        """Return iterator that yields normalized paths to existing files."""
         ...
 
     def stat(self, path: str) -> Stat:
-        """Get stats of a file at given *path*.
+        """Get stats of a file at given path.
+
+        This can be used to retrieve file size, file modification time etc.
 
         If the file does not exist, :exc:`flatfs.exc.PathNotFoundError` will be
         raised.
@@ -87,7 +89,7 @@ class FlatFsReader(SupportsReadChunks, Protocol):
         ...
 
     def exists(self, path: str) -> bool:
-        """Check if *path* points to an existing file.
+        """Check if path points to an existing file.
 
         :param path:
             The path to a file.
@@ -100,10 +102,10 @@ class FlatFsReaderWriter(FlatFsReader, SupportsWriteChunks, Protocol):
     """Read-write interface to access files in the flat filesystem."""
 
     def remove(self, path: str):
-        """Remove file at given *path*.
+        """Remove existing file at given path.
 
         Raises :exc:`flatfs.exc.PathNotFoundError` if there was no file to
-        remove at given *path*.
+        remove at given path.
 
         :param path:
             The path to a file to remove.
@@ -113,61 +115,57 @@ class FlatFsReaderWriter(FlatFsReader, SupportsWriteChunks, Protocol):
 
 @export
 class SupportsAsyncReadChunks(Protocol):
+    """Async variant of the :class:`SupportsReadChunks` protocol."""
+
     def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
-        """Read file in chunks and return iterator yielding chunks.
+        """Return a generator for reading file in chunks of given maximum size.
 
-        This method can be safely used with large files without the risk of
-        getting out of memory.
-
-        If *path* does not point to an existing file,
-        :exc:`flatfs.exc.PathNotFoundError` will be raised.
+        This is the async version of :meth:`SupportsReadChunks.read_chunks`
+        method.
 
         :param path:
             The path to a file to read.
 
         :param chunk_size:
-            The maximum chunk size (in bytes).
-
-            This is the maximum length of a single generated bytes object.
+            The maximum chunk size, in bytes.
         """
         ...
 
 
 @export
 class SupportsAsyncWriteChunks(Protocol):
-    async def write_chunks(self, path: str, chunks: AsyncIterable[bytes]) -> int:
-        """Create or overwrite file at given *path* and write chunks of data
-        generated by provided *chunks* iterable.
+    """Async variant of the :class:`SupportsWriteChunks` protocol."""
+
+    def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
+        """Generator for creating or overwriting files.
 
-        Returns the total number of bytes written, which is equal to the size
-        of a resulting file.
+        This is the async version of :meth:`SupportsWriteChunks.write_chunks`
+        method.
 
         :param path:
-            The path to a file to create or modify.
+            The path to a file to create or replace.
 
-        :param chunks:
-            Iterable providing chunks of bytes to write to a file.
+        :param chunk_gen:
+            Async generator yielding file chunks.
         """
         ...
 
 
 @export
 class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
-    """Async read-only interface to access files in the flat filesystem."""
+    """Async variant of the :class:`FlatFsReader` protocol."""
 
-    def scan(self) -> AsyncGenerator[str, None]:
-        """Scan filesystem and generate paths to existing files.
+    def scan(self) -> AsyncIterator[str]:
+        """Return iterator that yields normalized paths to existing files.
 
-        Each yielded path can later be used with other methods of this
-        interface.
+        This is the async version of :meth:`FlatFsReader.scan` method.
         """
         ...
 
     async def stat(self, path: str) -> Stat:
-        """Get stats of a file at given *path*.
+        """Get stats of a file at given path.
 
-        If the file does not exist, :exc:`flatfs.exc.PathNotFoundError` will be
-        raised.
+        This is the async version of :meth:`FlatFsReader.stat` method.
 
         :param path:
             The path to a file.
@@ -175,19 +173,24 @@ class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
         ...
 
     async def exists(self, path: str) -> bool:
-        """Check if *path* points to an existing file."""
+        """Check if file at given path exists.
+
+        This is the async version of :meth:`FlatFsReader.exists` method.
+
+        :param path:
+            The path to check.
+        """
         ...
 
 
 @export
 class AsyncFlatFsReaderWriter(AsyncFlatFsReader, SupportsAsyncWriteChunks, Protocol):
-    """Async read-write interface to access files in the flat filesystem."""
+    """Async variant of the :class:`FlatFsReaderWriter` protocol."""
 
     async def remove(self, path: str):
-        """Remove file at given *path*.
+        """Remove existing file.
 
-        Raises :exc:`flatfs.exc.PathNotFoundError` if there was no file to
-        remove at given *path*.
+        This is the async version of :meth:`FlatFsReaderWriter.remove` method.
 
         :param path:
             The path to a file to remove.

{flatfs-0.2.0 → flatfs-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flatfs"
-version = "0.2.0"
+version = "0.3.0"
 description = "Minimal filesystem abstraction with pluggable backends, treating paths as keys and file contents as values."
 authors = [
     {name = "Maciej Wiatrzyk", email = "maciej.wiatrzyk@gmail.com"}

flatfs-0.2.0/flatfs/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"

flatfs-0.2.0/flatfs/helpers.py

@@ -1,158 +0,0 @@
-from . import _export
-from .interface import SupportsAsyncReadChunks, SupportsAsyncWriteChunks, SupportsReadChunks, SupportsWriteChunks
-
-__all__ = export = _export.Export()  # type: ignore
-
-
-@export
-def read_bytes(fs: SupportsReadChunks, path: str, chunk_size: int = 65535) -> bytes:
-    """Read whole file and return as bytes.
-
-    :param fs:
-        FlatFS object to read from.
-
-    :param path:
-        Path to a file to read.
-
-    :param chunk_size:
-        The chunk size to use during read.
-
-        This is the maximum number of bytes to read from a file per single read
-        operation.
-
-        Defaults to 64kB.
-    """
-    out = b""
-    for chunk in fs.read_chunks(path, chunk_size):
-        out += chunk
-    return out
-
-
-@export
-def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
-    """Write given data to file at given path either creating the file, or
-    replacing with new content.
-
-    Returns the number of bytes written.
-
-    :param fs:
-        FlatFS object to write to.
-
-    :param path:
-        The path to a file to create or replace.
-
-    :param data:
-        The data to write.
-
-    :param chunk_size:
-        The chunk size to use during writing.
-
-        If *data* length is greater, it will be split into chunks and written
-        chunk by chunk, with each chunk having at most *chunk_size* bytes.
-
-        Defaults to 64kB.
-    """
-
-    def gen():
-        data_left = data
-        while True:
-            chunk = data_left[:chunk_size]
-            if not chunk:
-                break
-            data_left = data_left[chunk_size:]
-            yield chunk
-
-    return fs.write_chunks(path, gen())
-
-
-@export
-def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535) -> str:
-    """Read whole text file and return as string.
-
-    Uses :func:`read_bytes` underneath and then just decodes its output using
-    provided *encoding*, which is ``utf-8`` by default.
-    """
-    return read_bytes(fs, path, chunk_size).decode(encoding)
-
-
-@export
-def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535):
-    """Create new text file or replace existing with given text data.
-
-    Uses :func:`write_bytes` underneath and just encodes given *data* with
-    provided *encoding*, which is ``utf-8`` by default.
-    """
-    return write_bytes(fs, path, data.encode(encoding), chunk_size)
-
-
-@export
-async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
-    """Same as :func:`read_bytes`, but for async code."""
-    out = b""
-    async for chunk in fs.read_chunks(path, chunk_size):
-        out += chunk
-    return out
-
-
-@export
-async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
-    """Same as :func:`write_bytes`, but for async code."""
-
-    async def gen():
-        data_left = data
-        while True:
-            chunk = data_left[:chunk_size]
-            if not chunk:
-                break
-            data_left = data_left[chunk_size:]
-            yield chunk
-
-    return await fs.write_chunks(path, gen())
-
-
-@export
-async def async_read_text(
-    fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
-) -> str:
-    """Same as :func:`read_text`, but for async code."""
-    return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
-
-
-@export
-async def async_write_text(
-    fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
-):
-    """Same as :func:`write_text`, but for async code."""
-    await async_write_bytes(fs, path, data.encode(encoding), chunk_size)
-
-
-@export
-class BinaryReader:
-    """Helper for reading bytes from a file."""
-
-    def __init__(self, fs: SupportsReadChunks, path: str, chunk_size: int = 65535):
-        self.__chunk_gen = fs.read_chunks(path, chunk_size)
-        self.__current_chunk = b""
-
-    def read(self, count: int) -> bytes:
-        """Read at most *count* bytes from an open file.
-
-        :param count:
-            The maximum number of bytes to read.
-        """
-        if not self.__current_chunk:
-            self.__current_chunk = next(self.__chunk_gen, b"")
-        fragment = self.__current_chunk[:count]
-        self.__current_chunk = self.__current_chunk[count:]
-        return fragment
-
-    def close(self):
-        """Close this reader.
-
-        This also closes the underlying file that is opened by flat filesystem
-        object given in the constructor.
-
-        This method is idempotent.
-        """
-        self.__current_chunk = b""
-        self.__chunk_gen.close()