flatfs 0.0.0__tar.gz → 0.3.0__tar.gz

{flatfs-0.0.0 → flatfs-0.3.0}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: flatfs
-Version: 0.0.0
+Version: 0.3.0
 Summary: Minimal filesystem abstraction with pluggable backends, treating paths as keys and file contents as values.
 License: MIT
+License-File: LICENSE.txt
 Keywords: flat,filesystem,abstraction,library
 Author: Maciej Wiatrzyk
 Author-email: maciej.wiatrzyk@gmail.com

flatfs-0.3.0/flatfs/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

flatfs-0.3.0/flatfs/_utils.py

@@ -0,0 +1,110 @@
+import asyncio
+import datetime
+import functools
+import queue
+from typing import AsyncGenerator, Callable, Generator, TypeVar
+
+from flatfs.exc import PathAccessError
+
+T = TypeVar("T")
+
+
+def normalize_path(path: str) -> str:
+    """Normalizes path by removing duplicated slash characters and by
+    evaluating relative path components (e.g. ``../`` or ``./``).
+
+    Returns normalized path, or raises :exc:`flatfs.exc.FileAccessError` if
+    final path would go beyond filesystem bounds.
+
+    :param path:
+        The path to normalize.
+    """
+    parts: list[str] = []
+    for part in path.split("/"):
+        if not part or part == ".":
+            continue
+        if part == "..":
+            if not parts:
+                raise PathAccessError(path)
+            parts.pop()
+        else:
+            parts.append(part)
+    out = "/".join(parts)
+    if out[0] == "/":
+        return out
+    return "/" + out
+
+
+async def run_blocking(func: Callable[..., T], *args, **kwargs) -> T:
+    """Run blocking (i.e. non-async) task in an event loop's default
+    executor.
+
+    On success, return *func*'s result, on failure raise *func*'s exception.
+
+    :param func:
+        The non-async callable to run.
+
+    :param `*args`:
+        Positional args to be passed to *func*.
+
+    :param `**kwargs`:
+        Named args to be passed to *func*.
+    """
+    wrapped_func = functools.partial(func, *args, **kwargs)
+    return await asyncio.get_running_loop().run_in_executor(None, wrapped_func)
+
+
+def utcnow() -> datetime.datetime:
+    """Get the current date and time in UTC."""
+    return datetime.datetime.now(datetime.timezone.utc)
+
+
+def split_into_chunks(data: bytes, chunk_size: int) -> Generator[bytes, None, None]:
+    """Generator that splits given binary data into chunks of provided maximum size.
+
+    :param data:
+        The data to split.
+
+    :chunk_size:
+        The maximum size of a single chunk.
+    """
+    while chunk := data[:chunk_size]:
+        yield chunk
+        data = data[chunk_size:]
+
+
+async def async_split_into_chunks(data: bytes, chunk_size: int) -> AsyncGenerator[bytes, None]:
+    """Async variant of the :func:`split_into_chunks` function.
+
+    :param data:
+        The data to split.
+
+    :param chunk_size:
+        The maximum size of a single chunk.
+    """
+    while chunk := data[:chunk_size]:
+        yield chunk
+        data = data[chunk_size:]
+
+
+def generate_chunks_from_queue(chunk_queue: queue.Queue[bytes]) -> Generator[bytes, None, None]:
+    """Generator that yields chunks it receives via provided queue until empty
+    bytes object is received.
+
+    :param chunk_queue:
+        Chunk queue to adapt.
+    """
+    while True:
+        chunk = chunk_queue.get()
+        if not chunk:
+            return
+        yield chunk
+
+
+async def async_generate_chunks_from_queue(chunk_queue: asyncio.Queue[bytes]) -> AsyncGenerator[bytes, None]:
+    """Async variant of the :func:`generate_chunks_from_queue` function."""
+    while True:
+        chunk = await chunk_queue.get()
+        if not chunk:
+            return
+        yield chunk

{flatfs-0.0.0 → flatfs-0.3.0}/flatfs/backends.py

@@ -1,12 +1,14 @@
 import asyncio
+import dataclasses
+import datetime
 import fnmatch
 import os
 import pathlib
 from queue import Queue
-from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Iterator, Optional
+from typing import AsyncGenerator, AsyncIterator, Generator, Iterator, Optional
 
 from flatfs.exc import PathAccessError, PathNotFoundError
-from flatfs.interface import FlatFsReaderWriter
+from flatfs.interface import FlatFsReaderWriter, Stat
 
 from . import _utils, _export
 
@@ -80,11 +82,21 @@ class LocalFlatFs:
     def scan(self) -> Iterator[str]:
         return self.__walk(self.__root_dir, "")
 
+    def stat(self, path: str) -> Stat:
+        abspath = self.__make_abspath(path)
+        if not abspath.is_file():
+            raise PathNotFoundError(path)
+        stat_result = abspath.stat()
+        return Stat(
+            modified=datetime.datetime.fromtimestamp(stat_result.st_mtime, datetime.timezone.utc),
+            size=stat_result.st_size,
+        )
+
     def exists(self, path: str) -> bool:
         abspath = self.__make_abspath(path)
         return abspath.is_file()
 
-    def read_chunks(self, path: str, chunk_size: int = 65535) -> Iterator[bytes]:
+    def read_chunks(self, path: str, chunk_size: int = 65535) -> Generator[bytes, None, None]:
         abspath = self.__make_abspath(path)
         if not abspath.is_file():
             raise PathNotFoundError(path)
@@ -95,12 +107,12 @@ class LocalFlatFs:
                     break
                 yield chunk
 
-    def write_chunks(self, path: str, chunks: Iterable[bytes]):
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
         abspath = self.__make_abspath(path)
         abspath.parent.mkdir(parents=True, exist_ok=True)
         with abspath.open("wb") as fd:
-            for chunk in chunks:
-                fd.write(chunk)
+            for chunk in chunk_gen:
+                yield fd.write(chunk)
 
     def remove(self, path: str):
         abspath = self.__make_abspath(path)
@@ -117,8 +129,13 @@ class InMemoryFlatFs:
     disk usage.
     """
 
+    @dataclasses.dataclass
+    class _File:
+        payload: bytes
+        stat: Stat
+
     def __init__(self) -> None:
-        self.__storage: dict[str, bytes] = {}
+        self.__storage: dict[str, InMemoryFlatFs._File] = {}
 
     def __make_key(self, path: str) -> str:
         normalized_key = _utils.normalize_path(path)
@@ -127,21 +144,37 @@ class InMemoryFlatFs:
     def scan(self) -> Iterator[str]:
         return iter(self.__storage.keys())
 
+    def stat(self, path: str) -> Stat:
+        key = self.__make_key(path)
+        if key not in self.__storage:
+            raise PathNotFoundError(path)
+        return self.__storage[key].stat
+
     def exists(self, path: str) -> bool:
         return self.__make_key(path) in self.__storage
 
-    def read_chunks(self, path: str, chunk_size: int = 65535) -> Iterator[bytes]:
+    def read_chunks(self, path: str, chunk_size: int = 65535) -> Generator[bytes, None, None]:
         key = self.__make_key(path)
         if key not in self.__storage:
             raise PathNotFoundError(path)
-        data_left = self.__storage[key]
+        data_left = self.__storage[key].payload
         while len(data_left) > 0:
             chunk = data_left[:chunk_size]
             data_left = data_left[chunk_size:]
             yield chunk
 
-    def write_chunks(self, path: str, chunks: Iterable[bytes]):
-        self.__storage[self.__make_key(path)] = b"".join(chunks)
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
+        now = _utils.utcnow()
+        stat = Stat(
+            modified=now,
+            size=0,
+        )
+        self.__storage[self.__make_key(path)] = file = self._File(b"", stat)
+        for chunk in chunk_gen:
+            file.payload += chunk
+            write_count = len(chunk)
+            file.stat.size += write_count
+            yield write_count
 
     def remove(self, path: str):
         key = self.__make_key(path)
@@ -186,7 +219,10 @@ class AsyncFlatFsAdapter:
     async def exists(self, path: str) -> bool:
         return await _utils.run_blocking(self.__target.exists, path)
 
-    async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
+    async def stat(self, path: str) -> Stat:
+        return await _utils.run_blocking(self.__target.stat, path)
+
+    async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncIterator[bytes]:
 
         def reader():
             try:
@@ -207,21 +243,23 @@ class AsyncFlatFsAdapter:
                 raise chunk_or_exc
             yield chunk_or_exc
 
-    async def write_chunks(self, path: str, chunks: AsyncIterable[bytes]):
+    async def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
 
-        def gen() -> Generator[bytes, None, None]:
-            while True:
-                chunk = queue.get()
-                if not chunk:
-                    return
-                yield chunk
+        def blocking_writer():
+            for write_count in self.__target.write_chunks(path, _utils.generate_chunks_from_queue(chunk_queue)):
+                loop.call_soon_threadsafe(ack_queue.put_nowait, write_count)
 
-        queue: Queue[bytes] = Queue()
-        task = asyncio.create_task(_utils.run_blocking(self.__target.write_chunks, path, gen()))
-        async for chunk in chunks:
-            queue.put(chunk)
-        queue.put(b"")
-        await task
+        loop = asyncio.get_running_loop()
+        chunk_queue: Queue[bytes] = Queue()
+        ack_queue: asyncio.Queue[int] = asyncio.Queue()
+        task = asyncio.create_task(_utils.run_blocking(blocking_writer))
+        try:
+            async for chunk in chunk_gen:
+                chunk_queue.put(chunk)
+                yield await ack_queue.get()
+        finally:
+            chunk_queue.put(b"")
+            await task
 
     async def remove(self, path: str):
         await _utils.run_blocking(self.__target.remove, path)

flatfs-0.3.0/flatfs/helpers.py

@@ -0,0 +1,449 @@
+import asyncio
+from contextlib import AbstractAsyncContextManager, AbstractContextManager
+from queue import Queue
+from types import TracebackType
+from typing import Iterator, AsyncIterator
+
+from . import _export, _utils
+from .interface import SupportsAsyncReadChunks, SupportsAsyncWriteChunks, SupportsReadChunks, SupportsWriteChunks
+
+__all__ = export = _export.Export()  # type: ignore
+
+
+@export
+def read_bytes(fs: SupportsReadChunks, path: str, chunk_size: int = 65535) -> bytes:
+    """Read whole file and return as bytes.
+
+    :param fs:
+        FlatFS object to read from.
+
+    :param path:
+        Path to a file to read.
+
+    :param chunk_size:
+        The chunk size to use during read.
+
+        This is the maximum number of bytes to read from a file per single read
+        operation.
+
+        Defaults to 64kB.
+    """
+    out = b""
+    for chunk in fs.read_chunks(path, chunk_size):
+        out += chunk
+    return out
+
+
+@export
+async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
+    """Async variant of the :func:`read_bytes` helper."""
+    out = b""
+    async for chunk in fs.read_chunks(path, chunk_size):
+        out += chunk
+    return out
+
+
+@export
+def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
+    """Write given data to file at given path either creating the file, or
+    replacing with new content.
+
+    Returns the number of bytes written.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        The path to a file to create or replace.
+
+    :param data:
+        The data to write.
+
+    :param chunk_size:
+        The chunk size to use during writing.
+
+        If *data* length is greater, it will be split into chunks and written
+        chunk by chunk, with each chunk having at most *chunk_size* bytes.
+
+        Defaults to 64kB.
+    """
+    total_bytes = 0
+    for byte_count in fs.write_chunks(path, _utils.split_into_chunks(data, chunk_size)):
+        total_bytes += byte_count
+    return total_bytes
+
+
+@export
+async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
+    """Async variant of the :func:`write_bytes` helper."""
+    total_bytes = 0
+    async for byte_count in fs.write_chunks(path, _utils.async_split_into_chunks(data, chunk_size)):
+        total_bytes += byte_count
+    return total_bytes
+
+
+@export
+def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535) -> str:
+    """Read whole text file and return as string.
+
+    Uses :func:`read_bytes` underneath and then just decodes its output using
+    provided *encoding*, which is ``utf-8`` by default.
+
+    :param fs:
+        FlatFS object to read from.
+
+    :param path:
+        The path to a file to read.
+
+    :param encoding:
+        The encoding to use.
+
+    :param chunk_size:
+        The chunk size to use during reading.
+
+        This tells how many bytes can at most be read in single pass.
+    """
+    return read_bytes(fs, path, chunk_size).decode(encoding)
+
+
+@export
+async def async_read_text(
+    fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
+) -> str:
+    """Async variant of the :func:`read_text` helper."""
+    return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
+
+
+@export
+def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535):
+    """Create new text file or replace existing with given text data.
+
+    Uses :func:`write_bytes` underneath and just encodes given *data* with
+    provided *encoding*, which is ``utf-8`` by default.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        The path to a file to create or replace.
+
+    :param encoding:
+        The encoding to use.
+
+    :param chunk_size:
+        The chunk size to use during writing.
+
+        This tells how many bytes can at most be written in a single pass.
+    """
+    return write_bytes(fs, path, data.encode(encoding), chunk_size)
+
+
+@export
+async def async_write_text(
+    fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
+):
+    """Async variant of the :func:`write_text` helper."""
+    await async_write_bytes(fs, path, data.encode(encoding), chunk_size)
+
+
+@export
+class BufferedWriter(AbstractContextManager):
+    """Buffered writer for creating new or replacing existing files.
+
+    It allows efficient writing of arbitrary binary data thanks to the buffer
+    being used. Once the length of buffered data exceeds given buffer size, the
+    data is flushed to the underlying filesystem and buffer is cleared. This
+    allows to limit actual write operations to minimum.
+
+    This helper can be used as a context manager.
+
+    :param fs:
+        FlatFS object to write to.
+
+    :param path:
+        Path to a file to create or replace.
+
+    :param buffer_size:
+        The size of a buffer, in bytes.
+
+        This is the maximum amount of data that can be buffered before it is
+        flushed to the backend. This is equivalent to the maximum size of a
+        single chunk that is written to the underlying filesystem.
+    """
+
+    def __init__(self, fs: SupportsWriteChunks, path: str, buffer_size: int = 65535):
+        self.__chunk_queue: Queue[bytes] = Queue()
+        self.__buffer_size = buffer_size
+        self.__buffer = b""
+        self.__writer = fs.write_chunks(path, _utils.generate_chunks_from_queue(self.__chunk_queue))
+
+    def __enter__(self) -> "BufferedWriter":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def __write_chunk(self, chunk: bytes):
+        self.__chunk_queue.put(chunk)
+        next(self.__writer)
+
+    def flush(self):
+        """Flush the buffer and clear it.
+
+        This is automatically called on close or context manager exit.
+        """
+        if not self.__buffer:
+            return  # Nothing to do
+        self.__write_chunk(self.__buffer)
+        self.__buffer = b""
+
+    def close(self):
+        """Flush remaining data and close underlying file."""
+        self.flush()
+        return self.__writer.close()
+
+    def write(self, data: bytes) -> int:
+        """Write data to the buffer.
+
+        If the buffer is full, it will be flushed and remaining data (if any)
+        will be used to initialize emptied buffer.
+
+        :param data:
+            The data to write.
+        """
+        self.__buffer += data
+        while len(self.__buffer) > self.__buffer_size:
+            chunk = self.__buffer[: self.__buffer_size]
+            self.__write_chunk(chunk)
+            self.__buffer = self.__buffer[self.__buffer_size :]
+        return len(data)
+
+
+@export
+class AsyncBufferedWriter(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedWriter` class."""
+
+    def __init__(self, fs: SupportsAsyncWriteChunks, path: str, buffer_size: int = 65535):
+        self.__chunk_queue: asyncio.Queue[bytes] = asyncio.Queue()
+        self.__buffer_size = buffer_size
+        self.__buffer = b""
+        self.__writer = fs.write_chunks(path, _utils.async_generate_chunks_from_queue(self.__chunk_queue))
+
+    async def __aenter__(self) -> "AsyncBufferedWriter":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def __write_chunk(self, chunk: bytes):
+        self.__chunk_queue.put_nowait(chunk)
+        await anext(self.__writer)
+
+    async def write(self, data: bytes) -> int:
+        self.__buffer += data
+        while len(self.__buffer) > self.__buffer_size:
+            chunk = self.__buffer[: self.__buffer_size]
+            await self.__write_chunk(chunk)
+            self.__buffer = self.__buffer[self.__buffer_size :]
+        return len(data)
+
+    async def flush(self):
+        if not self.__buffer:
+            return  # Nothing to do
+        await self.__write_chunk(self.__buffer)
+        self.__buffer = b""
+
+    async def close(self):
+        await self.flush()
+        await self.__writer.aclose()
+
+
+@export
+class BufferedReader(AbstractContextManager):
+    """Helper for buffered reading from underlying flat filesystem.
+
+    If reads from an underlying filesystem chunks of bytes of given maximum
+    length and stores it locally for reading. When entire buffer is read, the
+    next chunk is fetched from the filesystem. The whole operation is repeated
+    until file is fully read or the reader is closed.
+
+    This helper can be used via context manager.
+
+    :param fs:
+        The filesystem to read from.
+
+    :param path:
+        The path to a file to read.
+
+    :param buffer_size:
+        The size of a read buffer, in bytes.
+
+        This is also the maximum size of a single chunk read from a file.
+    """
+
+    def __init__(self, fs: SupportsReadChunks, path: str, buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__buffer = b""
+
+    def __enter__(self) -> "BufferedReader":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def close(self):
+        self.__buffer = b""
+        self.__reader.close()
+
+    def read(self, count: int) -> bytes:
+        if not self.__buffer:
+            self.__buffer = next(self.__reader, b"")
+        data = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return data
+
+
+@export
+class AsyncBufferedReader(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedReader` class."""
+
+    def __init__(self, fs: SupportsAsyncReadChunks, path: str, buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__buffer = b""
+
+    async def __aenter__(self) -> "AsyncBufferedReader":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def close(self):
+        self.__buffer = b""
+        await self.__reader.aclose()
+
+    async def read(self, count: int) -> bytes:
+        if not self.__buffer:
+            self.__buffer = await anext(self.__reader, b"")
+        data = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return data
+
+
+@export
+class BufferedTextReader(AbstractContextManager):
+    """Equivalent of :class:`BufferedReader` for text files.
+
+    It reads at most *buffer_size* bytes from underlying file, stores in local
+    buffer, and then streams from that buffer until it expires. Once it is
+    expired and more data is needed, the next chunk of data is automatically
+    read and stored in the buffer.
+
+    :param fs:
+        The filesystem reader object.
+
+    :param path:
+        The path to a text file to read.
+
+    :param encoding:
+        The encoding to use.
+
+    :param buffer_size:
+        The maximum amount of data to read from an underlying file in one pass,
+        in bytes.
+    """
+
+    def __init__(self, fs: SupportsReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__encoding = encoding
+        self.__buffer = ""
+
+    def __read_next_chunk(self) -> str:
+        return next(self.__reader, b"").decode(self.__encoding)
+
+    def __enter__(self) -> "BufferedTextReader":
+        return self
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return self.close()
+
+    def __iter__(self) -> Iterator[str]:
+        while next_chunk := self.__read_next_chunk():
+            self.__buffer += next_chunk
+            lines = self.__buffer.splitlines()
+            if len(lines) > 1:
+                for line in lines[:-1]:
+                    yield line
+                self.__buffer = lines[-1]
+        yield self.__buffer
+
+    def read(self, count: int) -> str:
+        """Read at most *count* characters.
+
+        :param count:
+            The maximum number of characters to read.
+        """
+        if not self.__buffer:
+            self.__buffer = self.__read_next_chunk()
+        out = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return out
+
+    def close(self):
+        """Close this reader.
+
+        This also closes the underlying file.
+        """
+        self.__buffer = ""
+        self.__reader.close()
+
+
+@export
+class AsyncBufferedTextReader(AbstractAsyncContextManager):
+    """Async variant of the :class:`BufferedTextReader` class."""
+
+    def __init__(self, fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
+        self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
+        self.__encoding = encoding
+        self.__buffer = ""
+
+    async def __read_next_chunk(self) -> str:
+        return (await anext(self.__reader, b"")).decode(self.__encoding)
+
+    async def __aenter__(self) -> "AsyncBufferedTextReader":
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None
+    ) -> bool | None:
+        return await self.close()
+
+    async def __aiter__(self) -> AsyncIterator[str]:
+        while next_chunk := await self.__read_next_chunk():
+            self.__buffer += next_chunk
+            lines = self.__buffer.splitlines()
+            if len(lines) > 1:
+                for line in lines[:-1]:
+                    yield line
+                self.__buffer = lines[-1]
+        yield self.__buffer
+
+    async def read(self, count: int) -> str:
+        if not self.__buffer:
+            self.__buffer = await self.__read_next_chunk()
+        out = self.__buffer[:count]
+        self.__buffer = self.__buffer[count:]
+        return out
+
+    async def close(self):
+        self.__buffer = ""
+        await self.__reader.aclose()

flatfs-0.3.0/flatfs/interface.py

@@ -0,0 +1,198 @@
+import dataclasses
+import datetime
+from typing import AsyncGenerator, AsyncIterator, Generator, Iterator, Protocol
+
+from . import _export
+
+__all__ = export = _export.Export()  # type: ignore
+
+
+@export
+@dataclasses.dataclass
+class Stat:
+    """Object containing file stats."""
+
+    #: The file last modification date and time (in UTC).
+    modified: datetime.datetime
+
+    #: The file size in bytes.
+    size: int
+
+
+@export
+class SupportsReadChunks(Protocol):
+    """Protocol for objects that support chunked reading."""
+
+    def read_chunks(self, path: str, chunk_size: int = 65535) -> Generator[bytes, None, None]:
+        """Return generator for reading existing file in chunks of given
+        maximum size.
+
+        If *path* does not point to an existing file
+        :exc:`flatfs.exc.PathNotFoundError` will be raised.
+
+        Closing or expiring the returned generator automatically closes
+        underlying file.
+
+        :param path:
+            The path to a file to read.
+
+        :param chunk_size:
+            The maximum chunk size (in bytes).
+
+            This is the maximum length of a single generated bytes object.
+        """
+        ...
+
+
+@export
+class SupportsWriteChunks(Protocol):
+    """Protocol for object that support writing in chunks."""
+
+    def write_chunks(self, path: str, chunk_gen: Generator[bytes, None, None]) -> Generator[int, None, None]:
+        """Create or replace file at given path.
+
+        Returns a generator that consumes a chunk from provided *chunk_gen*,
+        writes it to the file, and yields number of bytes written for the
+        current chunk.
+
+        Closing or expiring returned generator automatically closes underlying
+        file and flushes what was written so far.
+
+        :param path:
+            The path to a file.
+
+        :param chunk_gen:
+            Generator producing chunks to be written.
+        """
+        ...
+
+
+@export
+class FlatFsReader(SupportsReadChunks, Protocol):
+    """Read-only interface to access files in the flat filesystem."""
+
+    def scan(self) -> Iterator[str]:
+        """Return iterator that yields normalized paths to existing files."""
+        ...
+
+    def stat(self, path: str) -> Stat:
+        """Get stats of a file at given path.
+
+        This can be used to retrieve file size, file modification time etc.
+
+        If the file does not exist, :exc:`flatfs.exc.PathNotFoundError` will be
+        raised.
+
+        :param path:
+            The path to a file.
+        """
+        ...
+
+    def exists(self, path: str) -> bool:
+        """Check if path points to an existing file.
+
+        :param path:
+            The path to a file.
+        """
+        ...
+
+
+@export
+class FlatFsReaderWriter(FlatFsReader, SupportsWriteChunks, Protocol):
+    """Read-write interface to access files in the flat filesystem."""
+
+    def remove(self, path: str):
+        """Remove existing file at given path.
+
+        Raises :exc:`flatfs.exc.PathNotFoundError` if there was no file to
+        remove at given path.
+
+        :param path:
+            The path to a file to remove.
+        """
+        ...
+
+
+@export
+class SupportsAsyncReadChunks(Protocol):
+    """Async variant of the :class:`SupportsReadChunks` protocol."""
+
+    def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
+        """Return a generator for reading file in chunks of given maximum size.
+
+        This is the async version of :meth:`SupportsReadChunks.read_chunks`
+        method.
+
+        :param path:
+            The path to a file to read.
+
+        :param chunk_size:
+            The maximum chunk size, in bytes.
+        """
+        ...
+
+
+@export
+class SupportsAsyncWriteChunks(Protocol):
+    """Async variant of the :class:`SupportsWriteChunks` protocol."""
+
+    def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
+        """Generator for creating or overwriting files.
+
+        This is the async version of :meth:`SupportsWriteChunks.write_chunks`
+        method.
+
+        :param path:
+            The path to a file to create or replace.
+
+        :param chunk_gen:
+            Async generator yielding file chunks.
+        """
+        ...
+
+
+@export
+class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
+    """Async variant of the :class:`FlatFsReader` protocol."""
+
+    def scan(self) -> AsyncIterator[str]:
+        """Return iterator that yields normalized paths to existing files.
+
+        This is the async version of :meth:`FlatFsReader.scan` method.
+        """
+        ...
+
+    async def stat(self, path: str) -> Stat:
+        """Get stats of a file at given path.
+
+        This is the async version of :meth:`FlatFsReader.stat` method.
+
+        :param path:
+            The path to a file.
+        """
+        ...
+
+    async def exists(self, path: str) -> bool:
+        """Check if file at given path exists.
+
+        This is the async version of :meth:`FlatFsReader.exists` method.
+
+        :param path:
+            The path to check.
+        """
+        ...
+
+
+@export
+class AsyncFlatFsReaderWriter(AsyncFlatFsReader, SupportsAsyncWriteChunks, Protocol):
+    """Async variant of the :class:`FlatFsReaderWriter` protocol."""
+
+    async def remove(self, path: str):
+        """Remove existing file.
+
+        This is the async version of :meth:`FlatFsReaderWriter.remove` method.
+
+        :param path:
+            The path to a file to remove.
+        """
+        ...

{flatfs-0.0.0 → flatfs-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flatfs"
-version = "0.0.0"
+version = "0.3.0"
 description = "Minimal filesystem abstraction with pluggable backends, treating paths as keys and file contents as values."
 authors = [
     {name = "Maciej Wiatrzyk", email = "maciej.wiatrzyk@gmail.com"}
@@ -35,3 +35,4 @@ mockify = "^0.14.0"
 pytest-cov = "^7.1.0"
 ruff = "^0.15.12"
 mypy = "^2.0.0"
+bumpify = "^0.5.1"

flatfs-0.0.0/flatfs/_utils.py

@@ -1,52 +0,0 @@
-import asyncio
-import functools
-from typing import Callable, TypeVar
-
-from flatfs.exc import PathAccessError
-
-T = TypeVar("T")
-
-
-def normalize_path(path: str) -> str:
-    """Normalizes path by removing duplicated slash characters and by
-    evaluating relative path components (e.g. ``../`` or ``./``).
-
-    Returns normalized path, or raises :exc:`flatfs.exc.FileAccessError` if
-    final path would go beyond filesystem bounds.
-
-    :param path:
-        The path to normalize.
-    """
-    parts: list[str] = []
-    for part in path.split("/"):
-        if not part or part == ".":
-            continue
-        if part == "..":
-            if not parts:
-                raise PathAccessError(path)
-            parts.pop()
-        else:
-            parts.append(part)
-    out = "/".join(parts)
-    if out[0] == "/":
-        return out
-    return "/" + out
-
-
-async def run_blocking(func: Callable[..., T], *args, **kwargs) -> T:
-    """Run blocking (i.e. non-async) task in an event loop's default
-    executor.
-
-    On success, return *func*'s result, on failure raise *func*'s exception.
-
-    :param func:
-        The non-async callable to run.
-
-    :param `*args`:
-        Positional args to be passed to *func*.
-
-    :param `**kwargs`:
-        Named args to be passed to *func*.
-    """
-    wrapped_func = functools.partial(func, *args, **kwargs)
-    return await asyncio.get_running_loop().run_in_executor(None, wrapped_func)

flatfs-0.0.0/flatfs/helpers.py

@@ -1,124 +0,0 @@
-from . import _export
-from .interface import SupportsAsyncReadChunks, SupportsAsyncWriteChunks, SupportsReadChunks, SupportsWriteChunks
-
-__all__ = export = _export.Export()  # type: ignore
-
-
-@export
-def read_bytes(fs: SupportsReadChunks, path: str, chunk_size: int = 65535) -> bytes:
-    """Read whole file and return as bytes.
-
-    :param fs:
-        FlatFS object to read from.
-
-    :param path:
-        Path to a file to read.
-
-    :param chunk_size:
-        The chunk size to use during read.
-
-        This is the maximum number of bytes to read from a file per single read
-        operation.
-
-        Defaults to 64kB.
-    """
-    out = b""
-    for chunk in fs.read_chunks(path, chunk_size):
-        out += chunk
-    return out
-
-
-@export
-def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int = 65535):
-    """Write given data to file at given path either creating the file, or
-    replacing with new content.
-
-    :param fs:
-        FlatFS object to write to.
-
-    :param path:
-        The path to a file to create or replace.
-
-    :param data:
-        The data to write.
-
-    :param chunk_size:
-        The chunk size to use during writing.
-
-        If *data* length is greater, it will be split into chunks and written
-        chunk by chunk, with each chunk having at most *chunk_size* bytes.
-
-        Defaults to 64kB.
-    """
-
-    def gen():
-        data_left = data
-        while True:
-            chunk = data_left[:chunk_size]
-            if not chunk:
-                break
-            data_left = data_left[chunk_size:]
-            yield chunk
-
-    fs.write_chunks(path, gen())
-
-
-@export
-def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535) -> str:
-    """Read whole text file and return as string.
-
-    Uses :func:`read_bytes` underneath and then just decodes its output using
-    provided *encoding*, which is ``utf-8`` by default.
-    """
-    return read_bytes(fs, path, chunk_size).decode(encoding)
-
-
-@export
-def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535):
-    """Create new text file or replace existing with given text data.
-
-    Uses :func:`write_bytes` underneath and just encodes given *data* with
-    provided *encoding*, which is ``utf-8`` by default.
-    """
-    return write_bytes(fs, path, data.encode(encoding), chunk_size)
-
-
-@export
-async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
-    """Same as :func:`read_bytes`, but for async code."""
-    out = b""
-    async for chunk in fs.read_chunks(path, chunk_size):
-        out += chunk
-    return out
-
-
-@export
-async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535):
-    """Same as :func:`write_bytes`, but for async code."""
-
-    async def gen():
-        data_left = data
-        while True:
-            chunk = data_left[:chunk_size]
-            if not chunk:
-                break
-            data_left = data_left[chunk_size:]
-            yield chunk
-
-    await fs.write_chunks(path, gen())
-
-
-@export
-async def async_read_text(
-    fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
-) -> str:
-    """Same as :func:`read_text`, but for async code."""
-    return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
-
-
-@export
-async def async_write_text(
-    fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
-):
-    """Same as :func:`write_text`, but for async code."""
-    await async_write_bytes(fs, path, data.encode(encoding), chunk_size)

flatfs-0.0.0/flatfs/interface.py

@@ -1,149 +0,0 @@
-from typing import AsyncGenerator, AsyncIterable, Iterable, Iterator, Protocol
-
-from . import _export
-
-__all__ = export = _export.Export()  # type: ignore
-
-
-@export
-class SupportsReadChunks(Protocol):
-    """Protocol for objects that support chunked reading."""
-
-    def read_chunks(self, path: str, chunk_size: int = 65535) -> Iterator[bytes]:
-        """Read file in chunks and return iterator yielding chunks.
-
-        This method can be safely used with large files without the risk of
-        getting out of memory.
-
-        If *path* does not point to an existing file,
-        :exc:`flatfs.exc.PathNotFoundError` will be raised.
-
-        :param path:
-            The path to a file to read.
-
-        :param chunk_size:
-            The maximum chunk size (in bytes).
-
-            This is the maximum length of a single generated bytes object.
-        """
-        ...
-
-
-@export
-class SupportsWriteChunks(Protocol):
-    """Protocol for object that support writing in chunks."""
-
-    def write_chunks(self, path: str, chunks: Iterable[bytes]):
-        """Create or overwrite file at given *path* and write chunks of data
-        generated by provided *chunks* iterable.
-
-        :param path:
-            The path to a file to create or modify.
-
-        :param chunks:
-            Iterable providing chunks of bytes to write to a file.
-        """
-        ...
-
-
-@export
-class FlatFsReader(SupportsReadChunks, Protocol):
-    """Read-only interface to access files in the flat filesystem."""
-
-    def scan(self) -> Iterator[str]:
-        """Scan filesystem and generate paths to existing files.
-
-        Each yielded path can later be used with other methods of this
-        interface.
-        """
-        ...
-
-    def exists(self, path: str) -> bool:
-        """Check if *path* points to an existing file."""
-        ...
-
-
-@export
-class FlatFsReaderWriter(FlatFsReader, SupportsWriteChunks, Protocol):
-    """Read-write interface to access files in the flat filesystem."""
-
-    def remove(self, path: str):
-        """Remove file at given *path*.
-
-        Raises :exc:`flatfs.exc.PathNotFoundError` if there was no file to
-        remove at given *path*.
-
-        :param path:
-            The path to a file to remove.
-        """
-        ...
-
-
-@export
-class SupportsAsyncReadChunks(Protocol):
-    def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
-        """Read file in chunks and return iterator yielding chunks.
-
-        This method can be safely used with large files without the risk of
-        getting out of memory.
-
-        If *path* does not point to an existing file,
-        :exc:`flatfs.exc.PathNotFoundError` will be raised.
-
-        :param path:
-            The path to a file to read.
-
-        :param chunk_size:
-            The maximum chunk size (in bytes).
-
-            This is the maximum length of a single generated bytes object.
-        """
-        ...
-
-
-@export
-class SupportsAsyncWriteChunks(Protocol):
-    async def write_chunks(self, path: str, chunks: AsyncIterable[bytes]):
-        """Create or overwrite file at given *path* and write chunks of data
-        generated by provided *chunks* iterable.
-
-        :param path:
-            The path to a file to create or modify.
-
-        :param chunks:
-            Iterable providing chunks of bytes to write to a file.
-        """
-        ...
-
-
-@export
-class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
-    """Async read-only interface to access files in the flat filesystem."""
-
-    def scan(self) -> AsyncGenerator[str, None]:
-        """Scan filesystem and generate paths to existing files.
-
-        Each yielded path can later be used with other methods of this
-        interface.
-        """
-        ...
-
-    async def exists(self, path: str) -> bool:
-        """Check if *path* points to an existing file."""
-        ...
-
-
-@export
-class AsyncFlatFsReaderWriter(AsyncFlatFsReader, SupportsAsyncWriteChunks, Protocol):
-    """Async read-write interface to access files in the flat filesystem."""
-
-    async def remove(self, path: str):
-        """Remove file at given *path*.
-
-        Raises :exc:`flatfs.exc.PathNotFoundError` if there was no file to
-        remove at given *path*.
-
-        :param path:
-            The path to a file to remove.
-        """
-        ...