flatfs 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flatfs-0.4.0 → flatfs-0.6.0}/PKG-INFO +1 -1
- flatfs-0.6.0/flatfs/__init__.py +1 -0
- flatfs-0.6.0/flatfs/aio/__init__.py +0 -0
- flatfs-0.6.0/flatfs/aio/api.py +15 -0
- flatfs-0.6.0/flatfs/aio/backends.py +93 -0
- flatfs-0.6.0/flatfs/aio/helpers.py +148 -0
- flatfs-0.6.0/flatfs/aio/interface.py +104 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/api.py +9 -1
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/backends.py +8 -86
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/helpers.py +7 -145
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/interface.py +14 -86
- {flatfs-0.4.0 → flatfs-0.6.0}/pyproject.toml +3 -1
- flatfs-0.4.0/flatfs/__init__.py +0 -1
- {flatfs-0.4.0 → flatfs-0.6.0}/LICENSE.txt +0 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/README.md +0 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/_compat.py +0 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/_export.py +0 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/_utils.py +0 -0
- {flatfs-0.4.0 → flatfs-0.6.0}/flatfs/exc.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.6.0"
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""The all-in-one import helper for :mod:`flatfs.aio` module."""
|
|
2
|
+
|
|
3
|
+
from .backends import __all__ as _backends_all
|
|
4
|
+
from .helpers import __all__ as _helpers_all
|
|
5
|
+
from .interface import __all__ as _interface_all
|
|
6
|
+
|
|
7
|
+
from .backends import *
|
|
8
|
+
from .helpers import *
|
|
9
|
+
from .interface import *
|
|
10
|
+
|
|
11
|
+
__all__ = (
|
|
12
|
+
*_backends_all,
|
|
13
|
+
*_helpers_all,
|
|
14
|
+
*_interface_all,
|
|
15
|
+
) # type: ignore
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from queue import Queue
|
|
3
|
+
from typing import AsyncGenerator, AsyncIterator, Union
|
|
4
|
+
|
|
5
|
+
from flatfs import _export, _utils
|
|
6
|
+
from flatfs.interface import FlatFsReaderWriter, Stat
|
|
7
|
+
|
|
8
|
+
__all__ = export = _export.Export() # type: ignore
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@export
|
|
12
|
+
class AsyncFlatFsAdapter:
|
|
13
|
+
"""Adapter that wraps given :class:`flatfs.interface.FlatFsReaderWriter`
|
|
14
|
+
with async interface.
|
|
15
|
+
|
|
16
|
+
You can use this to adapt e.g. :class:`LocalFlatFs` or
|
|
17
|
+
:class:`InMemoryFlatFs` objects with async interface allowing for
|
|
18
|
+
convenient use with async code. The blocking calls are dispatched to
|
|
19
|
+
background worker which communicates with the caller using queues.
|
|
20
|
+
|
|
21
|
+
:param target:
|
|
22
|
+
The target FlatFS reader-writer to wrap.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, target: FlatFsReaderWriter):
|
|
26
|
+
self.__target = target
|
|
27
|
+
|
|
28
|
+
def uri(self, path: str) -> str:
|
|
29
|
+
return self.__target.uri(path)
|
|
30
|
+
|
|
31
|
+
async def scan(self) -> AsyncGenerator[str, None]:
|
|
32
|
+
|
|
33
|
+
def scanner():
|
|
34
|
+
for path in self.__target.scan():
|
|
35
|
+
loop.call_soon_threadsafe(queue.put_nowait, path)
|
|
36
|
+
loop.call_soon_threadsafe(queue.put_nowait, "")
|
|
37
|
+
|
|
38
|
+
loop = asyncio.get_running_loop()
|
|
39
|
+
asyncio.create_task(_utils.run_blocking(scanner))
|
|
40
|
+
queue: asyncio.Queue[str] = asyncio.Queue()
|
|
41
|
+
while True:
|
|
42
|
+
path = await queue.get()
|
|
43
|
+
if not path:
|
|
44
|
+
break
|
|
45
|
+
yield path
|
|
46
|
+
|
|
47
|
+
async def exists(self, path: str) -> bool:
|
|
48
|
+
return await _utils.run_blocking(self.__target.exists, path)
|
|
49
|
+
|
|
50
|
+
async def stat(self, path: str) -> Stat:
|
|
51
|
+
return await _utils.run_blocking(self.__target.stat, path)
|
|
52
|
+
|
|
53
|
+
async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncIterator[bytes]:
|
|
54
|
+
|
|
55
|
+
def reader():
|
|
56
|
+
try:
|
|
57
|
+
for chunk in self.__target.read_chunks(path, chunk_size):
|
|
58
|
+
loop.call_soon_threadsafe(queue.put_nowait, chunk)
|
|
59
|
+
loop.call_soon_threadsafe(queue.put_nowait, b"")
|
|
60
|
+
except Exception as e:
|
|
61
|
+
loop.call_soon_threadsafe(queue.put_nowait, e)
|
|
62
|
+
|
|
63
|
+
loop = asyncio.get_running_loop()
|
|
64
|
+
queue: asyncio.Queue[Union[bytes, Exception]] = asyncio.Queue()
|
|
65
|
+
asyncio.create_task(_utils.run_blocking(reader))
|
|
66
|
+
while True:
|
|
67
|
+
chunk_or_exc = await queue.get()
|
|
68
|
+
if not chunk_or_exc:
|
|
69
|
+
break
|
|
70
|
+
if isinstance(chunk_or_exc, Exception):
|
|
71
|
+
raise chunk_or_exc
|
|
72
|
+
yield chunk_or_exc
|
|
73
|
+
|
|
74
|
+
async def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
|
|
75
|
+
|
|
76
|
+
def blocking_writer():
|
|
77
|
+
for write_count in self.__target.write_chunks(path, _utils.generate_chunks_from_queue(chunk_queue)):
|
|
78
|
+
loop.call_soon_threadsafe(ack_queue.put_nowait, write_count)
|
|
79
|
+
|
|
80
|
+
loop = asyncio.get_running_loop()
|
|
81
|
+
chunk_queue: Queue[bytes] = Queue()
|
|
82
|
+
ack_queue: asyncio.Queue[int] = asyncio.Queue()
|
|
83
|
+
task = asyncio.create_task(_utils.run_blocking(blocking_writer))
|
|
84
|
+
try:
|
|
85
|
+
async for chunk in chunk_gen:
|
|
86
|
+
chunk_queue.put(chunk)
|
|
87
|
+
yield await ack_queue.get()
|
|
88
|
+
finally:
|
|
89
|
+
chunk_queue.put(b"")
|
|
90
|
+
await task
|
|
91
|
+
|
|
92
|
+
async def remove(self, path: str):
|
|
93
|
+
await _utils.run_blocking(self.__target.remove, path)
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from contextlib import AbstractAsyncContextManager
|
|
3
|
+
from typing import AsyncIterator, Optional
|
|
4
|
+
|
|
5
|
+
from flatfs import _export, _utils, _compat
|
|
6
|
+
|
|
7
|
+
from .interface import SupportsAsyncReadChunks, SupportsAsyncWriteChunks
|
|
8
|
+
|
|
9
|
+
__all__ = export = _export.Export() # type: ignore
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@export
|
|
13
|
+
async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
|
|
14
|
+
"""Async variant of the :func:`read_bytes` helper."""
|
|
15
|
+
out = b""
|
|
16
|
+
async for chunk in fs.read_chunks(path, chunk_size):
|
|
17
|
+
out += chunk
|
|
18
|
+
return out
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@export
|
|
22
|
+
async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
|
|
23
|
+
"""Async variant of the :func:`write_bytes` helper."""
|
|
24
|
+
total_bytes = 0
|
|
25
|
+
async for byte_count in fs.write_chunks(path, _utils.async_split_into_chunks(data, chunk_size)):
|
|
26
|
+
total_bytes += byte_count
|
|
27
|
+
return total_bytes
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@export
|
|
31
|
+
async def async_read_text(
|
|
32
|
+
fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
|
|
33
|
+
) -> str:
|
|
34
|
+
"""Async variant of the :func:`read_text` helper."""
|
|
35
|
+
return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@export
|
|
39
|
+
async def async_write_text(
|
|
40
|
+
fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
|
|
41
|
+
) -> int:
|
|
42
|
+
"""Async variant of the :func:`write_text` helper."""
|
|
43
|
+
return await async_write_bytes(fs, path, data.encode(encoding), chunk_size)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@export
|
|
47
|
+
class AsyncBufferedWriter(AbstractAsyncContextManager):
|
|
48
|
+
"""Async variant of the :class:`BufferedWriter` class."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, fs: SupportsAsyncWriteChunks, path: str, buffer_size: int = 65535):
|
|
51
|
+
self.__chunk_queue: asyncio.Queue[bytes] = asyncio.Queue()
|
|
52
|
+
self.__buffer_size = buffer_size
|
|
53
|
+
self.__buffer = b""
|
|
54
|
+
self.__writer = fs.write_chunks(path, _utils.async_generate_chunks_from_queue(self.__chunk_queue))
|
|
55
|
+
|
|
56
|
+
async def __aenter__(self) -> "AsyncBufferedWriter":
|
|
57
|
+
return self
|
|
58
|
+
|
|
59
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
60
|
+
return await self.close()
|
|
61
|
+
|
|
62
|
+
async def __write_chunk(self, chunk: bytes):
|
|
63
|
+
self.__chunk_queue.put_nowait(chunk)
|
|
64
|
+
await _compat.anext(self.__writer)
|
|
65
|
+
|
|
66
|
+
async def write(self, data: bytes) -> int:
|
|
67
|
+
self.__buffer += data
|
|
68
|
+
while len(self.__buffer) > self.__buffer_size:
|
|
69
|
+
chunk = self.__buffer[: self.__buffer_size]
|
|
70
|
+
await self.__write_chunk(chunk)
|
|
71
|
+
self.__buffer = self.__buffer[self.__buffer_size :]
|
|
72
|
+
return len(data)
|
|
73
|
+
|
|
74
|
+
async def flush(self):
|
|
75
|
+
if not self.__buffer:
|
|
76
|
+
return # Nothing to do
|
|
77
|
+
await self.__write_chunk(self.__buffer)
|
|
78
|
+
self.__buffer = b""
|
|
79
|
+
|
|
80
|
+
async def close(self):
|
|
81
|
+
await self.flush()
|
|
82
|
+
await self.__writer.aclose()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@export
|
|
86
|
+
class AsyncBufferedReader(AbstractAsyncContextManager):
|
|
87
|
+
"""Async variant of the :class:`BufferedReader` class."""
|
|
88
|
+
|
|
89
|
+
def __init__(self, fs: SupportsAsyncReadChunks, path: str, buffer_size: int = 65535):
|
|
90
|
+
self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
|
|
91
|
+
self.__buffer = b""
|
|
92
|
+
|
|
93
|
+
async def __aenter__(self) -> "AsyncBufferedReader":
|
|
94
|
+
return self
|
|
95
|
+
|
|
96
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
97
|
+
return await self.close()
|
|
98
|
+
|
|
99
|
+
async def close(self):
|
|
100
|
+
self.__buffer = b""
|
|
101
|
+
await self.__reader.aclose()
|
|
102
|
+
|
|
103
|
+
async def read(self, count: int) -> bytes:
|
|
104
|
+
if not self.__buffer:
|
|
105
|
+
self.__buffer = await _compat.anext(self.__reader, b"")
|
|
106
|
+
data = self.__buffer[:count]
|
|
107
|
+
self.__buffer = self.__buffer[count:]
|
|
108
|
+
return data
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@export
|
|
112
|
+
class AsyncBufferedTextReader(AbstractAsyncContextManager):
|
|
113
|
+
"""Async variant of the :class:`BufferedTextReader` class."""
|
|
114
|
+
|
|
115
|
+
def __init__(self, fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
|
|
116
|
+
self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
|
|
117
|
+
self.__encoding = encoding
|
|
118
|
+
self.__buffer = ""
|
|
119
|
+
|
|
120
|
+
async def __read_next_chunk(self) -> str:
|
|
121
|
+
return (await _compat.anext(self.__reader, b"")).decode(self.__encoding)
|
|
122
|
+
|
|
123
|
+
async def __aenter__(self) -> "AsyncBufferedTextReader":
|
|
124
|
+
return self
|
|
125
|
+
|
|
126
|
+
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
127
|
+
return await self.close()
|
|
128
|
+
|
|
129
|
+
async def __aiter__(self) -> AsyncIterator[str]:
|
|
130
|
+
while next_chunk := await self.__read_next_chunk():
|
|
131
|
+
self.__buffer += next_chunk
|
|
132
|
+
lines = self.__buffer.splitlines()
|
|
133
|
+
if len(lines) > 1:
|
|
134
|
+
for line in lines[:-1]:
|
|
135
|
+
yield line
|
|
136
|
+
self.__buffer = lines[-1]
|
|
137
|
+
yield self.__buffer
|
|
138
|
+
|
|
139
|
+
async def read(self, count: int) -> str:
|
|
140
|
+
if not self.__buffer:
|
|
141
|
+
self.__buffer = await self.__read_next_chunk()
|
|
142
|
+
out = self.__buffer[:count]
|
|
143
|
+
self.__buffer = self.__buffer[count:]
|
|
144
|
+
return out
|
|
145
|
+
|
|
146
|
+
async def close(self):
|
|
147
|
+
self.__buffer = ""
|
|
148
|
+
await self.__reader.aclose()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
from typing import AsyncGenerator, AsyncIterator, Protocol
|
|
2
|
+
|
|
3
|
+
from flatfs import _export
|
|
4
|
+
from flatfs.interface import Stat
|
|
5
|
+
|
|
6
|
+
__all__ = export = _export.Export() # type: ignore
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@export
|
|
10
|
+
class SupportsAsyncReadChunks(Protocol):
|
|
11
|
+
"""Async variant of the :class:`SupportsReadChunks` protocol."""
|
|
12
|
+
|
|
13
|
+
def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
|
|
14
|
+
"""Return a generator for reading file in chunks of given maximum size.
|
|
15
|
+
|
|
16
|
+
This is the async version of :meth:`SupportsReadChunks.read_chunks`
|
|
17
|
+
method.
|
|
18
|
+
|
|
19
|
+
:param path:
|
|
20
|
+
The path to a file to read.
|
|
21
|
+
|
|
22
|
+
:param chunk_size:
|
|
23
|
+
The maximum chunk size, in bytes.
|
|
24
|
+
"""
|
|
25
|
+
...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@export
|
|
29
|
+
class SupportsAsyncWriteChunks(Protocol):
|
|
30
|
+
"""Async variant of the :class:`SupportsWriteChunks` protocol."""
|
|
31
|
+
|
|
32
|
+
def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
|
|
33
|
+
"""Generator for creating or overwriting files.
|
|
34
|
+
|
|
35
|
+
This is the async version of :meth:`SupportsWriteChunks.write_chunks`
|
|
36
|
+
method.
|
|
37
|
+
|
|
38
|
+
:param path:
|
|
39
|
+
The path to a file to create or replace.
|
|
40
|
+
|
|
41
|
+
:param chunk_gen:
|
|
42
|
+
Async generator yielding file chunks.
|
|
43
|
+
"""
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@export
|
|
48
|
+
class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
|
|
49
|
+
"""Async variant of the :class:`FlatFsReader` protocol."""
|
|
50
|
+
|
|
51
|
+
def uri(self, path: str) -> str:
|
|
52
|
+
"""Get a unique URI string for the given path.
|
|
53
|
+
|
|
54
|
+
This method is meant to be used to get a fullpath-like representation
|
|
55
|
+
of the given path in a form of a URI string.
|
|
56
|
+
|
|
57
|
+
.. important::
|
|
58
|
+
The returned URI-like string is meant to be used for presentation
|
|
59
|
+
purposes only, where a sort of fullpath is needed (e.g. to display
|
|
60
|
+
error pointing to a missing file).
|
|
61
|
+
"""
|
|
62
|
+
...
|
|
63
|
+
|
|
64
|
+
def scan(self) -> AsyncIterator[str]:
|
|
65
|
+
"""Return iterator that yields normalized paths to existing files.
|
|
66
|
+
|
|
67
|
+
This is the async version of :meth:`FlatFsReader.scan` method.
|
|
68
|
+
"""
|
|
69
|
+
...
|
|
70
|
+
|
|
71
|
+
async def stat(self, path: str) -> Stat:
|
|
72
|
+
"""Get stats of a file at given path.
|
|
73
|
+
|
|
74
|
+
This is the async version of :meth:`FlatFsReader.stat` method.
|
|
75
|
+
|
|
76
|
+
:param path:
|
|
77
|
+
The path to a file.
|
|
78
|
+
"""
|
|
79
|
+
...
|
|
80
|
+
|
|
81
|
+
async def exists(self, path: str) -> bool:
|
|
82
|
+
"""Check if file at given path exists.
|
|
83
|
+
|
|
84
|
+
This is the async version of :meth:`FlatFsReader.exists` method.
|
|
85
|
+
|
|
86
|
+
:param path:
|
|
87
|
+
The path to check.
|
|
88
|
+
"""
|
|
89
|
+
...
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@export
|
|
93
|
+
class AsyncFlatFsReaderWriter(AsyncFlatFsReader, SupportsAsyncWriteChunks, Protocol):
|
|
94
|
+
"""Async variant of the :class:`FlatFsReaderWriter` protocol."""
|
|
95
|
+
|
|
96
|
+
async def remove(self, path: str):
|
|
97
|
+
"""Remove existing file.
|
|
98
|
+
|
|
99
|
+
This is the async version of :meth:`FlatFsReaderWriter.remove` method.
|
|
100
|
+
|
|
101
|
+
:param path:
|
|
102
|
+
The path to a file to remove.
|
|
103
|
+
"""
|
|
104
|
+
...
|
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
"""The all-in-one import helper for FlatFS public API."""
|
|
2
2
|
|
|
3
|
+
from .aio.api import __all__ as _aio_all
|
|
3
4
|
from .backends import __all__ as _backends_all
|
|
4
5
|
from .exc import __all__ as _exc_all
|
|
5
6
|
from .helpers import __all__ as _helpers_all
|
|
6
7
|
from .interface import __all__ as _interface_all
|
|
7
8
|
|
|
9
|
+
from .aio.api import *
|
|
8
10
|
from .backends import *
|
|
9
11
|
from .exc import *
|
|
10
12
|
from .helpers import *
|
|
11
13
|
from .interface import *
|
|
12
14
|
|
|
13
|
-
__all__ =
|
|
15
|
+
__all__ = (
|
|
16
|
+
*_aio_all,
|
|
17
|
+
*_backends_all,
|
|
18
|
+
*_exc_all,
|
|
19
|
+
*_helpers_all,
|
|
20
|
+
*_interface_all,
|
|
21
|
+
) # type: ignore
|
|
@@ -1,14 +1,12 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import dataclasses
|
|
3
2
|
import datetime
|
|
4
3
|
import fnmatch
|
|
5
4
|
import os
|
|
6
5
|
import pathlib
|
|
7
|
-
from
|
|
8
|
-
from typing import AsyncGenerator, AsyncIterator, Generator, Iterator, Optional, Union
|
|
6
|
+
from typing import Generator, Iterator, Optional
|
|
9
7
|
|
|
10
8
|
from flatfs.exc import PathAccessError, PathNotFoundError
|
|
11
|
-
from flatfs.interface import
|
|
9
|
+
from flatfs.interface import Stat
|
|
12
10
|
|
|
13
11
|
from . import _utils, _export
|
|
14
12
|
|
|
@@ -79,6 +77,9 @@ class LocalFlatFs:
|
|
|
79
77
|
elif not self.__is_excluded(fullrelpath) and self.__is_included(fullrelpath):
|
|
80
78
|
yield fullrelpath
|
|
81
79
|
|
|
80
|
+
def uri(self, path: str) -> str:
|
|
81
|
+
return f"file://{self.__make_abspath(path)}"
|
|
82
|
+
|
|
82
83
|
def scan(self) -> Iterator[str]:
|
|
83
84
|
return self.__walk(self.__root_dir, "")
|
|
84
85
|
|
|
@@ -141,6 +142,9 @@ class InMemoryFlatFs:
|
|
|
141
142
|
normalized_key = _utils.normalize_path(path)
|
|
142
143
|
return normalized_key
|
|
143
144
|
|
|
145
|
+
def uri(self, path: str) -> str:
|
|
146
|
+
return f"mem://{self.__make_key(path)}"
|
|
147
|
+
|
|
144
148
|
def scan(self) -> Iterator[str]:
|
|
145
149
|
return iter(self.__storage.keys())
|
|
146
150
|
|
|
@@ -181,85 +185,3 @@ class InMemoryFlatFs:
|
|
|
181
185
|
if key not in self.__storage:
|
|
182
186
|
raise PathNotFoundError(path)
|
|
183
187
|
del self.__storage[key]
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
@export
|
|
187
|
-
class AsyncFlatFsAdapter:
|
|
188
|
-
"""Adapter that wraps given :class:`flatfs.interface.FlatFsReaderWriter`
|
|
189
|
-
with async interface.
|
|
190
|
-
|
|
191
|
-
You can use this to adapt e.g. :class:`LocalFlatFs` or
|
|
192
|
-
:class:`InMemoryFlatFs` objects with async interface allowing for
|
|
193
|
-
convenient use with async code. The blocking calls are dispatched to
|
|
194
|
-
background worker which communicates with the caller using queues.
|
|
195
|
-
|
|
196
|
-
:param target:
|
|
197
|
-
The target FlatFS reader-writer to wrap.
|
|
198
|
-
"""
|
|
199
|
-
|
|
200
|
-
def __init__(self, target: FlatFsReaderWriter):
|
|
201
|
-
self.__target = target
|
|
202
|
-
|
|
203
|
-
async def scan(self) -> AsyncGenerator[str, None]:
|
|
204
|
-
|
|
205
|
-
def scanner():
|
|
206
|
-
for path in self.__target.scan():
|
|
207
|
-
loop.call_soon_threadsafe(queue.put_nowait, path)
|
|
208
|
-
loop.call_soon_threadsafe(queue.put_nowait, "")
|
|
209
|
-
|
|
210
|
-
loop = asyncio.get_running_loop()
|
|
211
|
-
asyncio.create_task(_utils.run_blocking(scanner))
|
|
212
|
-
queue: asyncio.Queue[str] = asyncio.Queue()
|
|
213
|
-
while True:
|
|
214
|
-
path = await queue.get()
|
|
215
|
-
if not path:
|
|
216
|
-
break
|
|
217
|
-
yield path
|
|
218
|
-
|
|
219
|
-
async def exists(self, path: str) -> bool:
|
|
220
|
-
return await _utils.run_blocking(self.__target.exists, path)
|
|
221
|
-
|
|
222
|
-
async def stat(self, path: str) -> Stat:
|
|
223
|
-
return await _utils.run_blocking(self.__target.stat, path)
|
|
224
|
-
|
|
225
|
-
async def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncIterator[bytes]:
|
|
226
|
-
|
|
227
|
-
def reader():
|
|
228
|
-
try:
|
|
229
|
-
for chunk in self.__target.read_chunks(path, chunk_size):
|
|
230
|
-
loop.call_soon_threadsafe(queue.put_nowait, chunk)
|
|
231
|
-
loop.call_soon_threadsafe(queue.put_nowait, b"")
|
|
232
|
-
except Exception as e:
|
|
233
|
-
loop.call_soon_threadsafe(queue.put_nowait, e)
|
|
234
|
-
|
|
235
|
-
loop = asyncio.get_running_loop()
|
|
236
|
-
queue: asyncio.Queue[Union[bytes, Exception]] = asyncio.Queue()
|
|
237
|
-
asyncio.create_task(_utils.run_blocking(reader))
|
|
238
|
-
while True:
|
|
239
|
-
chunk_or_exc = await queue.get()
|
|
240
|
-
if not chunk_or_exc:
|
|
241
|
-
break
|
|
242
|
-
if isinstance(chunk_or_exc, Exception):
|
|
243
|
-
raise chunk_or_exc
|
|
244
|
-
yield chunk_or_exc
|
|
245
|
-
|
|
246
|
-
async def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
|
|
247
|
-
|
|
248
|
-
def blocking_writer():
|
|
249
|
-
for write_count in self.__target.write_chunks(path, _utils.generate_chunks_from_queue(chunk_queue)):
|
|
250
|
-
loop.call_soon_threadsafe(ack_queue.put_nowait, write_count)
|
|
251
|
-
|
|
252
|
-
loop = asyncio.get_running_loop()
|
|
253
|
-
chunk_queue: Queue[bytes] = Queue()
|
|
254
|
-
ack_queue: asyncio.Queue[int] = asyncio.Queue()
|
|
255
|
-
task = asyncio.create_task(_utils.run_blocking(blocking_writer))
|
|
256
|
-
try:
|
|
257
|
-
async for chunk in chunk_gen:
|
|
258
|
-
chunk_queue.put(chunk)
|
|
259
|
-
yield await ack_queue.get()
|
|
260
|
-
finally:
|
|
261
|
-
chunk_queue.put(b"")
|
|
262
|
-
await task
|
|
263
|
-
|
|
264
|
-
async def remove(self, path: str):
|
|
265
|
-
await _utils.run_blocking(self.__target.remove, path)
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import
|
|
2
|
-
from contextlib import AbstractAsyncContextManager, AbstractContextManager
|
|
1
|
+
from contextlib import AbstractContextManager
|
|
3
2
|
from queue import Queue
|
|
4
|
-
from typing import Iterator,
|
|
3
|
+
from typing import Iterator, Optional
|
|
5
4
|
|
|
6
|
-
from . import _export, _utils
|
|
7
|
-
from .interface import
|
|
5
|
+
from . import _export, _utils
|
|
6
|
+
from .interface import SupportsReadChunks, SupportsWriteChunks
|
|
8
7
|
|
|
9
8
|
__all__ = export = _export.Export() # type: ignore
|
|
10
9
|
|
|
@@ -33,15 +32,6 @@ def read_bytes(fs: SupportsReadChunks, path: str, chunk_size: int = 65535) -> by
|
|
|
33
32
|
return out
|
|
34
33
|
|
|
35
34
|
|
|
36
|
-
@export
|
|
37
|
-
async def async_read_bytes(fs: SupportsAsyncReadChunks, path: str, chunk_size: int = 65535) -> bytes:
|
|
38
|
-
"""Async variant of the :func:`read_bytes` helper."""
|
|
39
|
-
out = b""
|
|
40
|
-
async for chunk in fs.read_chunks(path, chunk_size):
|
|
41
|
-
out += chunk
|
|
42
|
-
return out
|
|
43
|
-
|
|
44
|
-
|
|
45
35
|
@export
|
|
46
36
|
def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
|
|
47
37
|
"""Write given data to file at given path either creating the file, or
|
|
@@ -72,15 +62,6 @@ def write_bytes(fs: SupportsWriteChunks, path: str, data: bytes, chunk_size: int
|
|
|
72
62
|
return total_bytes
|
|
73
63
|
|
|
74
64
|
|
|
75
|
-
@export
|
|
76
|
-
async def async_write_bytes(fs: SupportsAsyncWriteChunks, path: str, data: bytes, chunk_size: int = 65535) -> int:
|
|
77
|
-
"""Async variant of the :func:`write_bytes` helper."""
|
|
78
|
-
total_bytes = 0
|
|
79
|
-
async for byte_count in fs.write_chunks(path, _utils.async_split_into_chunks(data, chunk_size)):
|
|
80
|
-
total_bytes += byte_count
|
|
81
|
-
return total_bytes
|
|
82
|
-
|
|
83
|
-
|
|
84
65
|
@export
|
|
85
66
|
def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535) -> str:
|
|
86
67
|
"""Read whole text file and return as string.
|
|
@@ -106,20 +87,14 @@ def read_text(fs: SupportsReadChunks, path: str, encoding: str = "utf-8", chunk_
|
|
|
106
87
|
|
|
107
88
|
|
|
108
89
|
@export
|
|
109
|
-
|
|
110
|
-
fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", chunk_size: int = 65535
|
|
111
|
-
) -> str:
|
|
112
|
-
"""Async variant of the :func:`read_text` helper."""
|
|
113
|
-
return (await async_read_bytes(fs, path, chunk_size)).decode(encoding)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
@export
|
|
117
|
-
def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535):
|
|
90
|
+
def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535) -> int:
|
|
118
91
|
"""Create new text file or replace existing with given text data.
|
|
119
92
|
|
|
120
93
|
Uses :func:`write_bytes` underneath and just encodes given *data* with
|
|
121
94
|
provided *encoding*, which is ``utf-8`` by default.
|
|
122
95
|
|
|
96
|
+
Returns the number of bytes written.
|
|
97
|
+
|
|
123
98
|
:param fs:
|
|
124
99
|
FlatFS object to write to.
|
|
125
100
|
|
|
@@ -137,14 +112,6 @@ def write_text(fs: SupportsWriteChunks, path: str, data: str, encoding: str = "u
|
|
|
137
112
|
return write_bytes(fs, path, data.encode(encoding), chunk_size)
|
|
138
113
|
|
|
139
114
|
|
|
140
|
-
@export
|
|
141
|
-
async def async_write_text(
|
|
142
|
-
fs: SupportsAsyncWriteChunks, path: str, data: str, encoding: str = "utf-8", chunk_size: int = 65535
|
|
143
|
-
):
|
|
144
|
-
"""Async variant of the :func:`write_text` helper."""
|
|
145
|
-
await async_write_bytes(fs, path, data.encode(encoding), chunk_size)
|
|
146
|
-
|
|
147
|
-
|
|
148
115
|
@export
|
|
149
116
|
class BufferedWriter(AbstractContextManager):
|
|
150
117
|
"""Buffered writer for creating new or replacing existing files.
|
|
@@ -218,45 +185,6 @@ class BufferedWriter(AbstractContextManager):
|
|
|
218
185
|
return len(data)
|
|
219
186
|
|
|
220
187
|
|
|
221
|
-
@export
|
|
222
|
-
class AsyncBufferedWriter(AbstractAsyncContextManager):
|
|
223
|
-
"""Async variant of the :class:`BufferedWriter` class."""
|
|
224
|
-
|
|
225
|
-
def __init__(self, fs: SupportsAsyncWriteChunks, path: str, buffer_size: int = 65535):
|
|
226
|
-
self.__chunk_queue: asyncio.Queue[bytes] = asyncio.Queue()
|
|
227
|
-
self.__buffer_size = buffer_size
|
|
228
|
-
self.__buffer = b""
|
|
229
|
-
self.__writer = fs.write_chunks(path, _utils.async_generate_chunks_from_queue(self.__chunk_queue))
|
|
230
|
-
|
|
231
|
-
async def __aenter__(self) -> "AsyncBufferedWriter":
|
|
232
|
-
return self
|
|
233
|
-
|
|
234
|
-
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
235
|
-
return await self.close()
|
|
236
|
-
|
|
237
|
-
async def __write_chunk(self, chunk: bytes):
|
|
238
|
-
self.__chunk_queue.put_nowait(chunk)
|
|
239
|
-
await _compat.anext(self.__writer)
|
|
240
|
-
|
|
241
|
-
async def write(self, data: bytes) -> int:
|
|
242
|
-
self.__buffer += data
|
|
243
|
-
while len(self.__buffer) > self.__buffer_size:
|
|
244
|
-
chunk = self.__buffer[: self.__buffer_size]
|
|
245
|
-
await self.__write_chunk(chunk)
|
|
246
|
-
self.__buffer = self.__buffer[self.__buffer_size :]
|
|
247
|
-
return len(data)
|
|
248
|
-
|
|
249
|
-
async def flush(self):
|
|
250
|
-
if not self.__buffer:
|
|
251
|
-
return # Nothing to do
|
|
252
|
-
await self.__write_chunk(self.__buffer)
|
|
253
|
-
self.__buffer = b""
|
|
254
|
-
|
|
255
|
-
async def close(self):
|
|
256
|
-
await self.flush()
|
|
257
|
-
await self.__writer.aclose()
|
|
258
|
-
|
|
259
|
-
|
|
260
188
|
@export
|
|
261
189
|
class BufferedReader(AbstractContextManager):
|
|
262
190
|
"""Helper for buffered reading from underlying flat filesystem.
|
|
@@ -302,32 +230,6 @@ class BufferedReader(AbstractContextManager):
|
|
|
302
230
|
return data
|
|
303
231
|
|
|
304
232
|
|
|
305
|
-
@export
|
|
306
|
-
class AsyncBufferedReader(AbstractAsyncContextManager):
|
|
307
|
-
"""Async variant of the :class:`BufferedReader` class."""
|
|
308
|
-
|
|
309
|
-
def __init__(self, fs: SupportsAsyncReadChunks, path: str, buffer_size: int = 65535):
|
|
310
|
-
self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
|
|
311
|
-
self.__buffer = b""
|
|
312
|
-
|
|
313
|
-
async def __aenter__(self) -> "AsyncBufferedReader":
|
|
314
|
-
return self
|
|
315
|
-
|
|
316
|
-
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
317
|
-
return await self.close()
|
|
318
|
-
|
|
319
|
-
async def close(self):
|
|
320
|
-
self.__buffer = b""
|
|
321
|
-
await self.__reader.aclose()
|
|
322
|
-
|
|
323
|
-
async def read(self, count: int) -> bytes:
|
|
324
|
-
if not self.__buffer:
|
|
325
|
-
self.__buffer = await _compat.anext(self.__reader, b"")
|
|
326
|
-
data = self.__buffer[:count]
|
|
327
|
-
self.__buffer = self.__buffer[count:]
|
|
328
|
-
return data
|
|
329
|
-
|
|
330
|
-
|
|
331
233
|
@export
|
|
332
234
|
class BufferedTextReader(AbstractContextManager):
|
|
333
235
|
"""Equivalent of :class:`BufferedReader` for text files.
|
|
@@ -394,43 +296,3 @@ class BufferedTextReader(AbstractContextManager):
|
|
|
394
296
|
"""
|
|
395
297
|
self.__buffer = ""
|
|
396
298
|
self.__reader.close()
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
@export
|
|
400
|
-
class AsyncBufferedTextReader(AbstractAsyncContextManager):
|
|
401
|
-
"""Async variant of the :class:`BufferedTextReader` class."""
|
|
402
|
-
|
|
403
|
-
def __init__(self, fs: SupportsAsyncReadChunks, path: str, encoding: str = "utf-8", buffer_size: int = 65535):
|
|
404
|
-
self.__reader = fs.read_chunks(path, chunk_size=buffer_size)
|
|
405
|
-
self.__encoding = encoding
|
|
406
|
-
self.__buffer = ""
|
|
407
|
-
|
|
408
|
-
async def __read_next_chunk(self) -> str:
|
|
409
|
-
return (await _compat.anext(self.__reader, b"")).decode(self.__encoding)
|
|
410
|
-
|
|
411
|
-
async def __aenter__(self) -> "AsyncBufferedTextReader":
|
|
412
|
-
return self
|
|
413
|
-
|
|
414
|
-
async def __aexit__(self, exc_type, exc_value, traceback) -> Optional[bool]:
|
|
415
|
-
return await self.close()
|
|
416
|
-
|
|
417
|
-
async def __aiter__(self) -> AsyncIterator[str]:
|
|
418
|
-
while next_chunk := await self.__read_next_chunk():
|
|
419
|
-
self.__buffer += next_chunk
|
|
420
|
-
lines = self.__buffer.splitlines()
|
|
421
|
-
if len(lines) > 1:
|
|
422
|
-
for line in lines[:-1]:
|
|
423
|
-
yield line
|
|
424
|
-
self.__buffer = lines[-1]
|
|
425
|
-
yield self.__buffer
|
|
426
|
-
|
|
427
|
-
async def read(self, count: int) -> str:
|
|
428
|
-
if not self.__buffer:
|
|
429
|
-
self.__buffer = await self.__read_next_chunk()
|
|
430
|
-
out = self.__buffer[:count]
|
|
431
|
-
self.__buffer = self.__buffer[count:]
|
|
432
|
-
return out
|
|
433
|
-
|
|
434
|
-
async def close(self):
|
|
435
|
-
self.__buffer = ""
|
|
436
|
-
await self.__reader.aclose()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import dataclasses
|
|
2
2
|
import datetime
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Generator, Iterator, Protocol
|
|
4
4
|
|
|
5
5
|
from . import _export
|
|
6
6
|
|
|
@@ -71,6 +71,19 @@ class SupportsWriteChunks(Protocol):
|
|
|
71
71
|
class FlatFsReader(SupportsReadChunks, Protocol):
|
|
72
72
|
"""Read-only interface to access files in the flat filesystem."""
|
|
73
73
|
|
|
74
|
+
def uri(self, path: str) -> str:
|
|
75
|
+
"""Get a unique URI string for the given path.
|
|
76
|
+
|
|
77
|
+
This method is meant to be used to get a fullpath-like representation
|
|
78
|
+
of the given path in a form of a URI string.
|
|
79
|
+
|
|
80
|
+
.. important::
|
|
81
|
+
The returned URI-like string is meant to be used for presentation
|
|
82
|
+
purposes only, where a sort of fullpath is needed (e.g. to display
|
|
83
|
+
error pointing to a missing file).
|
|
84
|
+
"""
|
|
85
|
+
...
|
|
86
|
+
|
|
74
87
|
def scan(self) -> Iterator[str]:
|
|
75
88
|
"""Return iterator that yields normalized paths to existing files."""
|
|
76
89
|
...
|
|
@@ -111,88 +124,3 @@ class FlatFsReaderWriter(FlatFsReader, SupportsWriteChunks, Protocol):
|
|
|
111
124
|
The path to a file to remove.
|
|
112
125
|
"""
|
|
113
126
|
...
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
@export
|
|
117
|
-
class SupportsAsyncReadChunks(Protocol):
|
|
118
|
-
"""Async variant of the :class:`SupportsReadChunks` protocol."""
|
|
119
|
-
|
|
120
|
-
def read_chunks(self, path: str, chunk_size: int = 65535) -> AsyncGenerator[bytes, None]:
|
|
121
|
-
"""Return a generator for reading file in chunks of given maximum size.
|
|
122
|
-
|
|
123
|
-
This is the async version of :meth:`SupportsReadChunks.read_chunks`
|
|
124
|
-
method.
|
|
125
|
-
|
|
126
|
-
:param path:
|
|
127
|
-
The path to a file to read.
|
|
128
|
-
|
|
129
|
-
:param chunk_size:
|
|
130
|
-
The maximum chunk size, in bytes.
|
|
131
|
-
"""
|
|
132
|
-
...
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
@export
|
|
136
|
-
class SupportsAsyncWriteChunks(Protocol):
|
|
137
|
-
"""Async variant of the :class:`SupportsWriteChunks` protocol."""
|
|
138
|
-
|
|
139
|
-
def write_chunks(self, path: str, chunk_gen: AsyncGenerator[bytes, None]) -> AsyncGenerator[int, None]:
|
|
140
|
-
"""Generator for creating or overwriting files.
|
|
141
|
-
|
|
142
|
-
This is the async version of :meth:`SupportsWriteChunks.write_chunks`
|
|
143
|
-
method.
|
|
144
|
-
|
|
145
|
-
:param path:
|
|
146
|
-
The path to a file to create or replace.
|
|
147
|
-
|
|
148
|
-
:param chunk_gen:
|
|
149
|
-
Async generator yielding file chunks.
|
|
150
|
-
"""
|
|
151
|
-
...
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
@export
|
|
155
|
-
class AsyncFlatFsReader(SupportsAsyncReadChunks, Protocol):
|
|
156
|
-
"""Async variant of the :class:`FlatFsReader` protocol."""
|
|
157
|
-
|
|
158
|
-
def scan(self) -> AsyncIterator[str]:
|
|
159
|
-
"""Return iterator that yields normalized paths to existing files.
|
|
160
|
-
|
|
161
|
-
This is the async version of :meth:`FlatFsReader.scan` method.
|
|
162
|
-
"""
|
|
163
|
-
...
|
|
164
|
-
|
|
165
|
-
async def stat(self, path: str) -> Stat:
|
|
166
|
-
"""Get stats of a file at given path.
|
|
167
|
-
|
|
168
|
-
This is the async version of :meth:`FlatFsReader.stat` method.
|
|
169
|
-
|
|
170
|
-
:param path:
|
|
171
|
-
The path to a file.
|
|
172
|
-
"""
|
|
173
|
-
...
|
|
174
|
-
|
|
175
|
-
async def exists(self, path: str) -> bool:
|
|
176
|
-
"""Check if file at given path exists.
|
|
177
|
-
|
|
178
|
-
This is the async version of :meth:`FlatFsReader.exists` method.
|
|
179
|
-
|
|
180
|
-
:param path:
|
|
181
|
-
The path to check.
|
|
182
|
-
"""
|
|
183
|
-
...
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
@export
|
|
187
|
-
class AsyncFlatFsReaderWriter(AsyncFlatFsReader, SupportsAsyncWriteChunks, Protocol):
|
|
188
|
-
"""Async variant of the :class:`FlatFsReaderWriter` protocol."""
|
|
189
|
-
|
|
190
|
-
async def remove(self, path: str):
|
|
191
|
-
"""Remove existing file.
|
|
192
|
-
|
|
193
|
-
This is the async version of :meth:`FlatFsReaderWriter.remove` method.
|
|
194
|
-
|
|
195
|
-
:param path:
|
|
196
|
-
The path to a file to remove.
|
|
197
|
-
"""
|
|
198
|
-
...
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "flatfs"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.6.0"
|
|
4
4
|
description = "Minimal filesystem abstraction with pluggable backends, treating paths as keys and file contents as values."
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "Maciej Wiatrzyk", email = "maciej.wiatrzyk@gmail.com"}
|
|
@@ -36,3 +36,5 @@ pytest-cov = "^7.1.0"
|
|
|
36
36
|
ruff = "^0.15.12"
|
|
37
37
|
mypy = "^1.19.1"
|
|
38
38
|
bumpify = "^0.5.1"
|
|
39
|
+
sphinx = "7.4.7"
|
|
40
|
+
sphinx-rtd-theme = "^3.1.0"
|
flatfs-0.4.0/flatfs/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.4.0"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|