barecat 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- barecat/__init__.py +108 -0
- barecat/__main__.py +5 -0
- barecat/_api.py +143 -0
- barecat/_version.py +24 -0
- barecat/cli/__init__.py +0 -0
- barecat/cli/commands.py +540 -0
- barecat/cli/completions.py +27 -0
- barecat/cli/impl.py +609 -0
- barecat/cli/legacy.py +361 -0
- barecat/cli/main.py +1355 -0
- barecat/cli/shell.py +827 -0
- barecat/completions/__init__.py +1 -0
- barecat/completions/barecat.bash +246 -0
- barecat/completions/barecat.zsh +268 -0
- barecat/core/__init__.py +0 -0
- barecat/core/barecat.py +1209 -0
- barecat/core/index.py +1983 -0
- barecat/core/paths.py +61 -0
- barecat/core/sharder.py +328 -0
- barecat/core/types.py +340 -0
- barecat/exceptions.py +92 -0
- barecat/formats/__init__.py +0 -0
- barecat/formats/archive_formats.py +234 -0
- barecat/formats/from_tar_stream.py +53 -0
- barecat/formats/to_tar_stream.py +54 -0
- barecat/io/__init__.py +5 -0
- barecat/io/codecs.py +622 -0
- barecat/io/copyfile.py +550 -0
- barecat/io/fileobj.py +556 -0
- barecat/io/open.py +41 -0
- barecat/maintenance/__init__.py +0 -0
- barecat/maintenance/complete_database.py +79 -0
- barecat/maintenance/defrag.py +541 -0
- barecat/maintenance/merge.py +613 -0
- barecat/maintenance/reshard.py +153 -0
- barecat/maintenance/rsync.py +1282 -0
- barecat/maintenance/upgrade_database.py +200 -0
- barecat/maintenance/upgrade_database2.py +74 -0
- barecat/man/barecat.1 +284 -0
- barecat/sql/indexes.sql +6 -0
- barecat/sql/schema.sql +55 -0
- barecat/sql/triggers.sql +172 -0
- barecat/tui/__init__.py +0 -0
- barecat/tui/browse.py +448 -0
- barecat/tui/ncdu.py +318 -0
- barecat/util/__init__.py +0 -0
- barecat/util/consumed_threadpool.py +119 -0
- barecat/util/glob_helper.py +584 -0
- barecat/util/glob_to_regex.py +353 -0
- barecat/util/list_files.py +12 -0
- barecat/util/misc.py +174 -0
- barecat/util/physical_order.py +77 -0
- barecat/util/progbar.py +40 -0
- barecat/util/threading.py +59 -0
- barecat-0.3.2.dist-info/METADATA +276 -0
- barecat-0.3.2.dist-info/RECORD +60 -0
- barecat-0.3.2.dist-info/WHEEL +5 -0
- barecat-0.3.2.dist-info/entry_points.txt +15 -0
- barecat-0.3.2.dist-info/licenses/LICENSE +21 -0
- barecat-0.3.2.dist-info/top_level.txt +1 -0
barecat/core/barecat.py
ADDED
|
@@ -0,0 +1,1209 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import os.path as osp
|
|
5
|
+
import shutil
|
|
6
|
+
import stat
|
|
7
|
+
import warnings
|
|
8
|
+
from collections.abc import Callable, Iterator, MutableMapping
|
|
9
|
+
from contextlib import AbstractContextManager
|
|
10
|
+
from typing import Any, Optional, TYPE_CHECKING, Union
|
|
11
|
+
|
|
12
|
+
from ..io import codecs as barecat_codecs
|
|
13
|
+
from ..util import progbar as barecat_progbar
|
|
14
|
+
from ..util import misc as barecat_util
|
|
15
|
+
import crc32c as crc32c_lib
|
|
16
|
+
from ..core.sharder import Sharder
|
|
17
|
+
from ..io.fileobj import BarecatFileObjectHelper
|
|
18
|
+
from ..maintenance.merge import BarecatMergeHelper
|
|
19
|
+
from ..maintenance.defrag import BarecatDefragger
|
|
20
|
+
from ..exceptions import (
|
|
21
|
+
FileExistsBarecatError,
|
|
22
|
+
FileNotFoundBarecatError,
|
|
23
|
+
IsADirectoryBarecatError,
|
|
24
|
+
)
|
|
25
|
+
from ..util.misc import raise_if_readonly, raise_if_readonly_or_append_only
|
|
26
|
+
from ..util.threading import ThreadLocalStorage
|
|
27
|
+
from ..io.copyfile import accumulate_crc32c
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from barecat import (
|
|
31
|
+
BarecatDirInfo,
|
|
32
|
+
BarecatFileInfo,
|
|
33
|
+
BarecatEntryInfo,
|
|
34
|
+
Index,
|
|
35
|
+
BarecatFileObject,
|
|
36
|
+
)
|
|
37
|
+
from .paths import resolve_index_path
|
|
38
|
+
else:
|
|
39
|
+
from .types import (
|
|
40
|
+
BarecatDirInfo,
|
|
41
|
+
BarecatFileInfo,
|
|
42
|
+
BarecatEntryInfo,
|
|
43
|
+
)
|
|
44
|
+
from ..io.fileobj import (
|
|
45
|
+
BarecatFileObject,
|
|
46
|
+
)
|
|
47
|
+
from .index import Index, normalize_path
|
|
48
|
+
from .paths import resolve_index_path
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger(__name__)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Barecat(MutableMapping[str, Any], AbstractContextManager):
|
|
54
|
+
"""Object for reading or writing a Barecat archive.
|
|
55
|
+
|
|
56
|
+
A Barecat archive consists of several (large) shard files, each containing the data of multiple
|
|
57
|
+
small files, and an SQLite index database that maps file paths to the corresponding shard,
|
|
58
|
+
offset and size within the shard, as well as metadata such as modification time and checksum.
|
|
59
|
+
|
|
60
|
+
The ``Barecat`` object provides two main interfaces:
|
|
61
|
+
|
|
62
|
+
1. A dict-like interface, where keys are file paths and values are the file contents as bytes.
|
|
63
|
+
For automatic encoding/decoding based on file extension, wrap with :class:`DecodedView`.
|
|
64
|
+
2. A filesystem-like interface consisting of methods such as :meth:`open`, :meth:`exists`, \
|
|
65
|
+
:meth:`listdir`, :meth:`walk`, :meth:`glob`, etc., modeled after Python's ``os`` module.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
path: Path to the Barecat archive (e.g., 'archive.barecat'). Shards are stored
|
|
69
|
+
alongside as 'archive.barecat-shard-XXXXX'.
|
|
70
|
+
shard_size_limit: Maximum size of each shard file in bytes (int) or as a string like '1G'.
|
|
71
|
+
If None, the shard size is unlimited.
|
|
72
|
+
readonly: If True, the Barecat archive is opened in read-only mode.
|
|
73
|
+
overwrite: If True, the Barecat archive is first deleted if it already exists.
|
|
74
|
+
auto_codec: **Deprecated.** Use :class:`DecodedView` instead. Will be removed in 1.0.
|
|
75
|
+
exist_ok: If True, do not raise an error if the Barecat archive already exists.
|
|
76
|
+
append_only: If True, only allow appending to the Barecat archive.
|
|
77
|
+
threadsafe: If True, the Barecat archive is opened in thread-safe mode, where each thread
|
|
78
|
+
or process will hold its own database connection and file handles for the shards.
|
|
79
|
+
allow_writing_symlinked_shard: If True, allow writing to a shard file that is a symlink.
|
|
80
|
+
Setting it to False is recommended, since changing the contents of a symlinked shard
|
|
81
|
+
will bring the original index database out of sync with the actual shard contents.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(
|
|
85
|
+
self,
|
|
86
|
+
path: Union[str, os.PathLike],
|
|
87
|
+
shard_size_limit: Union[int, str, None] = None,
|
|
88
|
+
readonly: bool = True,
|
|
89
|
+
overwrite: bool = False,
|
|
90
|
+
auto_codec: bool = False,
|
|
91
|
+
exist_ok: bool = True,
|
|
92
|
+
append_only: bool = False,
|
|
93
|
+
threadsafe: bool = False,
|
|
94
|
+
allow_writing_symlinked_shard: bool = False,
|
|
95
|
+
wal: bool = False,
|
|
96
|
+
readonly_is_immutable: bool = False,
|
|
97
|
+
):
|
|
98
|
+
path = os.fspath(path)
|
|
99
|
+
if threadsafe and not readonly:
|
|
100
|
+
raise ValueError('Threadsafe mode is only supported for readonly Barecat.')
|
|
101
|
+
|
|
102
|
+
if not readonly and barecat_util.exists(path):
|
|
103
|
+
if not exist_ok:
|
|
104
|
+
raise FileExistsError(path)
|
|
105
|
+
if overwrite:
|
|
106
|
+
logger.info('Overwriting existing Barecat at %s', path)
|
|
107
|
+
barecat_util.remove(path)
|
|
108
|
+
|
|
109
|
+
if readonly and not barecat_util.exists(path):
|
|
110
|
+
raise FileNotFoundError(path)
|
|
111
|
+
|
|
112
|
+
self.path = path
|
|
113
|
+
self.readonly = readonly
|
|
114
|
+
self.append_only = append_only
|
|
115
|
+
self.auto_codec = auto_codec
|
|
116
|
+
self.threadsafe = threadsafe
|
|
117
|
+
self.allow_writing_symlinked_shard = allow_writing_symlinked_shard
|
|
118
|
+
self.readonly_is_immutable = readonly_is_immutable
|
|
119
|
+
self.wal = wal
|
|
120
|
+
|
|
121
|
+
# Index
|
|
122
|
+
self._index_storage = ThreadLocalStorage(threadsafe)
|
|
123
|
+
|
|
124
|
+
if not readonly and shard_size_limit is not None:
|
|
125
|
+
self.shard_size_limit = shard_size_limit
|
|
126
|
+
|
|
127
|
+
# Shards
|
|
128
|
+
self.sharder = Sharder(
|
|
129
|
+
path,
|
|
130
|
+
shard_size_limit=self.shard_size_limit,
|
|
131
|
+
append_only=append_only,
|
|
132
|
+
readonly=readonly,
|
|
133
|
+
threadsafe=threadsafe,
|
|
134
|
+
allow_writing_symlinked_shard=allow_writing_symlinked_shard,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
self.codec_registry = barecat_codecs.CodecRegistry(auto_codec=auto_codec)
|
|
138
|
+
self._merge_helper = BarecatMergeHelper(self)
|
|
139
|
+
self._fileobj_helper = BarecatFileObjectHelper(self)
|
|
140
|
+
|
|
141
|
+
if auto_codec:
|
|
142
|
+
warnings.warn(
|
|
143
|
+
'auto_codec is deprecated and will be removed in version 1.0. '
|
|
144
|
+
"Use DecodedView instead: dec = DecodedView(bc); dec['file.json'] = data",
|
|
145
|
+
DeprecationWarning,
|
|
146
|
+
stacklevel=2,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
## Dict-like API: keys are filepaths, values are the file contents (bytes or decoded objects)
|
|
150
|
+
def __getitem__(self, path: str) -> Union[bytes, Any]:
|
|
151
|
+
"""Get the contents of a file in the Barecat archive.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
path: Path to the file within the archive.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
The contents of the file. Either raw bytes, or decoded based on the file extension, if
|
|
158
|
+
``auto_codec`` was True in the constructor, or
|
|
159
|
+
if codecs have been registered for the file extension via ``register_codec``.
|
|
160
|
+
|
|
161
|
+
Raises:
|
|
162
|
+
KeyError: If a file with this path does not exist in the archive.
|
|
163
|
+
|
|
164
|
+
Examples:
|
|
165
|
+
|
|
166
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
167
|
+
>>> bc['file.txt'] = b'Hello, world!'
|
|
168
|
+
>>> bc['file.txt']
|
|
169
|
+
b'Hello, world!'
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
# Typically used in training loop
|
|
173
|
+
path = normalize_path(path)
|
|
174
|
+
row = self.index.fetch_one(
|
|
175
|
+
'SELECT shard, offset, size, crc32c FROM files WHERE path=?', (path,)
|
|
176
|
+
)
|
|
177
|
+
if row is None:
|
|
178
|
+
raise KeyError(path)
|
|
179
|
+
raw_data = self.sharder.read_from_address(
|
|
180
|
+
row['shard'], row['offset'], row['size'], row['crc32c']
|
|
181
|
+
)
|
|
182
|
+
return self.codec_registry.decode(path, raw_data)
|
|
183
|
+
|
|
184
|
+
def get(self, path: str, default: Any = None) -> Union[bytes, Any]:
|
|
185
|
+
"""Get the contents of a file in the Barecat archive, with a default value if the file does
|
|
186
|
+
not exist.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
path: Path to the file within the archive.
|
|
190
|
+
default: Default value to return if the file does not exist.
|
|
191
|
+
|
|
192
|
+
Returns:
|
|
193
|
+
The contents of the file (possibly decoded), or the default value if the file does not
|
|
194
|
+
exist.
|
|
195
|
+
"""
|
|
196
|
+
try:
|
|
197
|
+
return self[path]
|
|
198
|
+
except KeyError:
|
|
199
|
+
return default
|
|
200
|
+
|
|
201
|
+
def items(self) -> Iterator[tuple[str, Union[bytes, Any]]]:
|
|
202
|
+
"""Iterate over all files in the archive, yielding (path, content) pairs.
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
Iterator over (path, content) pairs.
|
|
206
|
+
"""
|
|
207
|
+
for finfo in self.index.iter_all_fileinfos():
|
|
208
|
+
data = self.read(finfo)
|
|
209
|
+
yield finfo.path, self.codec_registry.decode(finfo.path, data)
|
|
210
|
+
|
|
211
|
+
def keys(self) -> Iterator[str]:
|
|
212
|
+
"""Iterate over all file paths in the archive.
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
Iterator over file paths.
|
|
216
|
+
"""
|
|
217
|
+
for finfo in self.index.iter_all_fileinfos():
|
|
218
|
+
yield finfo.path
|
|
219
|
+
|
|
220
|
+
def values(self) -> Iterator[Union[bytes, Any]]:
|
|
221
|
+
"""Iterate over all file contents in the archive.
|
|
222
|
+
|
|
223
|
+
Returns:
|
|
224
|
+
Iterator over file contents, possibly decoded based on the file extension.
|
|
225
|
+
"""
|
|
226
|
+
for key, value in self.items():
|
|
227
|
+
yield value
|
|
228
|
+
|
|
229
|
+
def __contains__(self, path: str) -> bool:
|
|
230
|
+
"""Check if a file with the given path exists in the archive.
|
|
231
|
+
|
|
232
|
+
Directories are ignored in this check.
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
path: Path to the file within the archive.
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
True if the file exists, False otherwise.
|
|
239
|
+
"""
|
|
240
|
+
return self.index.isfile(path)
|
|
241
|
+
|
|
242
|
+
def __len__(self) -> int:
|
|
243
|
+
"""Get the number of files in the archive.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
Number of files in the archive.
|
|
247
|
+
"""
|
|
248
|
+
return self.index.num_files
|
|
249
|
+
|
|
250
|
+
def __iter__(self) -> Iterator[str]:
|
|
251
|
+
"""Iterate over all file paths in the archive.
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
Iterator over file paths.
|
|
255
|
+
"""
|
|
256
|
+
return self.files()
|
|
257
|
+
|
|
258
|
+
def __setitem__(self, path: str, content: Union[bytes, Any]):
|
|
259
|
+
"""Add a file to the Barecat archive.
|
|
260
|
+
|
|
261
|
+
Args:
|
|
262
|
+
path: Path to the file within the archive.
|
|
263
|
+
content: Contents of the file. Either raw bytes, or an object to be encoded based on the
|
|
264
|
+
file extension, if ``auto_codec`` was True in the constructor, or if codecs have
|
|
265
|
+
been registered for the file extension via :meth:`register_codec`.
|
|
266
|
+
|
|
267
|
+
Raises:
|
|
268
|
+
ValueError: If the archive is read-only.
|
|
269
|
+
FileExistsBarecatError: If a file or directory with the given path already exists in the
|
|
270
|
+
archive.
|
|
271
|
+
|
|
272
|
+
Examples:
|
|
273
|
+
|
|
274
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
275
|
+
>>> bc['file.txt'] = b'Hello, world!'
|
|
276
|
+
>>> bc['file.txt']
|
|
277
|
+
b'Hello, world!'
|
|
278
|
+
|
|
279
|
+
"""
|
|
280
|
+
|
|
281
|
+
self.add(path, data=self.codec_registry.encode(path, content))
|
|
282
|
+
|
|
283
|
+
def setdefault(self, key: str, default: Any = None, /):
|
|
284
|
+
try:
|
|
285
|
+
return self[key]
|
|
286
|
+
except KeyError:
|
|
287
|
+
self[key] = default
|
|
288
|
+
return default
|
|
289
|
+
|
|
290
|
+
def __delitem__(self, path: str):
|
|
291
|
+
"""Remove a file from the Barecat archive.
|
|
292
|
+
|
|
293
|
+
Args:
|
|
294
|
+
path: Path to the file within the archive.
|
|
295
|
+
|
|
296
|
+
Raises:
|
|
297
|
+
KeyError: If a file with this path does not exist in the archive.
|
|
298
|
+
|
|
299
|
+
Examples:
|
|
300
|
+
|
|
301
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
302
|
+
>>> bc['file.txt'] = b'Hello, world!'
|
|
303
|
+
>>> bc['file.txt']
|
|
304
|
+
b'Hello, world!'
|
|
305
|
+
>>> del bc['file.txt']
|
|
306
|
+
>>> bc['file.txt']
|
|
307
|
+
Traceback (most recent call last):
|
|
308
|
+
...
|
|
309
|
+
KeyError: 'file.txt'
|
|
310
|
+
|
|
311
|
+
"""
|
|
312
|
+
try:
|
|
313
|
+
self.remove(path)
|
|
314
|
+
except (FileNotFoundBarecatError, IsADirectoryBarecatError):
|
|
315
|
+
raise KeyError(path)
|
|
316
|
+
|
|
317
|
+
# Filesystem-like API
|
|
318
|
+
def open(
|
|
319
|
+
self,
|
|
320
|
+
item: Union[BarecatFileInfo, str],
|
|
321
|
+
mode: str = 'r',
|
|
322
|
+
encoding: Optional[str] = None,
|
|
323
|
+
errors: Optional[str] = None,
|
|
324
|
+
newline: Optional[str] = None,
|
|
325
|
+
) -> Union[BarecatFileObject, io.TextIOWrapper]:
|
|
326
|
+
"""Open a file in the archive, as a file-like object.
|
|
327
|
+
|
|
328
|
+
Args:
|
|
329
|
+
item: Either a BarecatFileInfo object, or a path to a file within the archive.
|
|
330
|
+
mode: Mode to open the file in.
|
|
331
|
+
'r'/'rt'/'rb' - read only, file must exist
|
|
332
|
+
'r+'/'r+b' - read/write, file must exist
|
|
333
|
+
'w'/'wt'/'wb' - write, truncate if exists, create if not
|
|
334
|
+
'w+'/'w+b' - read/write, truncate if exists, create if not
|
|
335
|
+
'x'/'xb' - exclusive create, fail if exists
|
|
336
|
+
'x+'/'x+b' - exclusive create read/write, fail if exists
|
|
337
|
+
'a'/'ab' - append, create if not exists
|
|
338
|
+
'a+'/'a+b' - append read/write, create if not exists
|
|
339
|
+
encoding: Text encoding (only for text mode, default 'utf-8').
|
|
340
|
+
errors: Error handling for encoding (only for text mode).
|
|
341
|
+
newline: Newline handling (only for text mode).
|
|
342
|
+
|
|
343
|
+
Returns:
|
|
344
|
+
File-like object representing the file. Returns TextIOWrapper for text
|
|
345
|
+
mode, BarecatFileObject for binary mode.
|
|
346
|
+
|
|
347
|
+
Raises:
|
|
348
|
+
FileNotFoundBarecatError: If file doesn't exist and mode requires it.
|
|
349
|
+
FileExistsBarecatError: If file exists and mode is exclusive create.
|
|
350
|
+
"""
|
|
351
|
+
return self._fileobj_helper.open(item, mode, encoding, errors, newline)
|
|
352
|
+
|
|
353
|
+
def exists(self, path: str) -> bool:
|
|
354
|
+
"""Check if a file or directory exists in the archive.
|
|
355
|
+
|
|
356
|
+
Args:
|
|
357
|
+
path: Path to the file or directory within the archive.
|
|
358
|
+
|
|
359
|
+
Returns:
|
|
360
|
+
True if and only if a file or directory exists with the given path.
|
|
361
|
+
"""
|
|
362
|
+
return self.index.exists(path)
|
|
363
|
+
|
|
364
|
+
def isfile(self, path):
|
|
365
|
+
"""Check if a file exists in the archive.
|
|
366
|
+
|
|
367
|
+
Args:
|
|
368
|
+
path: Path to the file within the archive.
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
True if and only if a file exists with the given path.
|
|
372
|
+
"""
|
|
373
|
+
return self.index.isfile(path)
|
|
374
|
+
|
|
375
|
+
def isdir(self, path):
|
|
376
|
+
"""Check if a directory exists in the archive.
|
|
377
|
+
|
|
378
|
+
Args:
|
|
379
|
+
path: Path to the directory within the archive.
|
|
380
|
+
|
|
381
|
+
Returns:
|
|
382
|
+
True if and only if a directory exists with the given path.
|
|
383
|
+
"""
|
|
384
|
+
return self.index.isdir(path)
|
|
385
|
+
|
|
386
|
+
def listdir(self, path: str) -> list[str]:
|
|
387
|
+
"""List all files and directories in a directory.
|
|
388
|
+
|
|
389
|
+
Args:
|
|
390
|
+
path: Path to the directory within the archive.
|
|
391
|
+
|
|
392
|
+
Returns:
|
|
393
|
+
List of all files and directories contained in the directory ``path``.
|
|
394
|
+
"""
|
|
395
|
+
return self.index.listdir_names(path)
|
|
396
|
+
|
|
397
|
+
def walk(self, path: str) -> Iterator[tuple[str, list[str], list[str]]]:
|
|
398
|
+
"""Recursively list all files and directories in the tree starting from a directory.
|
|
399
|
+
|
|
400
|
+
This is analogous to Python's :py:func:`os.walk`.
|
|
401
|
+
|
|
402
|
+
Args:
|
|
403
|
+
path: Path to the directory within the archive.
|
|
404
|
+
|
|
405
|
+
Returns:
|
|
406
|
+
Iterator over (dirpath, dirnames, filenames) tuples, where ``dirpath`` is the path to
|
|
407
|
+
the directory, ``dirnames`` is a list of all subdirectory names, and ``filenames`` is
|
|
408
|
+
a list of all filenames in the directory.
|
|
409
|
+
|
|
410
|
+
Examples:
|
|
411
|
+
|
|
412
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
413
|
+
>>> bc['dir/file.txt'] = b'Hello, world!'
|
|
414
|
+
>>> bc['dir/subdir/file2.txt'] = b'Hello, world2!'
|
|
415
|
+
>>> for dirpath, dirnames, filenames in bc.walk('dir'):
|
|
416
|
+
... print(dirpath, dirnames, filenames)
|
|
417
|
+
dir ['subdir'] ['file.txt']
|
|
418
|
+
dir/subdir [] ['file2.txt']
|
|
419
|
+
|
|
420
|
+
"""
|
|
421
|
+
return self.index.walk_names(path)
|
|
422
|
+
|
|
423
|
+
def scandir(self, path: str) -> Iterator[BarecatEntryInfo]:
|
|
424
|
+
"""Iterate over all immediate files and subdirectories of the given directory, as :class:`barecat.BarecatEntryInfo` objects.
|
|
425
|
+
|
|
426
|
+
Args:
|
|
427
|
+
path: Path to the directory within the archive.
|
|
428
|
+
|
|
429
|
+
Returns:
|
|
430
|
+
An iterator over members of the directory, as :class:`barecat.BarecatEntryInfo` objects.
|
|
431
|
+
"""
|
|
432
|
+
return self.index.iterdir_infos(path)
|
|
433
|
+
|
|
434
|
+
def glob(
|
|
435
|
+
self, pattern: str, recursive: bool = False, include_hidden: bool = False
|
|
436
|
+
) -> list[str]:
|
|
437
|
+
"""Find all files and directories matching a Unix-like glob pattern.
|
|
438
|
+
|
|
439
|
+
This function is equivalent to Python's :py:func:`glob.glob`.
|
|
440
|
+
|
|
441
|
+
Args:
|
|
442
|
+
pattern: Unix-like glob pattern to match.
|
|
443
|
+
recursive: If True, search recursively, with ``'/**/'`` matching any number of
|
|
444
|
+
directories.
|
|
445
|
+
include_hidden: If True, include hidden files and directories (starting with ``"."``).
|
|
446
|
+
|
|
447
|
+
Returns:
|
|
448
|
+
List of all file and directory paths matching the pattern.
|
|
449
|
+
|
|
450
|
+
Examples:
|
|
451
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
452
|
+
>>> bc['dir/file.txt'] = b'Hello, world!'
|
|
453
|
+
>>> bc['dir/subdir/file2.txt'] = b'Hello, world2!'
|
|
454
|
+
>>> bc.glob('dir/**/*.txt', recursive=True)
|
|
455
|
+
['dir/file.txt', 'dir/subdir/file2.txt']
|
|
456
|
+
"""
|
|
457
|
+
return self.index.glob_paths(pattern, recursive, include_hidden)
|
|
458
|
+
|
|
459
|
+
def globfiles(
|
|
460
|
+
self, pattern: str, recursive: bool = False, include_hidden: bool = False
|
|
461
|
+
) -> list[str]:
|
|
462
|
+
"""Find all files matching a Unix-like glob pattern.
|
|
463
|
+
|
|
464
|
+
Like ``glob``, but only returns files, not directories.
|
|
465
|
+
|
|
466
|
+
Args:
|
|
467
|
+
pattern: Unix-like glob pattern to match.
|
|
468
|
+
recursive: If True, search recursively, with ``'/**/'`` matching any number of
|
|
469
|
+
directories.
|
|
470
|
+
include_hidden: If True, include hidden files (starting with ``"."``).
|
|
471
|
+
|
|
472
|
+
Returns:
|
|
473
|
+
List of all file paths matching the pattern.
|
|
474
|
+
"""
|
|
475
|
+
return self.index.glob_paths(pattern, recursive, include_hidden, only_files=True)
|
|
476
|
+
|
|
477
|
+
def iglob(
|
|
478
|
+
self, pattern: str, recursive: bool = False, include_hidden: bool = False
|
|
479
|
+
) -> Iterator[str]:
|
|
480
|
+
"""Iterate over all files and directories matching a Unix-like glob pattern.
|
|
481
|
+
|
|
482
|
+
This function is equivalent to Python's :py:func:`glob.iglob`.
|
|
483
|
+
|
|
484
|
+
Args:
|
|
485
|
+
pattern: Unix-like glob pattern to match.
|
|
486
|
+
recursive: If True, search recursively, with ``'/**/'`` matching any number of
|
|
487
|
+
directories.
|
|
488
|
+
include_hidden: If True, include hidden files and directories (starting with ``'.'``).
|
|
489
|
+
|
|
490
|
+
Returns:
|
|
491
|
+
Iterator over all file and directory paths matching the pattern.
|
|
492
|
+
"""
|
|
493
|
+
return self.index.iterglob_paths(pattern, recursive, include_hidden)
|
|
494
|
+
|
|
495
|
+
def iglobfiles(
|
|
496
|
+
self, pattern: str, recursive: bool = False, include_hidden: bool = False
|
|
497
|
+
) -> Iterator[str]:
|
|
498
|
+
"""Iterate over all files matching a Unix-like glob pattern.
|
|
499
|
+
|
|
500
|
+
Like ``iglob``, but only returns files, not directories.
|
|
501
|
+
|
|
502
|
+
Args:
|
|
503
|
+
pattern: Unix-like glob pattern to match.
|
|
504
|
+
recursive: If True, search recursively, with ``'/**/'`` matching any number of
|
|
505
|
+
directories.
|
|
506
|
+
include_hidden: If True, include hidden files (starting with ``"."``).
|
|
507
|
+
|
|
508
|
+
Returns:
|
|
509
|
+
Iterator over all file paths matching the pattern.
|
|
510
|
+
"""
|
|
511
|
+
return self.index.iterglob_paths(pattern, recursive, include_hidden, only_files=True)
|
|
512
|
+
|
|
513
|
+
def files(self) -> Iterator[str]:
|
|
514
|
+
"""Iterate over all file paths in the archive.
|
|
515
|
+
|
|
516
|
+
Returns:
|
|
517
|
+
Iterator over file paths.
|
|
518
|
+
"""
|
|
519
|
+
return self.index.iter_all_filepaths()
|
|
520
|
+
|
|
521
|
+
def dirs(self) -> Iterator[str]:
|
|
522
|
+
"""Iterate over all directory paths in the archive.
|
|
523
|
+
|
|
524
|
+
Returns:
|
|
525
|
+
Iterator over directory paths.
|
|
526
|
+
"""
|
|
527
|
+
return self.index.iter_all_dirpaths()
|
|
528
|
+
|
|
529
|
+
@property
|
|
530
|
+
def num_files(self) -> int:
|
|
531
|
+
"""The number of files in the archive."""
|
|
532
|
+
return self.index.num_files
|
|
533
|
+
|
|
534
|
+
@property
|
|
535
|
+
def num_dirs(self) -> int:
|
|
536
|
+
"""The number of directories in the archive."""
|
|
537
|
+
return self.index.num_dirs
|
|
538
|
+
|
|
539
|
+
@property
|
|
540
|
+
def total_size(self) -> int:
|
|
541
|
+
"""The total size of all files in the archive, in bytes."""
|
|
542
|
+
return self.index.total_size
|
|
543
|
+
|
|
544
|
+
def readinto(self, item: Union[BarecatFileInfo, str], buffer, offset=0) -> int:
|
|
545
|
+
"""Read a file into a buffer, starting from an offset within the file.
|
|
546
|
+
|
|
547
|
+
Read until either the buffer is full, or the end of the file is reached.
|
|
548
|
+
|
|
549
|
+
Args:
|
|
550
|
+
item: Either a BarecatFileInfo object, or a path to a file within the archive.
|
|
551
|
+
buffer: Destination buffer to read the file into.
|
|
552
|
+
offset: Offset within the file to start reading from.
|
|
553
|
+
|
|
554
|
+
Returns:
|
|
555
|
+
Number of bytes read.
|
|
556
|
+
"""
|
|
557
|
+
|
|
558
|
+
# Used in fuse mount
|
|
559
|
+
if isinstance(item, BarecatFileInfo):
|
|
560
|
+
shard, offset_in_shard, size_in_shard, exp_crc32c = (
|
|
561
|
+
item.shard,
|
|
562
|
+
item.offset,
|
|
563
|
+
item.size,
|
|
564
|
+
item.crc32c,
|
|
565
|
+
)
|
|
566
|
+
else:
|
|
567
|
+
path = normalize_path(item)
|
|
568
|
+
row = self.index.fetch_one(
|
|
569
|
+
'SELECT shard, offset, size, crc32c FROM files WHERE path=?', (path,)
|
|
570
|
+
)
|
|
571
|
+
if row is None:
|
|
572
|
+
raise FileNotFoundBarecatError(path)
|
|
573
|
+
shard, offset_in_shard, size_in_shard, exp_crc32c = row
|
|
574
|
+
|
|
575
|
+
offset = max(0, min(offset, size_in_shard))
|
|
576
|
+
size_to_read = min(len(buffer), size_in_shard - offset)
|
|
577
|
+
|
|
578
|
+
if size_to_read != size_in_shard:
|
|
579
|
+
exp_crc32c = None
|
|
580
|
+
|
|
581
|
+
return self.sharder.readinto_from_address(
|
|
582
|
+
shard, offset_in_shard + offset, memoryview(buffer)[:size_to_read], exp_crc32c
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
def read(self, item: Union[BarecatFileInfo, str], offset: int = 0, size: int = -1) -> bytes:
|
|
586
|
+
"""Read a file from the archive, starting from an offset and reading a specific number of
|
|
587
|
+
bytes.
|
|
588
|
+
|
|
589
|
+
Args:
|
|
590
|
+
item: Either a BarecatFileInfo object, or a path to a file within the archive.
|
|
591
|
+
offset: Offset within the file to start reading from.
|
|
592
|
+
size: Number of bytes to read. If -1, read until the end of the file.
|
|
593
|
+
|
|
594
|
+
Returns:
|
|
595
|
+
The contents of the file, as bytes.
|
|
596
|
+
|
|
597
|
+
Raises:
|
|
598
|
+
ValueError: If the CRC32C checksum of the read data does not match the expected value.
|
|
599
|
+
FileNotFoundBarecatError: If a file with this path does not exist in the archive.
|
|
600
|
+
"""
|
|
601
|
+
finfo = self.index._as_fileinfo(item)
|
|
602
|
+
with self.open(finfo, 'rb') as f:
|
|
603
|
+
f.seek(offset)
|
|
604
|
+
data = f.read(size)
|
|
605
|
+
if offset == 0 and (size == -1 or size == finfo.size) and finfo.crc32c is not None:
|
|
606
|
+
crc32c = crc32c_lib.crc32c(data)
|
|
607
|
+
if crc32c != finfo.crc32c:
|
|
608
|
+
raise ValueError(
|
|
609
|
+
f'CRC32C mismatch for {finfo.path}. Expected {finfo.crc32c}, got {crc32c}'
|
|
610
|
+
)
|
|
611
|
+
return data
|
|
612
|
+
|
|
613
|
+
# WRITING
|
|
614
|
+
@raise_if_readonly
|
|
615
|
+
def add_by_path(
|
|
616
|
+
self, filesys_path: str, store_path: Optional[str] = None, dir_exist_ok: bool = False
|
|
617
|
+
):
|
|
618
|
+
"""Add a file or directory from the filesystem to the archive.
|
|
619
|
+
|
|
620
|
+
Args:
|
|
621
|
+
filesys_path: Path to the file or directory on the filesystem.
|
|
622
|
+
store_path: Path to store the file or directory in the archive. If None, the same path
|
|
623
|
+
is used as ``filesys_path``.
|
|
624
|
+
dir_exist_ok: If True, do not raise an error when adding a directory and that
|
|
625
|
+
directory already exists in the archive (as a directory).
|
|
626
|
+
|
|
627
|
+
Raises:
|
|
628
|
+
ValueError: If the file is larger than the shard size limit.
|
|
629
|
+
FileExistsBarecatError: If a file or directory with the same path already exists in the
|
|
630
|
+
archive, unless ``dir_exist_ok`` is True and the item is a directory.
|
|
631
|
+
|
|
632
|
+
Examples:
|
|
633
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
634
|
+
>>> bc.add_by_path('file.txt')
|
|
635
|
+
>>> bc.add_by_path('dir', store_path='dir2')
|
|
636
|
+
"""
|
|
637
|
+
|
|
638
|
+
if store_path is None:
|
|
639
|
+
store_path = filesys_path
|
|
640
|
+
|
|
641
|
+
statresult = os.stat(filesys_path)
|
|
642
|
+
if stat.S_ISDIR(statresult.st_mode):
|
|
643
|
+
finfo = BarecatDirInfo(path=store_path)
|
|
644
|
+
finfo.fill_from_statresult(statresult)
|
|
645
|
+
self.index.add_dir(finfo, exist_ok=dir_exist_ok)
|
|
646
|
+
return
|
|
647
|
+
|
|
648
|
+
finfo = BarecatFileInfo(path=store_path)
|
|
649
|
+
finfo.fill_from_statresult(statresult)
|
|
650
|
+
with open(filesys_path, 'rb') as in_file:
|
|
651
|
+
self.add(finfo, fileobj=in_file)
|
|
652
|
+
|
|
653
|
+
@raise_if_readonly
|
|
654
|
+
def add(
|
|
655
|
+
self,
|
|
656
|
+
item: Union[BarecatEntryInfo, str],
|
|
657
|
+
*,
|
|
658
|
+
data: Optional[bytes] = None,
|
|
659
|
+
fileobj=None,
|
|
660
|
+
bufsize: int = shutil.COPY_BUFSIZE,
|
|
661
|
+
dir_exist_ok: bool = False,
|
|
662
|
+
file_exist_ok: bool = False,
|
|
663
|
+
):
|
|
664
|
+
"""Add a file or directory to the archive.
|
|
665
|
+
|
|
666
|
+
Parent directories are automatically created if they don't exist (like ``mkdir -p``).
|
|
667
|
+
|
|
668
|
+
Args:
|
|
669
|
+
item: BarecatFileInfo or BarecatDirInfo object to add or a target path for a file.
|
|
670
|
+
data: File content. If None, the data is read from the file object.
|
|
671
|
+
fileobj: File-like object to read the data from.
|
|
672
|
+
bufsize: Buffer size to use when reading from the file object.
|
|
673
|
+
dir_exist_ok: If True, do not raise an error when adding a directory and that
|
|
674
|
+
directory already exists in the archive (as a directory).
|
|
675
|
+
file_exist_ok: If True, skip adding a file if a file with the same path already
|
|
676
|
+
exists in the archive. This is useful for merge operations.
|
|
677
|
+
|
|
678
|
+
Raises:
|
|
679
|
+
ValueError: If the file is larger than the shard size limit.
|
|
680
|
+
FileExistsBarecatError: If a file or directory with the same path already exists in the
|
|
681
|
+
archive, unless ``dir_exist_ok`` is True and the item is a directory, or
|
|
682
|
+
``file_exist_ok`` is True and the item is a file.
|
|
683
|
+
NotADirectoryBarecatError: If a parent path exists as a file.
|
|
684
|
+
|
|
685
|
+
Examples:
|
|
686
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
687
|
+
>>> bc.add(BarecatFileInfo(path='file.txt', mode=0o666), data=b'Hello, world!')
|
|
688
|
+
>>> bc.add(BarecatDirInfo(path='dir', mode=0o777))
|
|
689
|
+
"""
|
|
690
|
+
if isinstance(item, BarecatDirInfo):
|
|
691
|
+
self.index.add_dir(item, exist_ok=dir_exist_ok)
|
|
692
|
+
return
|
|
693
|
+
|
|
694
|
+
finfo = BarecatFileInfo(path=item) if isinstance(item, str) else item
|
|
695
|
+
|
|
696
|
+
# Check if file exists before writing data (to avoid wasted writes)
|
|
697
|
+
if file_exist_ok and finfo.path in self:
|
|
698
|
+
return
|
|
699
|
+
|
|
700
|
+
finfo.shard, finfo.offset, finfo.size, finfo.crc32c = self.sharder.add(
|
|
701
|
+
size=finfo.size, data=data, fileobj=fileobj, bufsize=bufsize
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
try:
|
|
705
|
+
self.index.add_file(finfo)
|
|
706
|
+
except FileExistsBarecatError:
|
|
707
|
+
# If the file already exists, we need to truncate the shard file back
|
|
708
|
+
shard_file = self.sharder.shard_files[finfo.shard]
|
|
709
|
+
with open(shard_file.name, 'r+b') as f:
|
|
710
|
+
f.truncate(finfo.offset)
|
|
711
|
+
raise
|
|
712
|
+
|
|
713
|
+
@raise_if_readonly_or_append_only
|
|
714
|
+
def update_file(
|
|
715
|
+
self,
|
|
716
|
+
old_item: Union[BarecatFileInfo, str],
|
|
717
|
+
new_item: Union[BarecatFileInfo, str, None] = None,
|
|
718
|
+
*,
|
|
719
|
+
data: Optional[bytes] = None,
|
|
720
|
+
fileobj=None,
|
|
721
|
+
size: Optional[int] = None,
|
|
722
|
+
bufsize: int = shutil.COPY_BUFSIZE,
|
|
723
|
+
):
|
|
724
|
+
"""Update an existing file's data and/or metadata, reusing space if new data fits.
|
|
725
|
+
|
|
726
|
+
If new data is smaller or equal to old data, writes in-place at the
|
|
727
|
+
existing location, avoiding fragmentation. If new data is larger,
|
|
728
|
+
writes at a new location (old location becomes a gap).
|
|
729
|
+
|
|
730
|
+
Args:
|
|
731
|
+
old_item: The existing file to update. Either a BarecatFileInfo (if user
|
|
732
|
+
already looked it up) or a path string.
|
|
733
|
+
new_item: Optional new metadata. If BarecatFileInfo, its metadata fields
|
|
734
|
+
(path, mtime_ns, mode, uid, gid) are the new values. If string,
|
|
735
|
+
it's the new path (rename). If None, only data is updated.
|
|
736
|
+
data: New file content. Either data or fileobj must be provided.
|
|
737
|
+
fileobj: File-like object to read new data from.
|
|
738
|
+
size: Size of new data. Required for fileobj if writing in-place.
|
|
739
|
+
bufsize: Buffer size for reading from fileobj.
|
|
740
|
+
|
|
741
|
+
Raises:
|
|
742
|
+
FileNotFoundBarecatError: If file does not exist.
|
|
743
|
+
ValueError: If neither data nor fileobj provided, or both provided.
|
|
744
|
+
"""
|
|
745
|
+
|
|
746
|
+
old_info = self.index._as_fileinfo(old_item)
|
|
747
|
+
|
|
748
|
+
# Extract new metadata from new_item
|
|
749
|
+
if isinstance(new_item, BarecatFileInfo):
|
|
750
|
+
new_mtime_ns = new_item.mtime_ns
|
|
751
|
+
new_mode = new_item.mode
|
|
752
|
+
new_uid = new_item.uid
|
|
753
|
+
new_gid = new_item.gid
|
|
754
|
+
else:
|
|
755
|
+
new_mtime_ns = None
|
|
756
|
+
new_mode = None
|
|
757
|
+
new_uid = None
|
|
758
|
+
new_gid = None
|
|
759
|
+
|
|
760
|
+
if data is not None:
|
|
761
|
+
size = len(data)
|
|
762
|
+
|
|
763
|
+
if size is not None and size <= old_info.size:
|
|
764
|
+
# Fits in existing space - write in-place
|
|
765
|
+
shard, offset, size, crc32c = self.sharder.add(
|
|
766
|
+
shard=old_info.shard,
|
|
767
|
+
offset=old_info.offset,
|
|
768
|
+
size=size,
|
|
769
|
+
data=data,
|
|
770
|
+
fileobj=fileobj,
|
|
771
|
+
bufsize=bufsize,
|
|
772
|
+
raise_if_cannot_fit=True,
|
|
773
|
+
)
|
|
774
|
+
self.index.update_file(
|
|
775
|
+
old_info.path,
|
|
776
|
+
new_size=size,
|
|
777
|
+
new_crc32c=crc32c,
|
|
778
|
+
new_mtime_ns=new_mtime_ns,
|
|
779
|
+
new_mode=new_mode,
|
|
780
|
+
new_uid=new_uid,
|
|
781
|
+
new_gid=new_gid,
|
|
782
|
+
)
|
|
783
|
+
else:
|
|
784
|
+
# Doesn't fit or unknown size - allocate new space, then update index
|
|
785
|
+
shard, offset, size, crc32c = self.sharder.add(
|
|
786
|
+
data=data, fileobj=fileobj, size=size, bufsize=bufsize
|
|
787
|
+
)
|
|
788
|
+
self.index.update_file(
|
|
789
|
+
old_info.path,
|
|
790
|
+
new_shard=shard,
|
|
791
|
+
new_offset=offset,
|
|
792
|
+
new_size=size,
|
|
793
|
+
new_crc32c=crc32c,
|
|
794
|
+
new_mtime_ns=new_mtime_ns,
|
|
795
|
+
new_mode=new_mode,
|
|
796
|
+
new_uid=new_uid,
|
|
797
|
+
new_gid=new_gid,
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
# DIRECTORY CREATION
|
|
801
|
+
@raise_if_readonly
|
|
802
|
+
def mkdir(self, path: str, mode: int = 0o755, exist_ok: bool = False):
|
|
803
|
+
"""Create a directory in the archive.
|
|
804
|
+
|
|
805
|
+
Parent directories are automatically created if they don't exist (like ``mkdir -p``).
|
|
806
|
+
|
|
807
|
+
Args:
|
|
808
|
+
path: Path of the directory to create.
|
|
809
|
+
mode: Permission mode for the directory (default: 0o755).
|
|
810
|
+
exist_ok: If True, do not raise an error if the directory already exists.
|
|
811
|
+
|
|
812
|
+
Raises:
|
|
813
|
+
FileExistsBarecatError: If a file or directory with this path already exists,
|
|
814
|
+
unless exist_ok is True and it's a directory.
|
|
815
|
+
NotADirectoryBarecatError: If the path or a parent exists as a file.
|
|
816
|
+
"""
|
|
817
|
+
self.add(BarecatDirInfo(path=path, mode=mode), dir_exist_ok=exist_ok)
|
|
818
|
+
|
|
819
|
+
# DELETION
|
|
820
|
+
@raise_if_readonly_or_append_only
|
|
821
|
+
def remove(self, item: Union[BarecatFileInfo, str]):
|
|
822
|
+
"""Remove (delete) a file from the archive.
|
|
823
|
+
|
|
824
|
+
Technically, the data is not erased from the shard file at this point, only the
|
|
825
|
+
corresponding row in the index database is removed.
|
|
826
|
+
An exception is when the file is the last file in the shard, in which case the shard file
|
|
827
|
+
is truncated to the end of the file.
|
|
828
|
+
|
|
829
|
+
Args:
|
|
830
|
+
item: Either a BarecatFileInfo object, or a path to a file within the archive.
|
|
831
|
+
|
|
832
|
+
Raises:
|
|
833
|
+
FileNotFoundBarecatError: If a file with this path does not exist in the archive.
|
|
834
|
+
IsADirectoryBarecatError: If the path refers to a directory, not a file.
|
|
835
|
+
"""
|
|
836
|
+
try:
|
|
837
|
+
finfo = self.index._as_fileinfo(item)
|
|
838
|
+
except FileNotFoundBarecatError:
|
|
839
|
+
if self.isdir(item):
|
|
840
|
+
raise IsADirectoryBarecatError(item)
|
|
841
|
+
raise
|
|
842
|
+
|
|
843
|
+
# If this is the last file in the shard, we can just truncate the shard file
|
|
844
|
+
end = finfo.offset + finfo.size
|
|
845
|
+
if (
|
|
846
|
+
end >= self.sharder.shard_files[finfo.shard].tell()
|
|
847
|
+
and end >= osp.getsize(self.sharder.shard_files[finfo.shard].name)
|
|
848
|
+
and end == self.index.logical_shard_end(finfo.shard)
|
|
849
|
+
):
|
|
850
|
+
with open(self.sharder.shard_files[finfo.shard].name, 'r+b') as f:
|
|
851
|
+
f.truncate(finfo.offset)
|
|
852
|
+
self.index.remove_file(finfo)
|
|
853
|
+
|
|
854
|
+
@raise_if_readonly_or_append_only
|
|
855
|
+
def rmdir(self, item: Union[BarecatDirInfo, str]):
|
|
856
|
+
"""Remove (delete) an empty directory from the archive.
|
|
857
|
+
|
|
858
|
+
Args:
|
|
859
|
+
item: Either a BarecatDirInfo object, or a path to a directory within the archive.
|
|
860
|
+
|
|
861
|
+
Raises:
|
|
862
|
+
FileNotFoundBarecatError: If a directory with this path does not exist in the archive.
|
|
863
|
+
DirectoryNotEmptyBarecatError: If the directory is not empty.
|
|
864
|
+
"""
|
|
865
|
+
self.index.remove_empty_dir(item)
|
|
866
|
+
|
|
867
|
+
@raise_if_readonly_or_append_only
|
|
868
|
+
def rmtree(self, item: Union[BarecatDirInfo, str]):
|
|
869
|
+
"""Remove (delete) a directory and all its contents recursively from the archive.
|
|
870
|
+
|
|
871
|
+
Technically, file contents are not erased from the shard file at this point, only the
|
|
872
|
+
corresponding rows in the index database are removed.
|
|
873
|
+
|
|
874
|
+
Args:
|
|
875
|
+
item: Either a BarecatDirInfo object, or a path to a directory within the archive.
|
|
876
|
+
|
|
877
|
+
Raises:
|
|
878
|
+
FileNotFoundBarecatError: If a directory with this path does not exist in the archive.
|
|
879
|
+
"""
|
|
880
|
+
self.index.remove_recursively(item)
|
|
881
|
+
|
|
882
|
+
# RENAMING
|
|
883
|
+
@raise_if_readonly_or_append_only
|
|
884
|
+
def rename(self, old_path: str, new_path: str):
|
|
885
|
+
"""Rename a file or directory in the archive.
|
|
886
|
+
|
|
887
|
+
Args:
|
|
888
|
+
old_path: Path to the file or directory to rename.
|
|
889
|
+
new_path: New path for the file or directory.
|
|
890
|
+
|
|
891
|
+
Raises:
|
|
892
|
+
FileNotFoundBarecatError: If a file or directory with the old path does not exist.
|
|
893
|
+
FileExistsBarecatError: If a file or directory with the new path already exists.
|
|
894
|
+
"""
|
|
895
|
+
self.index.rename(old_path, new_path)
|
|
896
|
+
|
|
897
|
+
@property
|
|
898
|
+
def total_physical_size_seek(self) -> int:
|
|
899
|
+
"""Total size of all shard files, as determined by seeking to the end of the shard files.
|
|
900
|
+
|
|
901
|
+
This is more up-to-date than :meth:`total_physical_size_stat`, but may be slower.
|
|
902
|
+
|
|
903
|
+
Returns:
|
|
904
|
+
Total size of all shard files, in bytes.
|
|
905
|
+
"""
|
|
906
|
+
return self.sharder.total_physical_size_seek
|
|
907
|
+
|
|
908
|
+
@property
|
|
909
|
+
def total_physical_size_stat(self) -> int:
|
|
910
|
+
"""Total size of all shard files, as determined by the file system's `stat` response.
|
|
911
|
+
|
|
912
|
+
This is faster than :meth:`total_physical_size_seek`, but may be less up-to-date.
|
|
913
|
+
|
|
914
|
+
Returns:
|
|
915
|
+
Total size of all shard files, in bytes.
|
|
916
|
+
"""
|
|
917
|
+
return self.sharder.total_physical_size_stat
|
|
918
|
+
|
|
919
|
+
@property
|
|
920
|
+
def total_logical_size(self) -> int:
|
|
921
|
+
"""Total size of all files in the archive, as determined by the index database.
|
|
922
|
+
|
|
923
|
+
Returns:
|
|
924
|
+
Total size of all files in the archive, in bytes.
|
|
925
|
+
"""
|
|
926
|
+
return self.index.total_size
|
|
927
|
+
|
|
928
|
+
# MERGING
|
|
929
|
+
@raise_if_readonly
|
|
930
|
+
def merge_from_other_barecat(
|
|
931
|
+
self,
|
|
932
|
+
source_path: str,
|
|
933
|
+
ignore_duplicates: bool = False,
|
|
934
|
+
prefix: str = '',
|
|
935
|
+
pattern: str = None,
|
|
936
|
+
filter_rules: list = None,
|
|
937
|
+
):
|
|
938
|
+
"""Merge the contents of another Barecat archive into this one.
|
|
939
|
+
|
|
940
|
+
Args:
|
|
941
|
+
source_path: Path to the other Barecat archive.
|
|
942
|
+
ignore_duplicates: If True, do not raise an error when a file with the same path already
|
|
943
|
+
exists in the archive.
|
|
944
|
+
prefix: Path prefix to prepend to all paths (default: '', no prefix).
|
|
945
|
+
pattern: Glob pattern to filter files (uses optimized iterglob_infos).
|
|
946
|
+
filter_rules: Rsync-style include/exclude rules as list of ('+'/'-', pattern) tuples.
|
|
947
|
+
|
|
948
|
+
Raises:
|
|
949
|
+
ValueError: If the shard size limit is set and a file in the source archive is larger
|
|
950
|
+
than the shard size limit.
|
|
951
|
+
"""
|
|
952
|
+
self._merge_helper.merge_from_other_barecat(
|
|
953
|
+
source_path, ignore_duplicates, prefix, pattern, filter_rules
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
@property
|
|
957
|
+
def shard_size_limit(self) -> int:
|
|
958
|
+
"""Maximum size of each shard file."""
|
|
959
|
+
return self.index.shard_size_limit
|
|
960
|
+
|
|
961
|
+
@shard_size_limit.setter
|
|
962
|
+
def shard_size_limit(self, value: Union[int, str]):
|
|
963
|
+
"""Set the maximum size of each shard file.
|
|
964
|
+
|
|
965
|
+
Args:
|
|
966
|
+
value: Size in bytes (int) or as a string like '1G', '500M', '100K'.
|
|
967
|
+
"""
|
|
968
|
+
self.index.shard_size_limit = value
|
|
969
|
+
|
|
970
|
+
def logical_shard_end(self, shard_number: int) -> int:
|
|
971
|
+
"""Logical end of a shard, in bytes, that is the position after the last byte of the last
|
|
972
|
+
file contained in the shard.
|
|
973
|
+
|
|
974
|
+
Args:
|
|
975
|
+
shard_number: Shard number, index starting from 0.
|
|
976
|
+
|
|
977
|
+
Returns:
|
|
978
|
+
Logical end of the shard, in bytes.
|
|
979
|
+
"""
|
|
980
|
+
return self.index.logical_shard_end(shard_number)
|
|
981
|
+
|
|
982
|
+
def physical_shard_end(self, shard_number):
|
|
983
|
+
"""Physical end of a shard, in bytes, that is the end seek position of the shard file.
|
|
984
|
+
|
|
985
|
+
Args:
|
|
986
|
+
shard_number: Shard number, index starting from 0.
|
|
987
|
+
|
|
988
|
+
Returns:
|
|
989
|
+
Physical end of the shard, in bytes.
|
|
990
|
+
"""
|
|
991
|
+
|
|
992
|
+
return self.sharder.physical_shard_end(shard_number)
|
|
993
|
+
|
|
994
|
+
def raise_if_readonly(self, message):
|
|
995
|
+
if self.readonly:
|
|
996
|
+
raise ValueError(message)
|
|
997
|
+
|
|
998
|
+
def raise_if_append_only(self, message):
|
|
999
|
+
if self.append_only:
|
|
1000
|
+
raise ValueError(message)
|
|
1001
|
+
|
|
1002
|
+
# THREADSAFE
|
|
1003
|
+
def _make_index(self):
|
|
1004
|
+
return Index(
|
|
1005
|
+
resolve_index_path(self.path),
|
|
1006
|
+
readonly=self.readonly,
|
|
1007
|
+
wal=self.wal,
|
|
1008
|
+
readonly_is_immutable=self.readonly_is_immutable,
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
@property
|
|
1012
|
+
def index(self) -> Index:
|
|
1013
|
+
"""Index object to manipulate the metadata database of the Barecat archive."""
|
|
1014
|
+
return self._index_storage.get(self._make_index)
|
|
1015
|
+
|
|
1016
|
+
# CONSISTENCY CHECKS
|
|
1017
|
+
def check_crc32c(self, item: Union[BarecatFileInfo, str]):
|
|
1018
|
+
"""Check the CRC32C checksum of a file in the archive.
|
|
1019
|
+
|
|
1020
|
+
Args:
|
|
1021
|
+
item: Either a BarecatFileInfo object, or a path to a file within the archive.
|
|
1022
|
+
|
|
1023
|
+
Returns:
|
|
1024
|
+
True if the CRC32C checksum of the file matches the expected value or no checksum is
|
|
1025
|
+
stored in the database.
|
|
1026
|
+
|
|
1027
|
+
Raises:
|
|
1028
|
+
LookupError: If a file with this path does not exist in the archive.
|
|
1029
|
+
"""
|
|
1030
|
+
|
|
1031
|
+
finfo = self.index._as_fileinfo(item)
|
|
1032
|
+
with self.open(finfo, 'rb') as f:
|
|
1033
|
+
crc32c = accumulate_crc32c(f)
|
|
1034
|
+
if finfo.crc32c is not None and crc32c != finfo.crc32c:
|
|
1035
|
+
logger.warning('CRC32C mismatch for %s. Expected %s, got %s', finfo.path, finfo.crc32c, crc32c)
|
|
1036
|
+
return False
|
|
1037
|
+
return True
|
|
1038
|
+
|
|
1039
|
+
def verify_integrity(self, quick=False):
|
|
1040
|
+
"""Verify the integrity of the Barecat archive.
|
|
1041
|
+
|
|
1042
|
+
This includes checking the CRC32C checksums of all files, and checking the integrity of the
|
|
1043
|
+
index database.
|
|
1044
|
+
|
|
1045
|
+
Args:
|
|
1046
|
+
quick: If True, only check the CRC32C checksums of the last file of the archive.
|
|
1047
|
+
|
|
1048
|
+
Returns:
|
|
1049
|
+
True if no problems were found, False otherwise.
|
|
1050
|
+
"""
|
|
1051
|
+
|
|
1052
|
+
is_good = True
|
|
1053
|
+
if quick:
|
|
1054
|
+
try:
|
|
1055
|
+
if not self.check_crc32c(self.index.get_last_file()):
|
|
1056
|
+
is_good = False
|
|
1057
|
+
except LookupError:
|
|
1058
|
+
pass # no files
|
|
1059
|
+
else:
|
|
1060
|
+
n_printed = 0
|
|
1061
|
+
for fi in barecat_progbar.progressbar(
|
|
1062
|
+
self.index.iter_all_fileinfos(), total=self.num_files
|
|
1063
|
+
):
|
|
1064
|
+
if not self.check_crc32c(fi):
|
|
1065
|
+
is_good = False
|
|
1066
|
+
if n_printed >= 10:
|
|
1067
|
+
logger.warning('... (further mismatches suppressed)')
|
|
1068
|
+
break
|
|
1069
|
+
n_printed += 1
|
|
1070
|
+
|
|
1071
|
+
if not self.index.verify_integrity():
|
|
1072
|
+
is_good = False
|
|
1073
|
+
return is_good
|
|
1074
|
+
|
|
1075
|
+
# CODECS
|
|
1076
|
+
def register_codec(
|
|
1077
|
+
self,
|
|
1078
|
+
exts: list[str],
|
|
1079
|
+
encoder: Callable[[Any], bytes],
|
|
1080
|
+
decoder: Callable[[bytes], Any],
|
|
1081
|
+
nonfinal: bool = False,
|
|
1082
|
+
):
|
|
1083
|
+
"""Register an encoder and decoder for one or more file extensions.
|
|
1084
|
+
|
|
1085
|
+
This allows automatic encoding and decoding (serialization/deserialization) of files based
|
|
1086
|
+
on their extension, used in the dictionary interface, e.g., :meth:`__getitem__`,
|
|
1087
|
+
:meth:`__setitem__` and :meth:`items` methods.
|
|
1088
|
+
|
|
1089
|
+
If ``auto_codec`` was True in the constructor, then the codecs are already
|
|
1090
|
+
registered by default for the following extensions:
|
|
1091
|
+
|
|
1092
|
+
**Data formats:**
|
|
1093
|
+
- ``.json`` — dict/list (stdlib json)
|
|
1094
|
+
- ``.pkl``, ``.pickle`` — any object (pickle)
|
|
1095
|
+
- ``.npy`` — numpy array
|
|
1096
|
+
- ``.npz`` — dict of numpy arrays
|
|
1097
|
+
- ``.msgpack`` — any object (requires msgpack-numpy)
|
|
1098
|
+
|
|
1099
|
+
**Image formats** (uses cv2 > PIL > imageio, whichever is available):
|
|
1100
|
+
- ``.jpg``, ``.jpeg``, ``.png``, ``.bmp``, ``.gif``
|
|
1101
|
+
- ``.tiff``, ``.tif``, ``.webp``, ``.exr``
|
|
1102
|
+
|
|
1103
|
+
**Compression** (stackable with other codecs):
|
|
1104
|
+
- ``.gz``, ``.gzip`` — gzip
|
|
1105
|
+
- ``.xz``, ``.lzma`` — lzma
|
|
1106
|
+
- ``.bz2`` — bzip2
|
|
1107
|
+
|
|
1108
|
+
|
|
1109
|
+
Args:
|
|
1110
|
+
exts: List of file extensions to register the codec for.
|
|
1111
|
+
encoder: Function to encode data into bytes.
|
|
1112
|
+
decoder: Function to decode bytes into data.
|
|
1113
|
+
nonfinal: If True, other codecs are allowed to be applied afterwards in a nested
|
|
1114
|
+
manner. This is useful for, e.g., compression codecs.
|
|
1115
|
+
|
|
1116
|
+
Examples:
|
|
1117
|
+
Simple text encoding:
|
|
1118
|
+
|
|
1119
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
1120
|
+
>>> def encode(data):
|
|
1121
|
+
... return data.encode('utf-8')
|
|
1122
|
+
>>> def decode(data):
|
|
1123
|
+
... return data.decode('utf-8')
|
|
1124
|
+
>>> bc.register_codec(['.txt'], encode, decode)
|
|
1125
|
+
|
|
1126
|
+
Or using a codec from a library:
|
|
1127
|
+
|
|
1128
|
+
>>> import cv2
|
|
1129
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
1130
|
+
>>> def encode_png(data):
|
|
1131
|
+
... return cv2.imencode('.png', data)[1].tobytes()
|
|
1132
|
+
>>> def decode_png(data):
|
|
1133
|
+
... return cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_UNCHANGED)
|
|
1134
|
+
>>> bc.register_codec(['.png'], encode_png, decode_png)
|
|
1135
|
+
|
|
1136
|
+
Or using a compression library:
|
|
1137
|
+
|
|
1138
|
+
>>> import zlib
|
|
1139
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
1140
|
+
>>> def encode_zlib(data):
|
|
1141
|
+
... return zlib.compress(data)
|
|
1142
|
+
>>> def decode_zlib(data):
|
|
1143
|
+
... return zlib.decompress(data)
|
|
1144
|
+
>>> bc.register_codec(['.gz'], encode_zlib, decode_zlib, nonfinal=True)
|
|
1145
|
+
|
|
1146
|
+
Or pickling:
|
|
1147
|
+
|
|
1148
|
+
>>> import pickle
|
|
1149
|
+
>>> bc = Barecat('test.barecat', readonly=False)
|
|
1150
|
+
>>> bc.register_codec(['.pkl'], pickle.dumps, pickle.loads)
|
|
1151
|
+
"""
|
|
1152
|
+
warnings.warn(
|
|
1153
|
+
'register_codec is deprecated and will be removed in version 1.0. '
|
|
1154
|
+
'Use DecodedView instead: dec = DecodedView(bc); dec.register_codec(...)',
|
|
1155
|
+
DeprecationWarning,
|
|
1156
|
+
stacklevel=2,
|
|
1157
|
+
)
|
|
1158
|
+
self.codec_registry.register_codec(exts, encoder, decoder, nonfinal)
|
|
1159
|
+
|
|
1160
|
+
# PICKLING
|
|
1161
|
+
def __reduce__(self):
|
|
1162
|
+
if not self.readonly:
|
|
1163
|
+
raise ValueError('Cannot pickle a non-readonly Barecat')
|
|
1164
|
+
return self.__class__, (
|
|
1165
|
+
self.path,
|
|
1166
|
+
None,
|
|
1167
|
+
True,
|
|
1168
|
+
False,
|
|
1169
|
+
self.auto_codec,
|
|
1170
|
+
True,
|
|
1171
|
+
False,
|
|
1172
|
+
self.threadsafe,
|
|
1173
|
+
)
|
|
1174
|
+
|
|
1175
|
+
def truncate_all_to_logical_size(self):
|
|
1176
|
+
logical_shard_ends = [
|
|
1177
|
+
self.index.logical_shard_end(i) for i in range(self.sharder.num_shards)
|
|
1178
|
+
]
|
|
1179
|
+
self.sharder.truncate_all_to_logical_size(logical_shard_ends)
|
|
1180
|
+
|
|
1181
|
+
# DEFRAG
|
|
1182
|
+
def defrag(self, quick=False):
|
|
1183
|
+
"""Defragment the Barecat archive.
|
|
1184
|
+
|
|
1185
|
+
Args:
|
|
1186
|
+
quick: Perform a faster, but less thorough defragmentation.
|
|
1187
|
+
"""
|
|
1188
|
+
defragger = BarecatDefragger(self)
|
|
1189
|
+
if quick:
|
|
1190
|
+
return defragger.defrag_quick()
|
|
1191
|
+
else:
|
|
1192
|
+
return defragger.defrag()
|
|
1193
|
+
|
|
1194
|
+
def close(self):
|
|
1195
|
+
"""Close the Barecat archive."""
|
|
1196
|
+
self._index_storage.close()
|
|
1197
|
+
self.sharder.close()
|
|
1198
|
+
|
|
1199
|
+
def __repr__(self):
|
|
1200
|
+
mode = 'readonly' if self.readonly else 'read-write'
|
|
1201
|
+
try:
|
|
1202
|
+
n = self.num_files
|
|
1203
|
+
return f"Barecat('{self.path}', {mode}, {n} files)"
|
|
1204
|
+
except Exception:
|
|
1205
|
+
return f"Barecat('{self.path}', closed)"
|
|
1206
|
+
|
|
1207
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
1208
|
+
"""Exit a context manager."""
|
|
1209
|
+
self.close()
|