barecat 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. barecat/__init__.py +108 -0
  2. barecat/__main__.py +5 -0
  3. barecat/_api.py +143 -0
  4. barecat/_version.py +24 -0
  5. barecat/cli/__init__.py +0 -0
  6. barecat/cli/commands.py +540 -0
  7. barecat/cli/completions.py +27 -0
  8. barecat/cli/impl.py +609 -0
  9. barecat/cli/legacy.py +361 -0
  10. barecat/cli/main.py +1355 -0
  11. barecat/cli/shell.py +827 -0
  12. barecat/completions/__init__.py +1 -0
  13. barecat/completions/barecat.bash +246 -0
  14. barecat/completions/barecat.zsh +268 -0
  15. barecat/core/__init__.py +0 -0
  16. barecat/core/barecat.py +1209 -0
  17. barecat/core/index.py +1983 -0
  18. barecat/core/paths.py +61 -0
  19. barecat/core/sharder.py +328 -0
  20. barecat/core/types.py +340 -0
  21. barecat/exceptions.py +92 -0
  22. barecat/formats/__init__.py +0 -0
  23. barecat/formats/archive_formats.py +234 -0
  24. barecat/formats/from_tar_stream.py +53 -0
  25. barecat/formats/to_tar_stream.py +54 -0
  26. barecat/io/__init__.py +5 -0
  27. barecat/io/codecs.py +622 -0
  28. barecat/io/copyfile.py +550 -0
  29. barecat/io/fileobj.py +556 -0
  30. barecat/io/open.py +41 -0
  31. barecat/maintenance/__init__.py +0 -0
  32. barecat/maintenance/complete_database.py +79 -0
  33. barecat/maintenance/defrag.py +541 -0
  34. barecat/maintenance/merge.py +613 -0
  35. barecat/maintenance/reshard.py +153 -0
  36. barecat/maintenance/rsync.py +1282 -0
  37. barecat/maintenance/upgrade_database.py +200 -0
  38. barecat/maintenance/upgrade_database2.py +74 -0
  39. barecat/man/barecat.1 +284 -0
  40. barecat/sql/indexes.sql +6 -0
  41. barecat/sql/schema.sql +55 -0
  42. barecat/sql/triggers.sql +172 -0
  43. barecat/tui/__init__.py +0 -0
  44. barecat/tui/browse.py +448 -0
  45. barecat/tui/ncdu.py +318 -0
  46. barecat/util/__init__.py +0 -0
  47. barecat/util/consumed_threadpool.py +119 -0
  48. barecat/util/glob_helper.py +584 -0
  49. barecat/util/glob_to_regex.py +353 -0
  50. barecat/util/list_files.py +12 -0
  51. barecat/util/misc.py +174 -0
  52. barecat/util/physical_order.py +77 -0
  53. barecat/util/progbar.py +40 -0
  54. barecat/util/threading.py +59 -0
  55. barecat-0.3.2.dist-info/METADATA +276 -0
  56. barecat-0.3.2.dist-info/RECORD +60 -0
  57. barecat-0.3.2.dist-info/WHEEL +5 -0
  58. barecat-0.3.2.dist-info/entry_points.txt +15 -0
  59. barecat-0.3.2.dist-info/licenses/LICENSE +21 -0
  60. barecat-0.3.2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1209 @@
1
+ import io
2
+ import logging
3
+ import os
4
+ import os.path as osp
5
+ import shutil
6
+ import stat
7
+ import warnings
8
+ from collections.abc import Callable, Iterator, MutableMapping
9
+ from contextlib import AbstractContextManager
10
+ from typing import Any, Optional, TYPE_CHECKING, Union
11
+
12
+ from ..io import codecs as barecat_codecs
13
+ from ..util import progbar as barecat_progbar
14
+ from ..util import misc as barecat_util
15
+ import crc32c as crc32c_lib
16
+ from ..core.sharder import Sharder
17
+ from ..io.fileobj import BarecatFileObjectHelper
18
+ from ..maintenance.merge import BarecatMergeHelper
19
+ from ..maintenance.defrag import BarecatDefragger
20
+ from ..exceptions import (
21
+ FileExistsBarecatError,
22
+ FileNotFoundBarecatError,
23
+ IsADirectoryBarecatError,
24
+ )
25
+ from ..util.misc import raise_if_readonly, raise_if_readonly_or_append_only
26
+ from ..util.threading import ThreadLocalStorage
27
+ from ..io.copyfile import accumulate_crc32c
28
+
29
+ if TYPE_CHECKING:
30
+ from barecat import (
31
+ BarecatDirInfo,
32
+ BarecatFileInfo,
33
+ BarecatEntryInfo,
34
+ Index,
35
+ BarecatFileObject,
36
+ )
37
+ from .paths import resolve_index_path
38
+ else:
39
+ from .types import (
40
+ BarecatDirInfo,
41
+ BarecatFileInfo,
42
+ BarecatEntryInfo,
43
+ )
44
+ from ..io.fileobj import (
45
+ BarecatFileObject,
46
+ )
47
+ from .index import Index, normalize_path
48
+ from .paths import resolve_index_path
49
+
50
+ logger = logging.getLogger(__name__)
51
+
52
+
53
+ class Barecat(MutableMapping[str, Any], AbstractContextManager):
54
+ """Object for reading or writing a Barecat archive.
55
+
56
+ A Barecat archive consists of several (large) shard files, each containing the data of multiple
57
+ small files, and an SQLite index database that maps file paths to the corresponding shard,
58
+ offset and size within the shard, as well as metadata such as modification time and checksum.
59
+
60
+ The ``Barecat`` object provides two main interfaces:
61
+
62
+ 1. A dict-like interface, where keys are file paths and values are the file contents as bytes.
63
+ For automatic encoding/decoding based on file extension, wrap with :class:`DecodedView`.
64
+ 2. A filesystem-like interface consisting of methods such as :meth:`open`, :meth:`exists`, \
65
+ :meth:`listdir`, :meth:`walk`, :meth:`glob`, etc., modeled after Python's ``os`` module.
66
+
67
+ Args:
68
+ path: Path to the Barecat archive (e.g., 'archive.barecat'). Shards are stored
69
+ alongside as 'archive.barecat-shard-XXXXX'.
70
+ shard_size_limit: Maximum size of each shard file in bytes (int) or as a string like '1G'.
71
+ If None, the shard size is unlimited.
72
+ readonly: If True, the Barecat archive is opened in read-only mode.
73
+ overwrite: If True, the Barecat archive is first deleted if it already exists.
74
+ auto_codec: **Deprecated.** Use :class:`DecodedView` instead. Will be removed in 1.0.
75
+ exist_ok: If True, do not raise an error if the Barecat archive already exists.
76
+ append_only: If True, only allow appending to the Barecat archive.
77
+ threadsafe: If True, the Barecat archive is opened in thread-safe mode, where each thread
78
+ or process will hold its own database connection and file handles for the shards.
79
+ allow_writing_symlinked_shard: If True, allow writing to a shard file that is a symlink.
80
+ Setting it to False is recommended, since changing the contents of a symlinked shard
81
+ will bring the original index database out of sync with the actual shard contents.
82
+ """
83
+
84
+ def __init__(
85
+ self,
86
+ path: Union[str, os.PathLike],
87
+ shard_size_limit: Union[int, str, None] = None,
88
+ readonly: bool = True,
89
+ overwrite: bool = False,
90
+ auto_codec: bool = False,
91
+ exist_ok: bool = True,
92
+ append_only: bool = False,
93
+ threadsafe: bool = False,
94
+ allow_writing_symlinked_shard: bool = False,
95
+ wal: bool = False,
96
+ readonly_is_immutable: bool = False,
97
+ ):
98
+ path = os.fspath(path)
99
+ if threadsafe and not readonly:
100
+ raise ValueError('Threadsafe mode is only supported for readonly Barecat.')
101
+
102
+ if not readonly and barecat_util.exists(path):
103
+ if not exist_ok:
104
+ raise FileExistsError(path)
105
+ if overwrite:
106
+ logger.info('Overwriting existing Barecat at %s', path)
107
+ barecat_util.remove(path)
108
+
109
+ if readonly and not barecat_util.exists(path):
110
+ raise FileNotFoundError(path)
111
+
112
+ self.path = path
113
+ self.readonly = readonly
114
+ self.append_only = append_only
115
+ self.auto_codec = auto_codec
116
+ self.threadsafe = threadsafe
117
+ self.allow_writing_symlinked_shard = allow_writing_symlinked_shard
118
+ self.readonly_is_immutable = readonly_is_immutable
119
+ self.wal = wal
120
+
121
+ # Index
122
+ self._index_storage = ThreadLocalStorage(threadsafe)
123
+
124
+ if not readonly and shard_size_limit is not None:
125
+ self.shard_size_limit = shard_size_limit
126
+
127
+ # Shards
128
+ self.sharder = Sharder(
129
+ path,
130
+ shard_size_limit=self.shard_size_limit,
131
+ append_only=append_only,
132
+ readonly=readonly,
133
+ threadsafe=threadsafe,
134
+ allow_writing_symlinked_shard=allow_writing_symlinked_shard,
135
+ )
136
+
137
+ self.codec_registry = barecat_codecs.CodecRegistry(auto_codec=auto_codec)
138
+ self._merge_helper = BarecatMergeHelper(self)
139
+ self._fileobj_helper = BarecatFileObjectHelper(self)
140
+
141
+ if auto_codec:
142
+ warnings.warn(
143
+ 'auto_codec is deprecated and will be removed in version 1.0. '
144
+ "Use DecodedView instead: dec = DecodedView(bc); dec['file.json'] = data",
145
+ DeprecationWarning,
146
+ stacklevel=2,
147
+ )
148
+
149
+ ## Dict-like API: keys are filepaths, values are the file contents (bytes or decoded objects)
150
+ def __getitem__(self, path: str) -> Union[bytes, Any]:
151
+ """Get the contents of a file in the Barecat archive.
152
+
153
+ Args:
154
+ path: Path to the file within the archive.
155
+
156
+ Returns:
157
+ The contents of the file. Either raw bytes, or decoded based on the file extension, if
158
+ ``auto_codec`` was True in the constructor, or
159
+ if codecs have been registered for the file extension via ``register_codec``.
160
+
161
+ Raises:
162
+ KeyError: If a file with this path does not exist in the archive.
163
+
164
+ Examples:
165
+
166
+ >>> bc = Barecat('test.barecat', readonly=False)
167
+ >>> bc['file.txt'] = b'Hello, world!'
168
+ >>> bc['file.txt']
169
+ b'Hello, world!'
170
+ """
171
+
172
+ # Typically used in training loop
173
+ path = normalize_path(path)
174
+ row = self.index.fetch_one(
175
+ 'SELECT shard, offset, size, crc32c FROM files WHERE path=?', (path,)
176
+ )
177
+ if row is None:
178
+ raise KeyError(path)
179
+ raw_data = self.sharder.read_from_address(
180
+ row['shard'], row['offset'], row['size'], row['crc32c']
181
+ )
182
+ return self.codec_registry.decode(path, raw_data)
183
+
184
+ def get(self, path: str, default: Any = None) -> Union[bytes, Any]:
185
+ """Get the contents of a file in the Barecat archive, with a default value if the file does
186
+ not exist.
187
+
188
+ Args:
189
+ path: Path to the file within the archive.
190
+ default: Default value to return if the file does not exist.
191
+
192
+ Returns:
193
+ The contents of the file (possibly decoded), or the default value if the file does not
194
+ exist.
195
+ """
196
+ try:
197
+ return self[path]
198
+ except KeyError:
199
+ return default
200
+
201
+ def items(self) -> Iterator[tuple[str, Union[bytes, Any]]]:
202
+ """Iterate over all files in the archive, yielding (path, content) pairs.
203
+
204
+ Returns:
205
+ Iterator over (path, content) pairs.
206
+ """
207
+ for finfo in self.index.iter_all_fileinfos():
208
+ data = self.read(finfo)
209
+ yield finfo.path, self.codec_registry.decode(finfo.path, data)
210
+
211
+ def keys(self) -> Iterator[str]:
212
+ """Iterate over all file paths in the archive.
213
+
214
+ Returns:
215
+ Iterator over file paths.
216
+ """
217
+ for finfo in self.index.iter_all_fileinfos():
218
+ yield finfo.path
219
+
220
+ def values(self) -> Iterator[Union[bytes, Any]]:
221
+ """Iterate over all file contents in the archive.
222
+
223
+ Returns:
224
+ Iterator over file contents, possibly decoded based on the file extension.
225
+ """
226
+ for key, value in self.items():
227
+ yield value
228
+
229
+ def __contains__(self, path: str) -> bool:
230
+ """Check if a file with the given path exists in the archive.
231
+
232
+ Directories are ignored in this check.
233
+
234
+ Args:
235
+ path: Path to the file within the archive.
236
+
237
+ Returns:
238
+ True if the file exists, False otherwise.
239
+ """
240
+ return self.index.isfile(path)
241
+
242
+ def __len__(self) -> int:
243
+ """Get the number of files in the archive.
244
+
245
+ Returns:
246
+ Number of files in the archive.
247
+ """
248
+ return self.index.num_files
249
+
250
+ def __iter__(self) -> Iterator[str]:
251
+ """Iterate over all file paths in the archive.
252
+
253
+ Returns:
254
+ Iterator over file paths.
255
+ """
256
+ return self.files()
257
+
258
+ def __setitem__(self, path: str, content: Union[bytes, Any]):
259
+ """Add a file to the Barecat archive.
260
+
261
+ Args:
262
+ path: Path to the file within the archive.
263
+ content: Contents of the file. Either raw bytes, or an object to be encoded based on the
264
+ file extension, if ``auto_codec`` was True in the constructor, or if codecs have
265
+ been registered for the file extension via :meth:`register_codec`.
266
+
267
+ Raises:
268
+ ValueError: If the archive is read-only.
269
+ FileExistsBarecatError: If a file or directory with the given path already exists in the
270
+ archive.
271
+
272
+ Examples:
273
+
274
+ >>> bc = Barecat('test.barecat', readonly=False)
275
+ >>> bc['file.txt'] = b'Hello, world!'
276
+ >>> bc['file.txt']
277
+ b'Hello, world!'
278
+
279
+ """
280
+
281
+ self.add(path, data=self.codec_registry.encode(path, content))
282
+
283
+ def setdefault(self, key: str, default: Any = None, /):
284
+ try:
285
+ return self[key]
286
+ except KeyError:
287
+ self[key] = default
288
+ return default
289
+
290
+ def __delitem__(self, path: str):
291
+ """Remove a file from the Barecat archive.
292
+
293
+ Args:
294
+ path: Path to the file within the archive.
295
+
296
+ Raises:
297
+ KeyError: If a file with this path does not exist in the archive.
298
+
299
+ Examples:
300
+
301
+ >>> bc = Barecat('test.barecat', readonly=False)
302
+ >>> bc['file.txt'] = b'Hello, world!'
303
+ >>> bc['file.txt']
304
+ b'Hello, world!'
305
+ >>> del bc['file.txt']
306
+ >>> bc['file.txt']
307
+ Traceback (most recent call last):
308
+ ...
309
+ KeyError: 'file.txt'
310
+
311
+ """
312
+ try:
313
+ self.remove(path)
314
+ except (FileNotFoundBarecatError, IsADirectoryBarecatError):
315
+ raise KeyError(path)
316
+
317
+ # Filesystem-like API
318
+ def open(
319
+ self,
320
+ item: Union[BarecatFileInfo, str],
321
+ mode: str = 'r',
322
+ encoding: Optional[str] = None,
323
+ errors: Optional[str] = None,
324
+ newline: Optional[str] = None,
325
+ ) -> Union[BarecatFileObject, io.TextIOWrapper]:
326
+ """Open a file in the archive, as a file-like object.
327
+
328
+ Args:
329
+ item: Either a BarecatFileInfo object, or a path to a file within the archive.
330
+ mode: Mode to open the file in.
331
+ 'r'/'rt'/'rb' - read only, file must exist
332
+ 'r+'/'r+b' - read/write, file must exist
333
+ 'w'/'wt'/'wb' - write, truncate if exists, create if not
334
+ 'w+'/'w+b' - read/write, truncate if exists, create if not
335
+ 'x'/'xb' - exclusive create, fail if exists
336
+ 'x+'/'x+b' - exclusive create read/write, fail if exists
337
+ 'a'/'ab' - append, create if not exists
338
+ 'a+'/'a+b' - append read/write, create if not exists
339
+ encoding: Text encoding (only for text mode, default 'utf-8').
340
+ errors: Error handling for encoding (only for text mode).
341
+ newline: Newline handling (only for text mode).
342
+
343
+ Returns:
344
+ File-like object representing the file. Returns TextIOWrapper for text
345
+ mode, BarecatFileObject for binary mode.
346
+
347
+ Raises:
348
+ FileNotFoundBarecatError: If file doesn't exist and mode requires it.
349
+ FileExistsBarecatError: If file exists and mode is exclusive create.
350
+ """
351
+ return self._fileobj_helper.open(item, mode, encoding, errors, newline)
352
+
353
+ def exists(self, path: str) -> bool:
354
+ """Check if a file or directory exists in the archive.
355
+
356
+ Args:
357
+ path: Path to the file or directory within the archive.
358
+
359
+ Returns:
360
+ True if and only if a file or directory exists with the given path.
361
+ """
362
+ return self.index.exists(path)
363
+
364
+ def isfile(self, path):
365
+ """Check if a file exists in the archive.
366
+
367
+ Args:
368
+ path: Path to the file within the archive.
369
+
370
+ Returns:
371
+ True if and only if a file exists with the given path.
372
+ """
373
+ return self.index.isfile(path)
374
+
375
+ def isdir(self, path):
376
+ """Check if a directory exists in the archive.
377
+
378
+ Args:
379
+ path: Path to the directory within the archive.
380
+
381
+ Returns:
382
+ True if and only if a directory exists with the given path.
383
+ """
384
+ return self.index.isdir(path)
385
+
386
+ def listdir(self, path: str) -> list[str]:
387
+ """List all files and directories in a directory.
388
+
389
+ Args:
390
+ path: Path to the directory within the archive.
391
+
392
+ Returns:
393
+ List of all files and directories contained in the directory ``path``.
394
+ """
395
+ return self.index.listdir_names(path)
396
+
397
+ def walk(self, path: str) -> Iterator[tuple[str, list[str], list[str]]]:
398
+ """Recursively list all files and directories in the tree starting from a directory.
399
+
400
+ This is analogous to Python's :py:func:`os.walk`.
401
+
402
+ Args:
403
+ path: Path to the directory within the archive.
404
+
405
+ Returns:
406
+ Iterator over (dirpath, dirnames, filenames) tuples, where ``dirpath`` is the path to
407
+ the directory, ``dirnames`` is a list of all subdirectory names, and ``filenames`` is
408
+ a list of all filenames in the directory.
409
+
410
+ Examples:
411
+
412
+ >>> bc = Barecat('test.barecat', readonly=False)
413
+ >>> bc['dir/file.txt'] = b'Hello, world!'
414
+ >>> bc['dir/subdir/file2.txt'] = b'Hello, world2!'
415
+ >>> for dirpath, dirnames, filenames in bc.walk('dir'):
416
+ ... print(dirpath, dirnames, filenames)
417
+ dir ['subdir'] ['file.txt']
418
+ dir/subdir [] ['file2.txt']
419
+
420
+ """
421
+ return self.index.walk_names(path)
422
+
423
+ def scandir(self, path: str) -> Iterator[BarecatEntryInfo]:
424
+ """Iterate over all immediate files and subdirectories of the given directory, as :class:`barecat.BarecatEntryInfo` objects.
425
+
426
+ Args:
427
+ path: Path to the directory within the archive.
428
+
429
+ Returns:
430
+ An iterator over members of the directory, as :class:`barecat.BarecatEntryInfo` objects.
431
+ """
432
+ return self.index.iterdir_infos(path)
433
+
434
+ def glob(
435
+ self, pattern: str, recursive: bool = False, include_hidden: bool = False
436
+ ) -> list[str]:
437
+ """Find all files and directories matching a Unix-like glob pattern.
438
+
439
+ This function is equivalent to Python's :py:func:`glob.glob`.
440
+
441
+ Args:
442
+ pattern: Unix-like glob pattern to match.
443
+ recursive: If True, search recursively, with ``'/**/'`` matching any number of
444
+ directories.
445
+ include_hidden: If True, include hidden files and directories (starting with ``"."``).
446
+
447
+ Returns:
448
+ List of all file and directory paths matching the pattern.
449
+
450
+ Examples:
451
+ >>> bc = Barecat('test.barecat', readonly=False)
452
+ >>> bc['dir/file.txt'] = b'Hello, world!'
453
+ >>> bc['dir/subdir/file2.txt'] = b'Hello, world2!'
454
+ >>> bc.glob('dir/**/*.txt', recursive=True)
455
+ ['dir/file.txt', 'dir/subdir/file2.txt']
456
+ """
457
+ return self.index.glob_paths(pattern, recursive, include_hidden)
458
+
459
+ def globfiles(
460
+ self, pattern: str, recursive: bool = False, include_hidden: bool = False
461
+ ) -> list[str]:
462
+ """Find all files matching a Unix-like glob pattern.
463
+
464
+ Like ``glob``, but only returns files, not directories.
465
+
466
+ Args:
467
+ pattern: Unix-like glob pattern to match.
468
+ recursive: If True, search recursively, with ``'/**/'`` matching any number of
469
+ directories.
470
+ include_hidden: If True, include hidden files (starting with ``"."``).
471
+
472
+ Returns:
473
+ List of all file paths matching the pattern.
474
+ """
475
+ return self.index.glob_paths(pattern, recursive, include_hidden, only_files=True)
476
+
477
+ def iglob(
478
+ self, pattern: str, recursive: bool = False, include_hidden: bool = False
479
+ ) -> Iterator[str]:
480
+ """Iterate over all files and directories matching a Unix-like glob pattern.
481
+
482
+ This function is equivalent to Python's :py:func:`glob.iglob`.
483
+
484
+ Args:
485
+ pattern: Unix-like glob pattern to match.
486
+ recursive: If True, search recursively, with ``'/**/'`` matching any number of
487
+ directories.
488
+ include_hidden: If True, include hidden files and directories (starting with ``'.'``).
489
+
490
+ Returns:
491
+ Iterator over all file and directory paths matching the pattern.
492
+ """
493
+ return self.index.iterglob_paths(pattern, recursive, include_hidden)
494
+
495
+ def iglobfiles(
496
+ self, pattern: str, recursive: bool = False, include_hidden: bool = False
497
+ ) -> Iterator[str]:
498
+ """Iterate over all files matching a Unix-like glob pattern.
499
+
500
+ Like ``iglob``, but only returns files, not directories.
501
+
502
+ Args:
503
+ pattern: Unix-like glob pattern to match.
504
+ recursive: If True, search recursively, with ``'/**/'`` matching any number of
505
+ directories.
506
+ include_hidden: If True, include hidden files (starting with ``"."``).
507
+
508
+ Returns:
509
+ Iterator over all file paths matching the pattern.
510
+ """
511
+ return self.index.iterglob_paths(pattern, recursive, include_hidden, only_files=True)
512
+
513
+ def files(self) -> Iterator[str]:
514
+ """Iterate over all file paths in the archive.
515
+
516
+ Returns:
517
+ Iterator over file paths.
518
+ """
519
+ return self.index.iter_all_filepaths()
520
+
521
+ def dirs(self) -> Iterator[str]:
522
+ """Iterate over all directory paths in the archive.
523
+
524
+ Returns:
525
+ Iterator over directory paths.
526
+ """
527
+ return self.index.iter_all_dirpaths()
528
+
529
+ @property
530
+ def num_files(self) -> int:
531
+ """The number of files in the archive."""
532
+ return self.index.num_files
533
+
534
+ @property
535
+ def num_dirs(self) -> int:
536
+ """The number of directories in the archive."""
537
+ return self.index.num_dirs
538
+
539
+ @property
540
+ def total_size(self) -> int:
541
+ """The total size of all files in the archive, in bytes."""
542
+ return self.index.total_size
543
+
544
+ def readinto(self, item: Union[BarecatFileInfo, str], buffer, offset=0) -> int:
545
+ """Read a file into a buffer, starting from an offset within the file.
546
+
547
+ Read until either the buffer is full, or the end of the file is reached.
548
+
549
+ Args:
550
+ item: Either a BarecatFileInfo object, or a path to a file within the archive.
551
+ buffer: Destination buffer to read the file into.
552
+ offset: Offset within the file to start reading from.
553
+
554
+ Returns:
555
+ Number of bytes read.
556
+ """
557
+
558
+ # Used in fuse mount
559
+ if isinstance(item, BarecatFileInfo):
560
+ shard, offset_in_shard, size_in_shard, exp_crc32c = (
561
+ item.shard,
562
+ item.offset,
563
+ item.size,
564
+ item.crc32c,
565
+ )
566
+ else:
567
+ path = normalize_path(item)
568
+ row = self.index.fetch_one(
569
+ 'SELECT shard, offset, size, crc32c FROM files WHERE path=?', (path,)
570
+ )
571
+ if row is None:
572
+ raise FileNotFoundBarecatError(path)
573
+ shard, offset_in_shard, size_in_shard, exp_crc32c = row
574
+
575
+ offset = max(0, min(offset, size_in_shard))
576
+ size_to_read = min(len(buffer), size_in_shard - offset)
577
+
578
+ if size_to_read != size_in_shard:
579
+ exp_crc32c = None
580
+
581
+ return self.sharder.readinto_from_address(
582
+ shard, offset_in_shard + offset, memoryview(buffer)[:size_to_read], exp_crc32c
583
+ )
584
+
585
+ def read(self, item: Union[BarecatFileInfo, str], offset: int = 0, size: int = -1) -> bytes:
586
+ """Read a file from the archive, starting from an offset and reading a specific number of
587
+ bytes.
588
+
589
+ Args:
590
+ item: Either a BarecatFileInfo object, or a path to a file within the archive.
591
+ offset: Offset within the file to start reading from.
592
+ size: Number of bytes to read. If -1, read until the end of the file.
593
+
594
+ Returns:
595
+ The contents of the file, as bytes.
596
+
597
+ Raises:
598
+ ValueError: If the CRC32C checksum of the read data does not match the expected value.
599
+ FileNotFoundBarecatError: If a file with this path does not exist in the archive.
600
+ """
601
+ finfo = self.index._as_fileinfo(item)
602
+ with self.open(finfo, 'rb') as f:
603
+ f.seek(offset)
604
+ data = f.read(size)
605
+ if offset == 0 and (size == -1 or size == finfo.size) and finfo.crc32c is not None:
606
+ crc32c = crc32c_lib.crc32c(data)
607
+ if crc32c != finfo.crc32c:
608
+ raise ValueError(
609
+ f'CRC32C mismatch for {finfo.path}. Expected {finfo.crc32c}, got {crc32c}'
610
+ )
611
+ return data
612
+
613
+ # WRITING
614
+ @raise_if_readonly
615
+ def add_by_path(
616
+ self, filesys_path: str, store_path: Optional[str] = None, dir_exist_ok: bool = False
617
+ ):
618
+ """Add a file or directory from the filesystem to the archive.
619
+
620
+ Args:
621
+ filesys_path: Path to the file or directory on the filesystem.
622
+ store_path: Path to store the file or directory in the archive. If None, the same path
623
+ is used as ``filesys_path``.
624
+ dir_exist_ok: If True, do not raise an error when adding a directory and that
625
+ directory already exists in the archive (as a directory).
626
+
627
+ Raises:
628
+ ValueError: If the file is larger than the shard size limit.
629
+ FileExistsBarecatError: If a file or directory with the same path already exists in the
630
+ archive, unless ``dir_exist_ok`` is True and the item is a directory.
631
+
632
+ Examples:
633
+ >>> bc = Barecat('test.barecat', readonly=False)
634
+ >>> bc.add_by_path('file.txt')
635
+ >>> bc.add_by_path('dir', store_path='dir2')
636
+ """
637
+
638
+ if store_path is None:
639
+ store_path = filesys_path
640
+
641
+ statresult = os.stat(filesys_path)
642
+ if stat.S_ISDIR(statresult.st_mode):
643
+ finfo = BarecatDirInfo(path=store_path)
644
+ finfo.fill_from_statresult(statresult)
645
+ self.index.add_dir(finfo, exist_ok=dir_exist_ok)
646
+ return
647
+
648
+ finfo = BarecatFileInfo(path=store_path)
649
+ finfo.fill_from_statresult(statresult)
650
+ with open(filesys_path, 'rb') as in_file:
651
+ self.add(finfo, fileobj=in_file)
652
+
653
+ @raise_if_readonly
654
+ def add(
655
+ self,
656
+ item: Union[BarecatEntryInfo, str],
657
+ *,
658
+ data: Optional[bytes] = None,
659
+ fileobj=None,
660
+ bufsize: int = shutil.COPY_BUFSIZE,
661
+ dir_exist_ok: bool = False,
662
+ file_exist_ok: bool = False,
663
+ ):
664
+ """Add a file or directory to the archive.
665
+
666
+ Parent directories are automatically created if they don't exist (like ``mkdir -p``).
667
+
668
+ Args:
669
+ item: BarecatFileInfo or BarecatDirInfo object to add or a target path for a file.
670
+ data: File content. If None, the data is read from the file object.
671
+ fileobj: File-like object to read the data from.
672
+ bufsize: Buffer size to use when reading from the file object.
673
+ dir_exist_ok: If True, do not raise an error when adding a directory and that
674
+ directory already exists in the archive (as a directory).
675
+ file_exist_ok: If True, skip adding a file if a file with the same path already
676
+ exists in the archive. This is useful for merge operations.
677
+
678
+ Raises:
679
+ ValueError: If the file is larger than the shard size limit.
680
+ FileExistsBarecatError: If a file or directory with the same path already exists in the
681
+ archive, unless ``dir_exist_ok`` is True and the item is a directory, or
682
+ ``file_exist_ok`` is True and the item is a file.
683
+ NotADirectoryBarecatError: If a parent path exists as a file.
684
+
685
+ Examples:
686
+ >>> bc = Barecat('test.barecat', readonly=False)
687
+ >>> bc.add(BarecatFileInfo(path='file.txt', mode=0o666), data=b'Hello, world!')
688
+ >>> bc.add(BarecatDirInfo(path='dir', mode=0o777))
689
+ """
690
+ if isinstance(item, BarecatDirInfo):
691
+ self.index.add_dir(item, exist_ok=dir_exist_ok)
692
+ return
693
+
694
+ finfo = BarecatFileInfo(path=item) if isinstance(item, str) else item
695
+
696
+ # Check if file exists before writing data (to avoid wasted writes)
697
+ if file_exist_ok and finfo.path in self:
698
+ return
699
+
700
+ finfo.shard, finfo.offset, finfo.size, finfo.crc32c = self.sharder.add(
701
+ size=finfo.size, data=data, fileobj=fileobj, bufsize=bufsize
702
+ )
703
+
704
+ try:
705
+ self.index.add_file(finfo)
706
+ except FileExistsBarecatError:
707
+ # If the file already exists, we need to truncate the shard file back
708
+ shard_file = self.sharder.shard_files[finfo.shard]
709
+ with open(shard_file.name, 'r+b') as f:
710
+ f.truncate(finfo.offset)
711
+ raise
712
+
713
+ @raise_if_readonly_or_append_only
714
+ def update_file(
715
+ self,
716
+ old_item: Union[BarecatFileInfo, str],
717
+ new_item: Union[BarecatFileInfo, str, None] = None,
718
+ *,
719
+ data: Optional[bytes] = None,
720
+ fileobj=None,
721
+ size: Optional[int] = None,
722
+ bufsize: int = shutil.COPY_BUFSIZE,
723
+ ):
724
+ """Update an existing file's data and/or metadata, reusing space if new data fits.
725
+
726
+ If new data is smaller or equal to old data, writes in-place at the
727
+ existing location, avoiding fragmentation. If new data is larger,
728
+ writes at a new location (old location becomes a gap).
729
+
730
+ Args:
731
+ old_item: The existing file to update. Either a BarecatFileInfo (if user
732
+ already looked it up) or a path string.
733
+ new_item: Optional new metadata. If BarecatFileInfo, its metadata fields
734
+ (path, mtime_ns, mode, uid, gid) are the new values. If string,
735
+ it's the new path (rename). If None, only data is updated.
736
+ data: New file content. Either data or fileobj must be provided.
737
+ fileobj: File-like object to read new data from.
738
+ size: Size of new data. Required for fileobj if writing in-place.
739
+ bufsize: Buffer size for reading from fileobj.
740
+
741
+ Raises:
742
+ FileNotFoundBarecatError: If file does not exist.
743
+ ValueError: If neither data nor fileobj provided, or both provided.
744
+ """
745
+
746
+ old_info = self.index._as_fileinfo(old_item)
747
+
748
+ # Extract new metadata from new_item
749
+ if isinstance(new_item, BarecatFileInfo):
750
+ new_mtime_ns = new_item.mtime_ns
751
+ new_mode = new_item.mode
752
+ new_uid = new_item.uid
753
+ new_gid = new_item.gid
754
+ else:
755
+ new_mtime_ns = None
756
+ new_mode = None
757
+ new_uid = None
758
+ new_gid = None
759
+
760
+ if data is not None:
761
+ size = len(data)
762
+
763
+ if size is not None and size <= old_info.size:
764
+ # Fits in existing space - write in-place
765
+ shard, offset, size, crc32c = self.sharder.add(
766
+ shard=old_info.shard,
767
+ offset=old_info.offset,
768
+ size=size,
769
+ data=data,
770
+ fileobj=fileobj,
771
+ bufsize=bufsize,
772
+ raise_if_cannot_fit=True,
773
+ )
774
+ self.index.update_file(
775
+ old_info.path,
776
+ new_size=size,
777
+ new_crc32c=crc32c,
778
+ new_mtime_ns=new_mtime_ns,
779
+ new_mode=new_mode,
780
+ new_uid=new_uid,
781
+ new_gid=new_gid,
782
+ )
783
+ else:
784
+ # Doesn't fit or unknown size - allocate new space, then update index
785
+ shard, offset, size, crc32c = self.sharder.add(
786
+ data=data, fileobj=fileobj, size=size, bufsize=bufsize
787
+ )
788
+ self.index.update_file(
789
+ old_info.path,
790
+ new_shard=shard,
791
+ new_offset=offset,
792
+ new_size=size,
793
+ new_crc32c=crc32c,
794
+ new_mtime_ns=new_mtime_ns,
795
+ new_mode=new_mode,
796
+ new_uid=new_uid,
797
+ new_gid=new_gid,
798
+ )
799
+
800
+ # DIRECTORY CREATION
801
+ @raise_if_readonly
802
+ def mkdir(self, path: str, mode: int = 0o755, exist_ok: bool = False):
803
+ """Create a directory in the archive.
804
+
805
+ Parent directories are automatically created if they don't exist (like ``mkdir -p``).
806
+
807
+ Args:
808
+ path: Path of the directory to create.
809
+ mode: Permission mode for the directory (default: 0o755).
810
+ exist_ok: If True, do not raise an error if the directory already exists.
811
+
812
+ Raises:
813
+ FileExistsBarecatError: If a file or directory with this path already exists,
814
+ unless exist_ok is True and it's a directory.
815
+ NotADirectoryBarecatError: If the path or a parent exists as a file.
816
+ """
817
+ self.add(BarecatDirInfo(path=path, mode=mode), dir_exist_ok=exist_ok)
818
+
819
+ # DELETION
820
+ @raise_if_readonly_or_append_only
821
+ def remove(self, item: Union[BarecatFileInfo, str]):
822
+ """Remove (delete) a file from the archive.
823
+
824
+ Technically, the data is not erased from the shard file at this point, only the
825
+ corresponding row in the index database is removed.
826
+ An exception is when the file is the last file in the shard, in which case the shard file
827
+ is truncated to the end of the file.
828
+
829
+ Args:
830
+ item: Either a BarecatFileInfo object, or a path to a file within the archive.
831
+
832
+ Raises:
833
+ FileNotFoundBarecatError: If a file with this path does not exist in the archive.
834
+ IsADirectoryBarecatError: If the path refers to a directory, not a file.
835
+ """
836
+ try:
837
+ finfo = self.index._as_fileinfo(item)
838
+ except FileNotFoundBarecatError:
839
+ if self.isdir(item):
840
+ raise IsADirectoryBarecatError(item)
841
+ raise
842
+
843
+ # If this is the last file in the shard, we can just truncate the shard file
844
+ end = finfo.offset + finfo.size
845
+ if (
846
+ end >= self.sharder.shard_files[finfo.shard].tell()
847
+ and end >= osp.getsize(self.sharder.shard_files[finfo.shard].name)
848
+ and end == self.index.logical_shard_end(finfo.shard)
849
+ ):
850
+ with open(self.sharder.shard_files[finfo.shard].name, 'r+b') as f:
851
+ f.truncate(finfo.offset)
852
+ self.index.remove_file(finfo)
853
+
854
+ @raise_if_readonly_or_append_only
855
+ def rmdir(self, item: Union[BarecatDirInfo, str]):
856
+ """Remove (delete) an empty directory from the archive.
857
+
858
+ Args:
859
+ item: Either a BarecatDirInfo object, or a path to a directory within the archive.
860
+
861
+ Raises:
862
+ FileNotFoundBarecatError: If a directory with this path does not exist in the archive.
863
+ DirectoryNotEmptyBarecatError: If the directory is not empty.
864
+ """
865
+ self.index.remove_empty_dir(item)
866
+
867
+ @raise_if_readonly_or_append_only
868
+ def rmtree(self, item: Union[BarecatDirInfo, str]):
869
+ """Remove (delete) a directory and all its contents recursively from the archive.
870
+
871
+ Technically, file contents are not erased from the shard file at this point, only the
872
+ corresponding rows in the index database are removed.
873
+
874
+ Args:
875
+ item: Either a BarecatDirInfo object, or a path to a directory within the archive.
876
+
877
+ Raises:
878
+ FileNotFoundBarecatError: If a directory with this path does not exist in the archive.
879
+ """
880
+ self.index.remove_recursively(item)
881
+
882
+ # RENAMING
883
+ @raise_if_readonly_or_append_only
884
+ def rename(self, old_path: str, new_path: str):
885
+ """Rename a file or directory in the archive.
886
+
887
+ Args:
888
+ old_path: Path to the file or directory to rename.
889
+ new_path: New path for the file or directory.
890
+
891
+ Raises:
892
+ FileNotFoundBarecatError: If a file or directory with the old path does not exist.
893
+ FileExistsBarecatError: If a file or directory with the new path already exists.
894
+ """
895
+ self.index.rename(old_path, new_path)
896
+
897
+ @property
898
+ def total_physical_size_seek(self) -> int:
899
+ """Total size of all shard files, as determined by seeking to the end of the shard files.
900
+
901
+ This is more up-to-date than :meth:`total_physical_size_stat`, but may be slower.
902
+
903
+ Returns:
904
+ Total size of all shard files, in bytes.
905
+ """
906
+ return self.sharder.total_physical_size_seek
907
+
908
+ @property
909
+ def total_physical_size_stat(self) -> int:
910
+ """Total size of all shard files, as determined by the file system's `stat` response.
911
+
912
+ This is faster than :meth:`total_physical_size_seek`, but may be less up-to-date.
913
+
914
+ Returns:
915
+ Total size of all shard files, in bytes.
916
+ """
917
+ return self.sharder.total_physical_size_stat
918
+
919
+ @property
920
+ def total_logical_size(self) -> int:
921
+ """Total size of all files in the archive, as determined by the index database.
922
+
923
+ Returns:
924
+ Total size of all files in the archive, in bytes.
925
+ """
926
+ return self.index.total_size
927
+
928
+ # MERGING
929
+ @raise_if_readonly
930
+ def merge_from_other_barecat(
931
+ self,
932
+ source_path: str,
933
+ ignore_duplicates: bool = False,
934
+ prefix: str = '',
935
+ pattern: str = None,
936
+ filter_rules: list = None,
937
+ ):
938
+ """Merge the contents of another Barecat archive into this one.
939
+
940
+ Args:
941
+ source_path: Path to the other Barecat archive.
942
+ ignore_duplicates: If True, do not raise an error when a file with the same path already
943
+ exists in the archive.
944
+ prefix: Path prefix to prepend to all paths (default: '', no prefix).
945
+ pattern: Glob pattern to filter files (uses optimized iterglob_infos).
946
+ filter_rules: Rsync-style include/exclude rules as list of ('+'/'-', pattern) tuples.
947
+
948
+ Raises:
949
+ ValueError: If the shard size limit is set and a file in the source archive is larger
950
+ than the shard size limit.
951
+ """
952
+ self._merge_helper.merge_from_other_barecat(
953
+ source_path, ignore_duplicates, prefix, pattern, filter_rules
954
+ )
955
+
956
+ @property
957
+ def shard_size_limit(self) -> int:
958
+ """Maximum size of each shard file."""
959
+ return self.index.shard_size_limit
960
+
961
+ @shard_size_limit.setter
962
+ def shard_size_limit(self, value: Union[int, str]):
963
+ """Set the maximum size of each shard file.
964
+
965
+ Args:
966
+ value: Size in bytes (int) or as a string like '1G', '500M', '100K'.
967
+ """
968
+ self.index.shard_size_limit = value
969
+
970
+ def logical_shard_end(self, shard_number: int) -> int:
971
+ """Logical end of a shard, in bytes, that is the position after the last byte of the last
972
+ file contained in the shard.
973
+
974
+ Args:
975
+ shard_number: Shard number, index starting from 0.
976
+
977
+ Returns:
978
+ Logical end of the shard, in bytes.
979
+ """
980
+ return self.index.logical_shard_end(shard_number)
981
+
982
+ def physical_shard_end(self, shard_number):
983
+ """Physical end of a shard, in bytes, that is the end seek position of the shard file.
984
+
985
+ Args:
986
+ shard_number: Shard number, index starting from 0.
987
+
988
+ Returns:
989
+ Physical end of the shard, in bytes.
990
+ """
991
+
992
+ return self.sharder.physical_shard_end(shard_number)
993
+
994
+ def raise_if_readonly(self, message):
995
+ if self.readonly:
996
+ raise ValueError(message)
997
+
998
+ def raise_if_append_only(self, message):
999
+ if self.append_only:
1000
+ raise ValueError(message)
1001
+
1002
+ # THREADSAFE
1003
+ def _make_index(self):
1004
+ return Index(
1005
+ resolve_index_path(self.path),
1006
+ readonly=self.readonly,
1007
+ wal=self.wal,
1008
+ readonly_is_immutable=self.readonly_is_immutable,
1009
+ )
1010
+
1011
+ @property
1012
+ def index(self) -> Index:
1013
+ """Index object to manipulate the metadata database of the Barecat archive."""
1014
+ return self._index_storage.get(self._make_index)
1015
+
1016
+ # CONSISTENCY CHECKS
1017
+ def check_crc32c(self, item: Union[BarecatFileInfo, str]):
1018
+ """Check the CRC32C checksum of a file in the archive.
1019
+
1020
+ Args:
1021
+ item: Either a BarecatFileInfo object, or a path to a file within the archive.
1022
+
1023
+ Returns:
1024
+ True if the CRC32C checksum of the file matches the expected value or no checksum is
1025
+ stored in the database.
1026
+
1027
+ Raises:
1028
+ LookupError: If a file with this path does not exist in the archive.
1029
+ """
1030
+
1031
+ finfo = self.index._as_fileinfo(item)
1032
+ with self.open(finfo, 'rb') as f:
1033
+ crc32c = accumulate_crc32c(f)
1034
+ if finfo.crc32c is not None and crc32c != finfo.crc32c:
1035
+ logger.warning('CRC32C mismatch for %s. Expected %s, got %s', finfo.path, finfo.crc32c, crc32c)
1036
+ return False
1037
+ return True
1038
+
1039
+ def verify_integrity(self, quick=False):
1040
+ """Verify the integrity of the Barecat archive.
1041
+
1042
+ This includes checking the CRC32C checksums of all files, and checking the integrity of the
1043
+ index database.
1044
+
1045
+ Args:
1046
+ quick: If True, only check the CRC32C checksums of the last file of the archive.
1047
+
1048
+ Returns:
1049
+ True if no problems were found, False otherwise.
1050
+ """
1051
+
1052
+ is_good = True
1053
+ if quick:
1054
+ try:
1055
+ if not self.check_crc32c(self.index.get_last_file()):
1056
+ is_good = False
1057
+ except LookupError:
1058
+ pass # no files
1059
+ else:
1060
+ n_printed = 0
1061
+ for fi in barecat_progbar.progressbar(
1062
+ self.index.iter_all_fileinfos(), total=self.num_files
1063
+ ):
1064
+ if not self.check_crc32c(fi):
1065
+ is_good = False
1066
+ if n_printed >= 10:
1067
+ logger.warning('... (further mismatches suppressed)')
1068
+ break
1069
+ n_printed += 1
1070
+
1071
+ if not self.index.verify_integrity():
1072
+ is_good = False
1073
+ return is_good
1074
+
1075
+ # CODECS
1076
+ def register_codec(
1077
+ self,
1078
+ exts: list[str],
1079
+ encoder: Callable[[Any], bytes],
1080
+ decoder: Callable[[bytes], Any],
1081
+ nonfinal: bool = False,
1082
+ ):
1083
+ """Register an encoder and decoder for one or more file extensions.
1084
+
1085
+ This allows automatic encoding and decoding (serialization/deserialization) of files based
1086
+ on their extension, used in the dictionary interface, e.g., :meth:`__getitem__`,
1087
+ :meth:`__setitem__` and :meth:`items` methods.
1088
+
1089
+ If ``auto_codec`` was True in the constructor, then the codecs are already
1090
+ registered by default for the following extensions:
1091
+
1092
+ **Data formats:**
1093
+ - ``.json`` — dict/list (stdlib json)
1094
+ - ``.pkl``, ``.pickle`` — any object (pickle)
1095
+ - ``.npy`` — numpy array
1096
+ - ``.npz`` — dict of numpy arrays
1097
+ - ``.msgpack`` — any object (requires msgpack-numpy)
1098
+
1099
+ **Image formats** (uses cv2 > PIL > imageio, whichever is available):
1100
+ - ``.jpg``, ``.jpeg``, ``.png``, ``.bmp``, ``.gif``
1101
+ - ``.tiff``, ``.tif``, ``.webp``, ``.exr``
1102
+
1103
+ **Compression** (stackable with other codecs):
1104
+ - ``.gz``, ``.gzip`` — gzip
1105
+ - ``.xz``, ``.lzma`` — lzma
1106
+ - ``.bz2`` — bzip2
1107
+
1108
+
1109
+ Args:
1110
+ exts: List of file extensions to register the codec for.
1111
+ encoder: Function to encode data into bytes.
1112
+ decoder: Function to decode bytes into data.
1113
+ nonfinal: If True, other codecs are allowed to be applied afterwards in a nested
1114
+ manner. This is useful for, e.g., compression codecs.
1115
+
1116
+ Examples:
1117
+ Simple text encoding:
1118
+
1119
+ >>> bc = Barecat('test.barecat', readonly=False)
1120
+ >>> def encode(data):
1121
+ ... return data.encode('utf-8')
1122
+ >>> def decode(data):
1123
+ ... return data.decode('utf-8')
1124
+ >>> bc.register_codec(['.txt'], encode, decode)
1125
+
1126
+ Or using a codec from a library:
1127
+
1128
+ >>> import cv2
1129
+ >>> bc = Barecat('test.barecat', readonly=False)
1130
+ >>> def encode_png(data):
1131
+ ... return cv2.imencode('.png', data)[1].tobytes()
1132
+ >>> def decode_png(data):
1133
+ ... return cv2.imdecode(np.frombuffer(data, np.uint8), cv2.IMREAD_UNCHANGED)
1134
+ >>> bc.register_codec(['.png'], encode_png, decode_png)
1135
+
1136
+ Or using a compression library:
1137
+
1138
+ >>> import zlib
1139
+ >>> bc = Barecat('test.barecat', readonly=False)
1140
+ >>> def encode_zlib(data):
1141
+ ... return zlib.compress(data)
1142
+ >>> def decode_zlib(data):
1143
+ ... return zlib.decompress(data)
1144
+ >>> bc.register_codec(['.gz'], encode_zlib, decode_zlib, nonfinal=True)
1145
+
1146
+ Or pickling:
1147
+
1148
+ >>> import pickle
1149
+ >>> bc = Barecat('test.barecat', readonly=False)
1150
+ >>> bc.register_codec(['.pkl'], pickle.dumps, pickle.loads)
1151
+ """
1152
+ warnings.warn(
1153
+ 'register_codec is deprecated and will be removed in version 1.0. '
1154
+ 'Use DecodedView instead: dec = DecodedView(bc); dec.register_codec(...)',
1155
+ DeprecationWarning,
1156
+ stacklevel=2,
1157
+ )
1158
+ self.codec_registry.register_codec(exts, encoder, decoder, nonfinal)
1159
+
1160
+ # PICKLING
1161
+ def __reduce__(self):
1162
+ if not self.readonly:
1163
+ raise ValueError('Cannot pickle a non-readonly Barecat')
1164
+ return self.__class__, (
1165
+ self.path,
1166
+ None,
1167
+ True,
1168
+ False,
1169
+ self.auto_codec,
1170
+ True,
1171
+ False,
1172
+ self.threadsafe,
1173
+ )
1174
+
1175
+ def truncate_all_to_logical_size(self):
1176
+ logical_shard_ends = [
1177
+ self.index.logical_shard_end(i) for i in range(self.sharder.num_shards)
1178
+ ]
1179
+ self.sharder.truncate_all_to_logical_size(logical_shard_ends)
1180
+
1181
+ # DEFRAG
1182
+ def defrag(self, quick=False):
1183
+ """Defragment the Barecat archive.
1184
+
1185
+ Args:
1186
+ quick: Perform a faster, but less thorough defragmentation.
1187
+ """
1188
+ defragger = BarecatDefragger(self)
1189
+ if quick:
1190
+ return defragger.defrag_quick()
1191
+ else:
1192
+ return defragger.defrag()
1193
+
1194
+ def close(self):
1195
+ """Close the Barecat archive."""
1196
+ self._index_storage.close()
1197
+ self.sharder.close()
1198
+
1199
+ def __repr__(self):
1200
+ mode = 'readonly' if self.readonly else 'read-write'
1201
+ try:
1202
+ n = self.num_files
1203
+ return f"Barecat('{self.path}', {mode}, {n} files)"
1204
+ except Exception:
1205
+ return f"Barecat('{self.path}', closed)"
1206
+
1207
+ def __exit__(self, exc_type, exc_val, exc_tb):
1208
+ """Exit a context manager."""
1209
+ self.close()