barecat 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- barecat/__init__.py +108 -0
- barecat/__main__.py +5 -0
- barecat/_api.py +143 -0
- barecat/_version.py +24 -0
- barecat/cli/__init__.py +0 -0
- barecat/cli/commands.py +540 -0
- barecat/cli/completions.py +27 -0
- barecat/cli/impl.py +609 -0
- barecat/cli/legacy.py +361 -0
- barecat/cli/main.py +1355 -0
- barecat/cli/shell.py +827 -0
- barecat/completions/__init__.py +1 -0
- barecat/completions/barecat.bash +246 -0
- barecat/completions/barecat.zsh +268 -0
- barecat/core/__init__.py +0 -0
- barecat/core/barecat.py +1209 -0
- barecat/core/index.py +1983 -0
- barecat/core/paths.py +61 -0
- barecat/core/sharder.py +328 -0
- barecat/core/types.py +340 -0
- barecat/exceptions.py +92 -0
- barecat/formats/__init__.py +0 -0
- barecat/formats/archive_formats.py +234 -0
- barecat/formats/from_tar_stream.py +53 -0
- barecat/formats/to_tar_stream.py +54 -0
- barecat/io/__init__.py +5 -0
- barecat/io/codecs.py +622 -0
- barecat/io/copyfile.py +550 -0
- barecat/io/fileobj.py +556 -0
- barecat/io/open.py +41 -0
- barecat/maintenance/__init__.py +0 -0
- barecat/maintenance/complete_database.py +79 -0
- barecat/maintenance/defrag.py +541 -0
- barecat/maintenance/merge.py +613 -0
- barecat/maintenance/reshard.py +153 -0
- barecat/maintenance/rsync.py +1282 -0
- barecat/maintenance/upgrade_database.py +200 -0
- barecat/maintenance/upgrade_database2.py +74 -0
- barecat/man/barecat.1 +284 -0
- barecat/sql/indexes.sql +6 -0
- barecat/sql/schema.sql +55 -0
- barecat/sql/triggers.sql +172 -0
- barecat/tui/__init__.py +0 -0
- barecat/tui/browse.py +448 -0
- barecat/tui/ncdu.py +318 -0
- barecat/util/__init__.py +0 -0
- barecat/util/consumed_threadpool.py +119 -0
- barecat/util/glob_helper.py +584 -0
- barecat/util/glob_to_regex.py +353 -0
- barecat/util/list_files.py +12 -0
- barecat/util/misc.py +174 -0
- barecat/util/physical_order.py +77 -0
- barecat/util/progbar.py +40 -0
- barecat/util/threading.py +59 -0
- barecat-0.3.2.dist-info/METADATA +276 -0
- barecat-0.3.2.dist-info/RECORD +60 -0
- barecat-0.3.2.dist-info/WHEEL +5 -0
- barecat-0.3.2.dist-info/entry_points.txt +15 -0
- barecat-0.3.2.dist-info/licenses/LICENSE +21 -0
- barecat-0.3.2.dist-info/top_level.txt +1 -0
barecat/__init__.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Barecat is a fast random-access, mountable archive format for storing and accessing many small
|
|
2
|
+
files."""
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
from ._version import version as __version__
|
|
6
|
+
except ImportError:
|
|
7
|
+
__version__ = '0.0.0.dev0'
|
|
8
|
+
|
|
9
|
+
# Core classes
|
|
10
|
+
from .core.barecat import Barecat
|
|
11
|
+
from .core.index import Index
|
|
12
|
+
from .core.sharder import Sharder
|
|
13
|
+
|
|
14
|
+
# Data types
|
|
15
|
+
from .core.types import (
|
|
16
|
+
BarecatDirInfo,
|
|
17
|
+
BarecatEntryInfo,
|
|
18
|
+
BarecatFileInfo,
|
|
19
|
+
Order,
|
|
20
|
+
SHARD_SIZE_UNLIMITED,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# File objects
|
|
24
|
+
from .io.fileobj import (
|
|
25
|
+
BarecatFileObject,
|
|
26
|
+
BarecatReadOnlyFileObject,
|
|
27
|
+
BarecatReadWriteFileObject,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Codec view
|
|
31
|
+
from .io.codecs import DecodedView
|
|
32
|
+
|
|
33
|
+
# Exceptions
|
|
34
|
+
from .exceptions import (
|
|
35
|
+
BarecatError,
|
|
36
|
+
BarecatIntegrityError,
|
|
37
|
+
DirectoryNotEmptyBarecatError,
|
|
38
|
+
FileExistsBarecatError,
|
|
39
|
+
FileNotFoundBarecatError,
|
|
40
|
+
FileTooLargeBarecatError,
|
|
41
|
+
IsADirectoryBarecatError,
|
|
42
|
+
NotADirectoryBarecatError,
|
|
43
|
+
NotEnoughSpaceBarecatError,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# CLI utilities (for programmatic use)
|
|
47
|
+
from .cli.impl import (
|
|
48
|
+
archive2barecat,
|
|
49
|
+
barecat2archive,
|
|
50
|
+
extract,
|
|
51
|
+
merge,
|
|
52
|
+
merge_symlink,
|
|
53
|
+
read_index,
|
|
54
|
+
write_index,
|
|
55
|
+
)
|
|
56
|
+
from .cli.completions import get_completion_script
|
|
57
|
+
|
|
58
|
+
# Archive utilities
|
|
59
|
+
from .util.misc import exists, remove
|
|
60
|
+
|
|
61
|
+
# Convenience API
|
|
62
|
+
from ._api import get_cached_reader, open
|
|
63
|
+
|
|
64
|
+
__all__ = [
|
|
65
|
+
# Version
|
|
66
|
+
'__version__',
|
|
67
|
+
# Core classes
|
|
68
|
+
'Barecat',
|
|
69
|
+
'Index',
|
|
70
|
+
'Sharder',
|
|
71
|
+
# Data types
|
|
72
|
+
'BarecatDirInfo',
|
|
73
|
+
'BarecatEntryInfo',
|
|
74
|
+
'BarecatFileInfo',
|
|
75
|
+
'Order',
|
|
76
|
+
'SHARD_SIZE_UNLIMITED',
|
|
77
|
+
# File objects
|
|
78
|
+
'BarecatFileObject',
|
|
79
|
+
'BarecatReadOnlyFileObject',
|
|
80
|
+
'BarecatReadWriteFileObject',
|
|
81
|
+
# Codec view
|
|
82
|
+
'DecodedView',
|
|
83
|
+
# Exceptions
|
|
84
|
+
'BarecatError',
|
|
85
|
+
'BarecatIntegrityError',
|
|
86
|
+
'DirectoryNotEmptyBarecatError',
|
|
87
|
+
'FileExistsBarecatError',
|
|
88
|
+
'FileNotFoundBarecatError',
|
|
89
|
+
'FileTooLargeBarecatError',
|
|
90
|
+
'IsADirectoryBarecatError',
|
|
91
|
+
'NotADirectoryBarecatError',
|
|
92
|
+
'NotEnoughSpaceBarecatError',
|
|
93
|
+
# CLI utilities
|
|
94
|
+
'archive2barecat',
|
|
95
|
+
'barecat2archive',
|
|
96
|
+
'extract',
|
|
97
|
+
'merge',
|
|
98
|
+
'merge_symlink',
|
|
99
|
+
'read_index',
|
|
100
|
+
'write_index',
|
|
101
|
+
'get_completion_script',
|
|
102
|
+
# Archive utilities
|
|
103
|
+
'exists',
|
|
104
|
+
'remove',
|
|
105
|
+
# Convenience API
|
|
106
|
+
'get_cached_reader',
|
|
107
|
+
'open',
|
|
108
|
+
]
|
barecat/__main__.py
ADDED
barecat/_api.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""High-level convenience functions for opening Barecat archives."""
|
|
2
|
+
|
|
3
|
+
import functools
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
from .core.barecat import Barecat
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def open(path, mode='r', auto_codec=False, threadsafe_reader=True):
|
|
10
|
+
"""Open a Barecat archive.
|
|
11
|
+
|
|
12
|
+
.. deprecated::
|
|
13
|
+
Use ``Barecat(path, readonly=True)`` or ``Barecat(path, readonly=False)`` directly.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
path: Path to the archive (without suffix).
|
|
17
|
+
mode: 'r' (read), 'r+' (read-write), 'w+' (overwrite), 'a+' (append), 'x+' (exclusive create).
|
|
18
|
+
auto_codec: **Deprecated.** Use :class:`DecodedView` instead. Will be removed in 1.0.
|
|
19
|
+
threadsafe_reader: Use thread-local storage for read mode. Default: True.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
Barecat: The opened archive.
|
|
23
|
+
"""
|
|
24
|
+
warnings.warn(
|
|
25
|
+
'barecat.open() is deprecated and will be removed in version 1.0. '
|
|
26
|
+
'Use Barecat(path, readonly=True) or Barecat(path, readonly=False) directly.',
|
|
27
|
+
DeprecationWarning,
|
|
28
|
+
stacklevel=2,
|
|
29
|
+
)
|
|
30
|
+
if auto_codec:
|
|
31
|
+
warnings.warn(
|
|
32
|
+
'auto_codec is deprecated and will be removed in version 1.0. '
|
|
33
|
+
"Use DecodedView instead: dec = DecodedView(bc); dec['file.json'] = data",
|
|
34
|
+
DeprecationWarning,
|
|
35
|
+
stacklevel=2,
|
|
36
|
+
)
|
|
37
|
+
if mode == 'r':
|
|
38
|
+
return Barecat(path, readonly=True, threadsafe=threadsafe_reader, auto_codec=auto_codec)
|
|
39
|
+
elif mode == 'w+':
|
|
40
|
+
return Barecat(
|
|
41
|
+
path,
|
|
42
|
+
readonly=False,
|
|
43
|
+
overwrite=True,
|
|
44
|
+
exist_ok=True,
|
|
45
|
+
append_only=False,
|
|
46
|
+
auto_codec=auto_codec,
|
|
47
|
+
)
|
|
48
|
+
elif mode == 'r+':
|
|
49
|
+
return Barecat(
|
|
50
|
+
path,
|
|
51
|
+
readonly=False,
|
|
52
|
+
overwrite=False,
|
|
53
|
+
exist_ok=True,
|
|
54
|
+
append_only=False,
|
|
55
|
+
auto_codec=auto_codec,
|
|
56
|
+
)
|
|
57
|
+
elif mode == 'a+':
|
|
58
|
+
return Barecat(
|
|
59
|
+
path,
|
|
60
|
+
readonly=False,
|
|
61
|
+
overwrite=False,
|
|
62
|
+
exist_ok=True,
|
|
63
|
+
append_only=True,
|
|
64
|
+
auto_codec=auto_codec,
|
|
65
|
+
)
|
|
66
|
+
elif mode == 'ax+':
|
|
67
|
+
return Barecat(
|
|
68
|
+
path,
|
|
69
|
+
readonly=False,
|
|
70
|
+
overwrite=False,
|
|
71
|
+
exist_ok=False,
|
|
72
|
+
append_only=True,
|
|
73
|
+
auto_codec=auto_codec,
|
|
74
|
+
)
|
|
75
|
+
elif mode == 'x+':
|
|
76
|
+
return Barecat(
|
|
77
|
+
path,
|
|
78
|
+
readonly=False,
|
|
79
|
+
overwrite=False,
|
|
80
|
+
exist_ok=False,
|
|
81
|
+
append_only=False,
|
|
82
|
+
auto_codec=auto_codec,
|
|
83
|
+
)
|
|
84
|
+
else:
|
|
85
|
+
raise ValueError(f'Invalid mode: {mode}')
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def get_cached_reader(path, auto_codec=None):
|
|
89
|
+
"""Get a thread-locally cached read-only Barecat reader.
|
|
90
|
+
|
|
91
|
+
Each thread/process gets its own cached instance. Useful for multi-threaded
|
|
92
|
+
data loading where each worker needs its own Barecat handle.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
path: Path to the archive.
|
|
96
|
+
auto_codec: Whether to automatically decode files based on extension.
|
|
97
|
+
Currently defaults to True for backwards compatibility, but the default
|
|
98
|
+
will change to False in version 1.0. Pass True or False explicitly to
|
|
99
|
+
silence the deprecation warning. Use DecodedView for new code.
|
|
100
|
+
"""
|
|
101
|
+
import multiprocessing_utils
|
|
102
|
+
|
|
103
|
+
if auto_codec is None:
|
|
104
|
+
warnings.warn(
|
|
105
|
+
'get_cached_reader() currently defaults to auto_codec=True, but this '
|
|
106
|
+
'default will change to False in version 1.0. Pass auto_codec=True '
|
|
107
|
+
'explicitly to keep the current behavior, or auto_codec=False '
|
|
108
|
+
'(and use DecodedView) to adopt the new default.',
|
|
109
|
+
FutureWarning,
|
|
110
|
+
stacklevel=2,
|
|
111
|
+
)
|
|
112
|
+
auto_codec = True
|
|
113
|
+
elif auto_codec:
|
|
114
|
+
warnings.warn(
|
|
115
|
+
'auto_codec=True in get_cached_reader() is deprecated and will be removed '
|
|
116
|
+
'in version 1.0. Use DecodedView instead.',
|
|
117
|
+
DeprecationWarning,
|
|
118
|
+
stacklevel=2,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
return _cached_reader_open(path, auto_codec)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _threadlocal_decorate(decorator):
|
|
125
|
+
import multiprocessing_utils
|
|
126
|
+
|
|
127
|
+
def my_decorator(fun):
|
|
128
|
+
local = multiprocessing_utils.local()
|
|
129
|
+
|
|
130
|
+
@functools.wraps(fun)
|
|
131
|
+
def wrapper(*args, **kwargs):
|
|
132
|
+
if not hasattr(local, 'fn'):
|
|
133
|
+
local.fn = decorator(fun)
|
|
134
|
+
return local.fn(*args, **kwargs)
|
|
135
|
+
|
|
136
|
+
return wrapper
|
|
137
|
+
|
|
138
|
+
return my_decorator
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@_threadlocal_decorate(functools.lru_cache())
|
|
142
|
+
def _cached_reader_open(path, auto_codec=True):
|
|
143
|
+
return Barecat(path, readonly=True, auto_codec=auto_codec)
|
barecat/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.3.2'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 2)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
barecat/cli/__init__.py
ADDED
|
File without changes
|