barecat 0.2.7__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- barecat-0.3.0/.github/workflows/ci.yml +38 -0
- {barecat-0.2.7 → barecat-0.3.0}/.gitignore +4 -1
- barecat-0.3.0/.pre-commit-config.yaml +17 -0
- barecat-0.3.0/MANIFEST.in +4 -0
- barecat-0.3.0/Makefile +22 -0
- {barecat-0.2.7 → barecat-0.3.0}/PKG-INFO +136 -43
- {barecat-0.2.7 → barecat-0.3.0}/README.md +104 -41
- barecat-0.3.0/docs/explanation/architecture.rst +430 -0
- barecat-0.3.0/docs/explanation/comparison.rst +271 -0
- barecat-0.3.0/docs/explanation/integrity.rst +140 -0
- barecat-0.3.0/docs/explanation/performance.rst +261 -0
- barecat-0.3.0/docs/howto/automatic-encoding.rst +197 -0
- barecat-0.3.0/docs/howto/convert-archives.rst +199 -0
- barecat-0.3.0/docs/howto/merge-archives.rst +197 -0
- barecat-0.3.0/docs/howto/mount-fuse.rst +224 -0
- barecat-0.3.0/docs/howto/pytorch-dataloader.rst +319 -0
- barecat-0.3.0/docs/howto/shell-completions.rst +126 -0
- barecat-0.3.0/docs/howto/verify-repair.rst +197 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/index.rst +53 -25
- barecat-0.3.0/docs/reference/cli.rst +782 -0
- barecat-0.3.0/docs/reference/file-format.rst +291 -0
- barecat-0.3.0/docs/reference/python-api.rst +509 -0
- barecat-0.3.0/docs/tutorials/getting-started.rst +232 -0
- barecat-0.3.0/docs/tutorials/image-datasets.rst +262 -0
- barecat-0.3.0/pyproject.toml +145 -0
- barecat-0.3.0/src/barecat/__init__.py +108 -0
- barecat-0.3.0/src/barecat/__main__.py +5 -0
- barecat-0.3.0/src/barecat/_api.py +131 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat/_version.py +3 -3
- barecat-0.3.0/src/barecat/cli/commands.py +540 -0
- barecat-0.3.0/src/barecat/cli/completions.py +27 -0
- barecat-0.3.0/src/barecat/cli/impl.py +609 -0
- barecat-0.2.7/src/barecat/cli.py → barecat-0.3.0/src/barecat/cli/legacy.py +112 -11
- barecat-0.3.0/src/barecat/cli/main.py +1355 -0
- barecat-0.3.0/src/barecat/cli/shell.py +827 -0
- barecat-0.3.0/src/barecat/completions/__init__.py +1 -0
- barecat-0.3.0/src/barecat/completions/barecat.bash +246 -0
- barecat-0.3.0/src/barecat/completions/barecat.zsh +268 -0
- barecat-0.3.0/src/barecat/core/__init__.py +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/barecat.py +281 -256
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/index.py +603 -416
- barecat-0.3.0/src/barecat/core/paths.py +61 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/sharder.py +76 -47
- barecat-0.2.7/src/barecat/common.py → barecat-0.3.0/src/barecat/core/types.py +46 -146
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat/exceptions.py +13 -0
- barecat-0.3.0/src/barecat/formats/__init__.py +0 -0
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/archive_formats.py +102 -64
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/from_tar_stream.py +2 -2
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/to_tar_stream.py +10 -10
- barecat-0.3.0/src/barecat/io/__init__.py +5 -0
- barecat-0.3.0/src/barecat/io/codecs.py +622 -0
- barecat-0.3.0/src/barecat/io/copyfile.py +550 -0
- barecat-0.3.0/src/barecat/io/fileobj.py +556 -0
- barecat-0.3.0/src/barecat/io/open.py +41 -0
- barecat-0.3.0/src/barecat/maintenance/__init__.py +0 -0
- barecat-0.3.0/src/barecat/maintenance/complete_database.py +79 -0
- barecat-0.3.0/src/barecat/maintenance/defrag.py +541 -0
- barecat-0.3.0/src/barecat/maintenance/merge.py +613 -0
- barecat-0.3.0/src/barecat/maintenance/reshard.py +153 -0
- barecat-0.3.0/src/barecat/maintenance/rsync.py +1282 -0
- barecat-0.3.0/src/barecat/maintenance/upgrade_database.py +200 -0
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/maintenance}/upgrade_database2.py +15 -14
- barecat-0.3.0/src/barecat/man/barecat.1 +284 -0
- barecat-0.3.0/src/barecat/sql/indexes.sql +6 -0
- barecat-0.3.0/src/barecat/sql/schema.sql +55 -0
- barecat-0.2.7/src/barecat/sql/schema.sql → barecat-0.3.0/src/barecat/sql/triggers.sql +33 -78
- barecat-0.3.0/src/barecat/tui/__init__.py +0 -0
- barecat-0.3.0/src/barecat/tui/browse.py +448 -0
- barecat-0.3.0/src/barecat/tui/ncdu.py +318 -0
- barecat-0.3.0/src/barecat/util/__init__.py +0 -0
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/util}/consumed_threadpool.py +2 -0
- barecat-0.3.0/src/barecat/util/glob_helper.py +584 -0
- barecat-0.3.0/src/barecat/util/glob_to_regex.py +353 -0
- barecat-0.3.0/src/barecat/util/list_files.py +12 -0
- barecat-0.3.0/src/barecat/util/misc.py +174 -0
- barecat-0.3.0/src/barecat/util/physical_order.py +77 -0
- {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/util}/progbar.py +3 -0
- barecat-0.3.0/src/barecat/util/threading.py +59 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/PKG-INFO +136 -43
- barecat-0.3.0/src/barecat.egg-info/SOURCES.txt +121 -0
- barecat-0.3.0/src/barecat.egg-info/entry_points.txt +15 -0
- barecat-0.3.0/src/barecat.egg-info/requires.txt +34 -0
- {barecat-0.2.7 → barecat-0.3.0}/tests/test_barecat.py +2 -3
- barecat-0.3.0/tests/test_cli.py +941 -0
- barecat-0.3.0/tests/test_codecs.py +402 -0
- barecat-0.3.0/tests/test_copyfile.py +1185 -0
- barecat-0.3.0/tests/test_dict_api.py +322 -0
- barecat-0.3.0/tests/test_edge_cases.py +959 -0
- barecat-0.3.0/tests/test_fileobj_api.py +464 -0
- barecat-0.3.0/tests/test_glob_compliance.py +690 -0
- barecat-0.3.0/tests/test_glob_to_regex_hypothesis.py +407 -0
- barecat-0.3.0/tests/test_glob_to_sqlite.py +384 -0
- barecat-0.3.0/tests/test_index.py +547 -0
- barecat-0.3.0/tests/test_merge.py +1931 -0
- barecat-0.3.0/tests/test_open.py +440 -0
- barecat-0.3.0/tests/test_rsync.py +817 -0
- barecat-0.3.0/tests/test_shell.py +360 -0
- barecat-0.2.7/MANIFEST.in +0 -1
- barecat-0.2.7/pyproject.toml +0 -92
- barecat-0.2.7/src/barecat/__init__.py +0 -87
- barecat-0.2.7/src/barecat/cli_impl.py +0 -279
- barecat-0.2.7/src/barecat/codecs.py +0 -58
- barecat-0.2.7/src/barecat/defrag.py +0 -206
- barecat-0.2.7/src/barecat/glob_to_regex.py +0 -138
- barecat-0.2.7/src/barecat/threadsafe.py +0 -25
- barecat-0.2.7/src/barecat/upgrade_database.py +0 -113
- barecat-0.2.7/src/barecat/util.py +0 -297
- barecat-0.2.7/src/barecat/viewerqt6.py +0 -465
- barecat-0.2.7/src/barecat.egg-info/SOURCES.txt +0 -59
- barecat-0.2.7/src/barecat.egg-info/entry_points.txt +0 -15
- barecat-0.2.7/src/barecat.egg-info/requires.txt +0 -3
- barecat-0.2.7/tests/test_cli.py +0 -178
- {barecat-0.2.7 → barecat-0.3.0}/.github/workflows/python-publish.yml +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/.readthedocs.yaml +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/LICENSE +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/Makefile +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_static/styles/my_theme.css +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/index.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/attribute.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/class.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/data.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/exception.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/function.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/method.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/module.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/package.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/property.rst +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/copyright.html +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/abbrev_long.bib +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/conf.py +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/make.bat +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/references.bib +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/docs/requirements.txt +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/figure.png +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/setup.cfg +0 -0
- {barecat-0.2.7/src/barecat/core → barecat-0.3.0/src/barecat/cli}/__init__.py +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/dependency_links.txt +0 -0
- {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
name: Test Python ${{ matrix.python-version }}
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
with:
|
|
20
|
+
fetch-depth: 0
|
|
21
|
+
|
|
22
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
23
|
+
uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: ${{ matrix.python-version }}
|
|
26
|
+
|
|
27
|
+
- name: Install dependencies
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip
|
|
30
|
+
pip install -e ".[dev]"
|
|
31
|
+
|
|
32
|
+
- name: Run tests with coverage
|
|
33
|
+
run: pytest -v --cov=barecat --cov-report=xml
|
|
34
|
+
|
|
35
|
+
- name: Lint with ruff
|
|
36
|
+
run: |
|
|
37
|
+
ruff check .
|
|
38
|
+
ruff format --check .
|
|
@@ -3,6 +3,9 @@ __pycache__/
|
|
|
3
3
|
*.py[cod]
|
|
4
4
|
*$py.class
|
|
5
5
|
|
|
6
|
+
# setuptools-scm generated
|
|
7
|
+
src/barecat/_version.py
|
|
8
|
+
|
|
6
9
|
# C extensions
|
|
7
10
|
*.so
|
|
8
11
|
|
|
@@ -159,4 +162,4 @@ cython_debug/
|
|
|
159
162
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
160
163
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
161
164
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
162
|
-
|
|
165
|
+
.idea/
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Run `pre-commit install` to set up hooks
|
|
2
|
+
# Run `pre-commit autoupdate` to update hook versions
|
|
3
|
+
repos:
|
|
4
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
5
|
+
rev: v0.8.0
|
|
6
|
+
hooks:
|
|
7
|
+
- id: ruff
|
|
8
|
+
args: [--fix]
|
|
9
|
+
- id: ruff-format
|
|
10
|
+
|
|
11
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
12
|
+
rev: v5.0.0
|
|
13
|
+
hooks:
|
|
14
|
+
- id: trailing-whitespace
|
|
15
|
+
- id: end-of-file-fixer
|
|
16
|
+
- id: check-yaml
|
|
17
|
+
- id: check-added-large-files
|
barecat-0.3.0/Makefile
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
.PHONY: install test lint format docs clean
|
|
2
|
+
|
|
3
|
+
install:
|
|
4
|
+
pip install -e ".[dev]"
|
|
5
|
+
|
|
6
|
+
test:
|
|
7
|
+
pytest -v --cov=barecat
|
|
8
|
+
|
|
9
|
+
lint:
|
|
10
|
+
ruff check .
|
|
11
|
+
ruff format --check .
|
|
12
|
+
|
|
13
|
+
format:
|
|
14
|
+
ruff check --fix .
|
|
15
|
+
ruff format .
|
|
16
|
+
|
|
17
|
+
docs:
|
|
18
|
+
cd docs && make html
|
|
19
|
+
|
|
20
|
+
clean:
|
|
21
|
+
rm -rf build dist *.egg-info .pytest_cache .ruff_cache .coverage
|
|
22
|
+
find . -type d -name __pycache__ -exec rm -rf {} +
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: barecat
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Scalable archive format for storing millions of small files with random access and SQLite indexing.
|
|
5
5
|
Author-email: István Sárándi <istvan.sarandi@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -41,7 +41,11 @@ Classifier: Topic :: System :: Filesystems
|
|
|
41
41
|
Classifier: License :: OSI Approved :: MIT License
|
|
42
42
|
Classifier: Programming Language :: Python
|
|
43
43
|
Classifier: Programming Language :: Python :: 3
|
|
44
|
-
Classifier: Programming Language ::
|
|
44
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
45
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
46
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
47
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
48
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
45
49
|
Classifier: Operating System :: POSIX :: Linux
|
|
46
50
|
Requires-Python: >=3.9
|
|
47
51
|
Description-Content-Type: text/markdown
|
|
@@ -49,14 +53,42 @@ License-File: LICENSE
|
|
|
49
53
|
Requires-Dist: multiprocessing-utils
|
|
50
54
|
Requires-Dist: tqdm
|
|
51
55
|
Requires-Dist: crc32c
|
|
56
|
+
Provides-Extra: test
|
|
57
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
58
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
59
|
+
Provides-Extra: lint
|
|
60
|
+
Requires-Dist: ruff; extra == "lint"
|
|
61
|
+
Requires-Dist: pre-commit; extra == "lint"
|
|
62
|
+
Provides-Extra: docs
|
|
63
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
64
|
+
Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
|
|
65
|
+
Requires-Dist: sphinx-autoapi; extra == "docs"
|
|
66
|
+
Requires-Dist: sphinx-autobuild; extra == "docs"
|
|
67
|
+
Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
|
|
68
|
+
Requires-Dist: sphinxcontrib-prettyspecialmethods; extra == "docs"
|
|
69
|
+
Requires-Dist: sphinx-autodoc-napoleon-typehints; extra == "docs"
|
|
70
|
+
Requires-Dist: sphinx-codeautolink; extra == "docs"
|
|
71
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
72
|
+
Requires-Dist: pydata-sphinx-theme; extra == "docs"
|
|
73
|
+
Requires-Dist: sphinxcontrib-napoleon; extra == "docs"
|
|
74
|
+
Requires-Dist: Cython; extra == "docs"
|
|
75
|
+
Requires-Dist: numpy; extra == "docs"
|
|
76
|
+
Requires-Dist: setuptools-scm; extra == "docs"
|
|
77
|
+
Requires-Dist: toml; extra == "docs"
|
|
78
|
+
Provides-Extra: dev
|
|
79
|
+
Requires-Dist: barecat[docs,lint,test]; extra == "dev"
|
|
80
|
+
Provides-Extra: mount
|
|
81
|
+
Requires-Dist: barecat-mount; extra == "mount"
|
|
52
82
|
Dynamic: license-file
|
|
53
83
|
|
|
54
84
|
# Barecat
|
|
55
85
|
|
|
56
|
-
**[
|
|
86
|
+
**[Documentation](https://barecat.readthedocs.io)** |
|
|
87
|
+
**[API Reference](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)** |
|
|
88
|
+
**[GitHub](https://github.com/isarandi/barecat)**
|
|
57
89
|
|
|
58
90
|
Barecat (**bare** con**cat**enation) is a highly scalable, simple aggregate storage format for
|
|
59
|
-
storing many (tens of millions and more) small files, with focus on fast random access and
|
|
91
|
+
storing many (tens of millions and more) small files, with focus on fast random access and
|
|
60
92
|
minimal overhead.
|
|
61
93
|
|
|
62
94
|
Barecat can be thought of as a simple filesystem, or as something akin to an indexed tarball, or a
|
|
@@ -69,16 +101,28 @@ experience (`listdir`, `walk`, `glob`, etc).
|
|
|
69
101
|
|
|
70
102
|
Internally, all the data is simply concatenated one after another into one or more data shard files.
|
|
71
103
|
Additionally, an index is maintained in an SQLite database, which stores the shard number, the offset
|
|
72
|
-
and the size of each inner file (as well as a checksum, and further filesystem-like metadata
|
|
104
|
+
and the size of each inner file (as well as a checksum, and further filesystem-like metadata
|
|
73
105
|
like modification time). Barecat also maintains aggregate statistics for each directory, such as the
|
|
74
106
|
total number of files and total file size.
|
|
75
107
|
|
|
76
|
-
|
|
77
108
|

|
|
78
109
|
|
|
79
110
|
As you can see, the Barecat format is very simple. Readers/writers are easy to write in any language, since
|
|
80
111
|
SQLite is a widely-supported format.
|
|
81
112
|
|
|
113
|
+
## Installation
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install barecat
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Optional extras:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install barecat[mount] # FUSE mount support
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Requires Python 3.9+.
|
|
82
126
|
|
|
83
127
|
## Background
|
|
84
128
|
|
|
@@ -105,77 +149,126 @@ statistics (total file size, number of files) for each directory.
|
|
|
105
149
|
The index is stored in a separate SQLite database file, which itself does not need to be loaded
|
|
106
150
|
entirely into memory. Ideal for storing training image data for deep learning jobs.
|
|
107
151
|
- **Sharding**: To make it easier to move the data around or to distribute it across multiple
|
|
108
|
-
storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
|
|
152
|
+
storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
|
|
109
153
|
The shards do not have to be concatenated to be used, the library will keep all shard files open
|
|
110
154
|
and load data from the appropriate one during normal operations.
|
|
111
155
|
- **Fast browsing**: The SQLite database contains an index for the parent directories, allowing
|
|
112
156
|
fast listing of directory contents and aggregate statistics (total file size, number of files).
|
|
113
157
|
- **Intuitive API**: Familiar filesystem-like API, as well as a dictionary-like one.
|
|
114
|
-
- **Mountable**: The archive can be efficiently mounted in readonly or read-write mode.
|
|
158
|
+
- **Mountable**: The archive can be efficiently mounted in readonly or read-write mode via FUSE.
|
|
115
159
|
- **Simple storage format**: The files are simply concatenated after each other and the index contains
|
|
116
160
|
the offsets and sizes of each file. There is no header format to understand. The index can be
|
|
117
161
|
dumped into any format with simple SQL queries.
|
|
118
162
|
|
|
119
|
-
## Command
|
|
163
|
+
## Command Line Interface
|
|
120
164
|
|
|
121
|
-
|
|
122
|
-
are automatically installed executables with the pip package.
|
|
165
|
+
Barecat provides a unified `barecat` command with subcommands:
|
|
123
166
|
|
|
124
167
|
```bash
|
|
125
|
-
|
|
168
|
+
# Create archive from files and directories
|
|
169
|
+
barecat create mydata.barecat /path/to/images/
|
|
170
|
+
|
|
171
|
+
# Create from find output
|
|
172
|
+
find /data -name '*.jpg' -print0 | barecat create -0 mydata.barecat -T -
|
|
173
|
+
|
|
174
|
+
# Extract archive
|
|
175
|
+
barecat extract mydata.barecat -C /output/
|
|
176
|
+
|
|
177
|
+
# List contents
|
|
178
|
+
barecat list mydata.barecat
|
|
179
|
+
barecat list -l mydata.barecat # Long format with sizes
|
|
180
|
+
|
|
181
|
+
# Add files to an existing archive
|
|
182
|
+
barecat add mydata.barecat newdir/
|
|
183
|
+
|
|
184
|
+
# Print a file to stdout
|
|
185
|
+
barecat cat mydata.barecat path/to/file.txt
|
|
186
|
+
|
|
187
|
+
# Convert from/to tar or zip
|
|
188
|
+
barecat convert data.tar.gz data.barecat
|
|
189
|
+
|
|
190
|
+
# Interactive shell
|
|
191
|
+
barecat shell mydata.barecat
|
|
126
192
|
|
|
127
|
-
|
|
193
|
+
# Merge multiple archives
|
|
194
|
+
barecat merge -o combined.barecat archive1.barecat archive2.barecat
|
|
128
195
|
|
|
129
|
-
|
|
196
|
+
# Verify integrity
|
|
197
|
+
barecat verify mydata.barecat
|
|
198
|
+
|
|
199
|
+
# Defragment after deletions
|
|
200
|
+
barecat defrag mydata.barecat
|
|
201
|
+
|
|
202
|
+
# Mount via FUSE (requires barecat[mount])
|
|
203
|
+
barecat mount mydata.barecat /mnt/mydata
|
|
130
204
|
```
|
|
131
205
|
|
|
132
206
|
This may yield the following files:
|
|
133
207
|
|
|
134
208
|
```
|
|
135
|
-
mydata.barecat
|
|
136
|
-
mydata.barecat-shard-
|
|
137
|
-
mydata.barecat-
|
|
209
|
+
mydata.barecat # SQLite index database
|
|
210
|
+
mydata.barecat-shard-00000 # Data shard 0
|
|
211
|
+
mydata.barecat-shard-00001 # Data shard 1
|
|
138
212
|
```
|
|
139
213
|
|
|
140
|
-
The files can be extracted out
|
|
214
|
+
The files can be extracted back out. Unix-like permissions, modification times, and owner info are
|
|
141
215
|
preserved.
|
|
142
216
|
|
|
143
|
-
|
|
144
|
-
barecat-extract --file=mydata.barecat --target-directory=targetdir/
|
|
145
|
-
```
|
|
217
|
+
See the [documentation](https://barecat.readthedocs.io) for the full CLI reference.
|
|
146
218
|
|
|
147
219
|
## Python API
|
|
148
220
|
|
|
149
221
|
```python
|
|
150
|
-
|
|
151
222
|
import barecat
|
|
152
223
|
|
|
224
|
+
# Write
|
|
153
225
|
with barecat.Barecat('mydata.barecat', readonly=False) as bc:
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
226
|
+
bc['path/to/file.jpg'] = binary_file_data
|
|
227
|
+
bc.add_by_path('path/to/file/on/disk.jpg')
|
|
228
|
+
|
|
229
|
+
with open('local_file.bin', 'rb') as f:
|
|
230
|
+
bc.add('path/in/archive.bin', fileobj=f)
|
|
231
|
+
|
|
232
|
+
# Read
|
|
160
233
|
with barecat.Barecat('mydata.barecat') as bc:
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
234
|
+
binary_file_data = bc['path/to/file.jpg']
|
|
235
|
+
entrynames = bc.listdir('path/to')
|
|
236
|
+
|
|
237
|
+
for root, dirs, files in bc.walk('path/to/something'):
|
|
238
|
+
print(root, dirs, files)
|
|
239
|
+
|
|
240
|
+
paths = bc.glob('path/to/**/*.jpg', recursive=True)
|
|
241
|
+
|
|
242
|
+
with bc.open('path/to/file.jpg', 'rb') as f:
|
|
243
|
+
data = f.read(123)
|
|
170
244
|
```
|
|
171
245
|
|
|
172
|
-
|
|
246
|
+
### Automatic Encoding/Decoding
|
|
173
247
|
|
|
174
|
-
|
|
175
|
-
archive.
|
|
248
|
+
Wrap the archive with `DecodedView` for automatic serialization based on file extension:
|
|
176
249
|
|
|
177
|
-
```
|
|
178
|
-
barecat
|
|
250
|
+
```python
|
|
251
|
+
from barecat import Barecat, DecodedView
|
|
252
|
+
import numpy as np
|
|
253
|
+
|
|
254
|
+
with Barecat('data.barecat', readonly=False) as bc:
|
|
255
|
+
dec = DecodedView(bc)
|
|
256
|
+
dec['config.json'] = {'learning_rate': 0.001}
|
|
257
|
+
dec['weights.npy'] = np.random.randn(100, 100)
|
|
258
|
+
|
|
259
|
+
with Barecat('data.barecat') as bc:
|
|
260
|
+
dec = DecodedView(bc)
|
|
261
|
+
config = dec['config.json'] # Returns dict
|
|
262
|
+
weights = dec['weights.npy'] # Returns numpy array
|
|
179
263
|
```
|
|
180
264
|
|
|
181
|
-
|
|
265
|
+
Supported formats: `.json`, `.pkl`, `.npy`, `.npz`, `.msgpack`, images (`.png`, `.jpg`, etc.),
|
|
266
|
+
with stackable compression (`.gz`, `.bz2`, `.xz`).
|
|
267
|
+
|
|
268
|
+
## Documentation
|
|
269
|
+
|
|
270
|
+
Full documentation is available at [barecat.readthedocs.io](https://barecat.readthedocs.io).
|
|
271
|
+
|
|
272
|
+
## License
|
|
273
|
+
|
|
274
|
+
MIT
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
# Barecat
|
|
2
2
|
|
|
3
|
-
**[
|
|
3
|
+
**[Documentation](https://barecat.readthedocs.io)** |
|
|
4
|
+
**[API Reference](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)** |
|
|
5
|
+
**[GitHub](https://github.com/isarandi/barecat)**
|
|
4
6
|
|
|
5
7
|
Barecat (**bare** con**cat**enation) is a highly scalable, simple aggregate storage format for
|
|
6
|
-
storing many (tens of millions and more) small files, with focus on fast random access and
|
|
8
|
+
storing many (tens of millions and more) small files, with focus on fast random access and
|
|
7
9
|
minimal overhead.
|
|
8
10
|
|
|
9
11
|
Barecat can be thought of as a simple filesystem, or as something akin to an indexed tarball, or a
|
|
@@ -16,16 +18,28 @@ experience (`listdir`, `walk`, `glob`, etc).
|
|
|
16
18
|
|
|
17
19
|
Internally, all the data is simply concatenated one after another into one or more data shard files.
|
|
18
20
|
Additionally, an index is maintained in an SQLite database, which stores the shard number, the offset
|
|
19
|
-
and the size of each inner file (as well as a checksum, and further filesystem-like metadata
|
|
21
|
+
and the size of each inner file (as well as a checksum, and further filesystem-like metadata
|
|
20
22
|
like modification time). Barecat also maintains aggregate statistics for each directory, such as the
|
|
21
23
|
total number of files and total file size.
|
|
22
24
|
|
|
23
|
-
|
|
24
25
|

|
|
25
26
|
|
|
26
27
|
As you can see, the Barecat format is very simple. Readers/writers are easy to write in any language, since
|
|
27
28
|
SQLite is a widely-supported format.
|
|
28
29
|
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install barecat
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Optional extras:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install barecat[mount] # FUSE mount support
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Requires Python 3.9+.
|
|
29
43
|
|
|
30
44
|
## Background
|
|
31
45
|
|
|
@@ -52,77 +66,126 @@ statistics (total file size, number of files) for each directory.
|
|
|
52
66
|
The index is stored in a separate SQLite database file, which itself does not need to be loaded
|
|
53
67
|
entirely into memory. Ideal for storing training image data for deep learning jobs.
|
|
54
68
|
- **Sharding**: To make it easier to move the data around or to distribute it across multiple
|
|
55
|
-
storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
|
|
69
|
+
storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
|
|
56
70
|
The shards do not have to be concatenated to be used, the library will keep all shard files open
|
|
57
71
|
and load data from the appropriate one during normal operations.
|
|
58
72
|
- **Fast browsing**: The SQLite database contains an index for the parent directories, allowing
|
|
59
73
|
fast listing of directory contents and aggregate statistics (total file size, number of files).
|
|
60
74
|
- **Intuitive API**: Familiar filesystem-like API, as well as a dictionary-like one.
|
|
61
|
-
- **Mountable**: The archive can be efficiently mounted in readonly or read-write mode.
|
|
75
|
+
- **Mountable**: The archive can be efficiently mounted in readonly or read-write mode via FUSE.
|
|
62
76
|
- **Simple storage format**: The files are simply concatenated after each other and the index contains
|
|
63
77
|
the offsets and sizes of each file. There is no header format to understand. The index can be
|
|
64
78
|
dumped into any format with simple SQL queries.
|
|
65
79
|
|
|
66
|
-
## Command
|
|
80
|
+
## Command Line Interface
|
|
67
81
|
|
|
68
|
-
|
|
69
|
-
are automatically installed executables with the pip package.
|
|
82
|
+
Barecat provides a unified `barecat` command with subcommands:
|
|
70
83
|
|
|
71
84
|
```bash
|
|
72
|
-
|
|
85
|
+
# Create archive from files and directories
|
|
86
|
+
barecat create mydata.barecat /path/to/images/
|
|
87
|
+
|
|
88
|
+
# Create from find output
|
|
89
|
+
find /data -name '*.jpg' -print0 | barecat create -0 mydata.barecat -T -
|
|
90
|
+
|
|
91
|
+
# Extract archive
|
|
92
|
+
barecat extract mydata.barecat -C /output/
|
|
93
|
+
|
|
94
|
+
# List contents
|
|
95
|
+
barecat list mydata.barecat
|
|
96
|
+
barecat list -l mydata.barecat # Long format with sizes
|
|
97
|
+
|
|
98
|
+
# Add files to an existing archive
|
|
99
|
+
barecat add mydata.barecat newdir/
|
|
100
|
+
|
|
101
|
+
# Print a file to stdout
|
|
102
|
+
barecat cat mydata.barecat path/to/file.txt
|
|
103
|
+
|
|
104
|
+
# Convert from/to tar or zip
|
|
105
|
+
barecat convert data.tar.gz data.barecat
|
|
106
|
+
|
|
107
|
+
# Interactive shell
|
|
108
|
+
barecat shell mydata.barecat
|
|
73
109
|
|
|
74
|
-
|
|
110
|
+
# Merge multiple archives
|
|
111
|
+
barecat merge -o combined.barecat archive1.barecat archive2.barecat
|
|
75
112
|
|
|
76
|
-
|
|
113
|
+
# Verify integrity
|
|
114
|
+
barecat verify mydata.barecat
|
|
115
|
+
|
|
116
|
+
# Defragment after deletions
|
|
117
|
+
barecat defrag mydata.barecat
|
|
118
|
+
|
|
119
|
+
# Mount via FUSE (requires barecat[mount])
|
|
120
|
+
barecat mount mydata.barecat /mnt/mydata
|
|
77
121
|
```
|
|
78
122
|
|
|
79
123
|
This may yield the following files:
|
|
80
124
|
|
|
81
125
|
```
|
|
82
|
-
mydata.barecat
|
|
83
|
-
mydata.barecat-shard-
|
|
84
|
-
mydata.barecat-
|
|
126
|
+
mydata.barecat # SQLite index database
|
|
127
|
+
mydata.barecat-shard-00000 # Data shard 0
|
|
128
|
+
mydata.barecat-shard-00001 # Data shard 1
|
|
85
129
|
```
|
|
86
130
|
|
|
87
|
-
The files can be extracted out
|
|
131
|
+
The files can be extracted back out. Unix-like permissions, modification times, and owner info are
|
|
88
132
|
preserved.
|
|
89
133
|
|
|
90
|
-
|
|
91
|
-
barecat-extract --file=mydata.barecat --target-directory=targetdir/
|
|
92
|
-
```
|
|
134
|
+
See the [documentation](https://barecat.readthedocs.io) for the full CLI reference.
|
|
93
135
|
|
|
94
136
|
## Python API
|
|
95
137
|
|
|
96
138
|
```python
|
|
97
|
-
|
|
98
139
|
import barecat
|
|
99
140
|
|
|
141
|
+
# Write
|
|
100
142
|
with barecat.Barecat('mydata.barecat', readonly=False) as bc:
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
143
|
+
bc['path/to/file.jpg'] = binary_file_data
|
|
144
|
+
bc.add_by_path('path/to/file/on/disk.jpg')
|
|
145
|
+
|
|
146
|
+
with open('local_file.bin', 'rb') as f:
|
|
147
|
+
bc.add('path/in/archive.bin', fileobj=f)
|
|
148
|
+
|
|
149
|
+
# Read
|
|
107
150
|
with barecat.Barecat('mydata.barecat') as bc:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
151
|
+
binary_file_data = bc['path/to/file.jpg']
|
|
152
|
+
entrynames = bc.listdir('path/to')
|
|
153
|
+
|
|
154
|
+
for root, dirs, files in bc.walk('path/to/something'):
|
|
155
|
+
print(root, dirs, files)
|
|
156
|
+
|
|
157
|
+
paths = bc.glob('path/to/**/*.jpg', recursive=True)
|
|
158
|
+
|
|
159
|
+
with bc.open('path/to/file.jpg', 'rb') as f:
|
|
160
|
+
data = f.read(123)
|
|
117
161
|
```
|
|
118
162
|
|
|
119
|
-
|
|
163
|
+
### Automatic Encoding/Decoding
|
|
120
164
|
|
|
121
|
-
|
|
122
|
-
archive.
|
|
165
|
+
Wrap the archive with `DecodedView` for automatic serialization based on file extension:
|
|
123
166
|
|
|
124
|
-
```
|
|
125
|
-
barecat
|
|
167
|
+
```python
|
|
168
|
+
from barecat import Barecat, DecodedView
|
|
169
|
+
import numpy as np
|
|
170
|
+
|
|
171
|
+
with Barecat('data.barecat', readonly=False) as bc:
|
|
172
|
+
dec = DecodedView(bc)
|
|
173
|
+
dec['config.json'] = {'learning_rate': 0.001}
|
|
174
|
+
dec['weights.npy'] = np.random.randn(100, 100)
|
|
175
|
+
|
|
176
|
+
with Barecat('data.barecat') as bc:
|
|
177
|
+
dec = DecodedView(bc)
|
|
178
|
+
config = dec['config.json'] # Returns dict
|
|
179
|
+
weights = dec['weights.npy'] # Returns numpy array
|
|
126
180
|
```
|
|
127
181
|
|
|
128
|
-
|
|
182
|
+
Supported formats: `.json`, `.pkl`, `.npy`, `.npz`, `.msgpack`, images (`.png`, `.jpg`, etc.),
|
|
183
|
+
with stackable compression (`.gz`, `.bz2`, `.xz`).
|
|
184
|
+
|
|
185
|
+
## Documentation
|
|
186
|
+
|
|
187
|
+
Full documentation is available at [barecat.readthedocs.io](https://barecat.readthedocs.io).
|
|
188
|
+
|
|
189
|
+
## License
|
|
190
|
+
|
|
191
|
+
MIT
|