barecat 0.2.7__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. barecat-0.3.0/.github/workflows/ci.yml +38 -0
  2. {barecat-0.2.7 → barecat-0.3.0}/.gitignore +4 -1
  3. barecat-0.3.0/.pre-commit-config.yaml +17 -0
  4. barecat-0.3.0/MANIFEST.in +4 -0
  5. barecat-0.3.0/Makefile +22 -0
  6. {barecat-0.2.7 → barecat-0.3.0}/PKG-INFO +136 -43
  7. {barecat-0.2.7 → barecat-0.3.0}/README.md +104 -41
  8. barecat-0.3.0/docs/explanation/architecture.rst +430 -0
  9. barecat-0.3.0/docs/explanation/comparison.rst +271 -0
  10. barecat-0.3.0/docs/explanation/integrity.rst +140 -0
  11. barecat-0.3.0/docs/explanation/performance.rst +261 -0
  12. barecat-0.3.0/docs/howto/automatic-encoding.rst +197 -0
  13. barecat-0.3.0/docs/howto/convert-archives.rst +199 -0
  14. barecat-0.3.0/docs/howto/merge-archives.rst +197 -0
  15. barecat-0.3.0/docs/howto/mount-fuse.rst +224 -0
  16. barecat-0.3.0/docs/howto/pytorch-dataloader.rst +319 -0
  17. barecat-0.3.0/docs/howto/shell-completions.rst +126 -0
  18. barecat-0.3.0/docs/howto/verify-repair.rst +197 -0
  19. {barecat-0.2.7 → barecat-0.3.0}/docs/index.rst +53 -25
  20. barecat-0.3.0/docs/reference/cli.rst +782 -0
  21. barecat-0.3.0/docs/reference/file-format.rst +291 -0
  22. barecat-0.3.0/docs/reference/python-api.rst +509 -0
  23. barecat-0.3.0/docs/tutorials/getting-started.rst +232 -0
  24. barecat-0.3.0/docs/tutorials/image-datasets.rst +262 -0
  25. barecat-0.3.0/pyproject.toml +145 -0
  26. barecat-0.3.0/src/barecat/__init__.py +108 -0
  27. barecat-0.3.0/src/barecat/__main__.py +5 -0
  28. barecat-0.3.0/src/barecat/_api.py +131 -0
  29. {barecat-0.2.7 → barecat-0.3.0}/src/barecat/_version.py +3 -3
  30. barecat-0.3.0/src/barecat/cli/commands.py +540 -0
  31. barecat-0.3.0/src/barecat/cli/completions.py +27 -0
  32. barecat-0.3.0/src/barecat/cli/impl.py +609 -0
  33. barecat-0.2.7/src/barecat/cli.py → barecat-0.3.0/src/barecat/cli/legacy.py +112 -11
  34. barecat-0.3.0/src/barecat/cli/main.py +1355 -0
  35. barecat-0.3.0/src/barecat/cli/shell.py +827 -0
  36. barecat-0.3.0/src/barecat/completions/__init__.py +1 -0
  37. barecat-0.3.0/src/barecat/completions/barecat.bash +246 -0
  38. barecat-0.3.0/src/barecat/completions/barecat.zsh +268 -0
  39. barecat-0.3.0/src/barecat/core/__init__.py +0 -0
  40. {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/barecat.py +281 -256
  41. {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/index.py +603 -416
  42. barecat-0.3.0/src/barecat/core/paths.py +61 -0
  43. {barecat-0.2.7 → barecat-0.3.0}/src/barecat/core/sharder.py +76 -47
  44. barecat-0.2.7/src/barecat/common.py → barecat-0.3.0/src/barecat/core/types.py +46 -146
  45. {barecat-0.2.7 → barecat-0.3.0}/src/barecat/exceptions.py +13 -0
  46. barecat-0.3.0/src/barecat/formats/__init__.py +0 -0
  47. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/archive_formats.py +102 -64
  48. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/from_tar_stream.py +2 -2
  49. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/formats}/to_tar_stream.py +10 -10
  50. barecat-0.3.0/src/barecat/io/__init__.py +5 -0
  51. barecat-0.3.0/src/barecat/io/codecs.py +622 -0
  52. barecat-0.3.0/src/barecat/io/copyfile.py +550 -0
  53. barecat-0.3.0/src/barecat/io/fileobj.py +556 -0
  54. barecat-0.3.0/src/barecat/io/open.py +41 -0
  55. barecat-0.3.0/src/barecat/maintenance/__init__.py +0 -0
  56. barecat-0.3.0/src/barecat/maintenance/complete_database.py +79 -0
  57. barecat-0.3.0/src/barecat/maintenance/defrag.py +541 -0
  58. barecat-0.3.0/src/barecat/maintenance/merge.py +613 -0
  59. barecat-0.3.0/src/barecat/maintenance/reshard.py +153 -0
  60. barecat-0.3.0/src/barecat/maintenance/rsync.py +1282 -0
  61. barecat-0.3.0/src/barecat/maintenance/upgrade_database.py +200 -0
  62. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/maintenance}/upgrade_database2.py +15 -14
  63. barecat-0.3.0/src/barecat/man/barecat.1 +284 -0
  64. barecat-0.3.0/src/barecat/sql/indexes.sql +6 -0
  65. barecat-0.3.0/src/barecat/sql/schema.sql +55 -0
  66. barecat-0.2.7/src/barecat/sql/schema.sql → barecat-0.3.0/src/barecat/sql/triggers.sql +33 -78
  67. barecat-0.3.0/src/barecat/tui/__init__.py +0 -0
  68. barecat-0.3.0/src/barecat/tui/browse.py +448 -0
  69. barecat-0.3.0/src/barecat/tui/ncdu.py +318 -0
  70. barecat-0.3.0/src/barecat/util/__init__.py +0 -0
  71. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/util}/consumed_threadpool.py +2 -0
  72. barecat-0.3.0/src/barecat/util/glob_helper.py +584 -0
  73. barecat-0.3.0/src/barecat/util/glob_to_regex.py +353 -0
  74. barecat-0.3.0/src/barecat/util/list_files.py +12 -0
  75. barecat-0.3.0/src/barecat/util/misc.py +174 -0
  76. barecat-0.3.0/src/barecat/util/physical_order.py +77 -0
  77. {barecat-0.2.7/src/barecat → barecat-0.3.0/src/barecat/util}/progbar.py +3 -0
  78. barecat-0.3.0/src/barecat/util/threading.py +59 -0
  79. {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/PKG-INFO +136 -43
  80. barecat-0.3.0/src/barecat.egg-info/SOURCES.txt +121 -0
  81. barecat-0.3.0/src/barecat.egg-info/entry_points.txt +15 -0
  82. barecat-0.3.0/src/barecat.egg-info/requires.txt +34 -0
  83. {barecat-0.2.7 → barecat-0.3.0}/tests/test_barecat.py +2 -3
  84. barecat-0.3.0/tests/test_cli.py +941 -0
  85. barecat-0.3.0/tests/test_codecs.py +402 -0
  86. barecat-0.3.0/tests/test_copyfile.py +1185 -0
  87. barecat-0.3.0/tests/test_dict_api.py +322 -0
  88. barecat-0.3.0/tests/test_edge_cases.py +959 -0
  89. barecat-0.3.0/tests/test_fileobj_api.py +464 -0
  90. barecat-0.3.0/tests/test_glob_compliance.py +690 -0
  91. barecat-0.3.0/tests/test_glob_to_regex_hypothesis.py +407 -0
  92. barecat-0.3.0/tests/test_glob_to_sqlite.py +384 -0
  93. barecat-0.3.0/tests/test_index.py +547 -0
  94. barecat-0.3.0/tests/test_merge.py +1931 -0
  95. barecat-0.3.0/tests/test_open.py +440 -0
  96. barecat-0.3.0/tests/test_rsync.py +817 -0
  97. barecat-0.3.0/tests/test_shell.py +360 -0
  98. barecat-0.2.7/MANIFEST.in +0 -1
  99. barecat-0.2.7/pyproject.toml +0 -92
  100. barecat-0.2.7/src/barecat/__init__.py +0 -87
  101. barecat-0.2.7/src/barecat/cli_impl.py +0 -279
  102. barecat-0.2.7/src/barecat/codecs.py +0 -58
  103. barecat-0.2.7/src/barecat/defrag.py +0 -206
  104. barecat-0.2.7/src/barecat/glob_to_regex.py +0 -138
  105. barecat-0.2.7/src/barecat/threadsafe.py +0 -25
  106. barecat-0.2.7/src/barecat/upgrade_database.py +0 -113
  107. barecat-0.2.7/src/barecat/util.py +0 -297
  108. barecat-0.2.7/src/barecat/viewerqt6.py +0 -465
  109. barecat-0.2.7/src/barecat.egg-info/SOURCES.txt +0 -59
  110. barecat-0.2.7/src/barecat.egg-info/entry_points.txt +0 -15
  111. barecat-0.2.7/src/barecat.egg-info/requires.txt +0 -3
  112. barecat-0.2.7/tests/test_cli.py +0 -178
  113. {barecat-0.2.7 → barecat-0.3.0}/.github/workflows/python-publish.yml +0 -0
  114. {barecat-0.2.7 → barecat-0.3.0}/.readthedocs.yaml +0 -0
  115. {barecat-0.2.7 → barecat-0.3.0}/LICENSE +0 -0
  116. {barecat-0.2.7 → barecat-0.3.0}/docs/Makefile +0 -0
  117. {barecat-0.2.7 → barecat-0.3.0}/docs/_static/styles/my_theme.css +0 -0
  118. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/index.rst +0 -0
  119. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/attribute.rst +0 -0
  120. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/class.rst +0 -0
  121. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/data.rst +0 -0
  122. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/exception.rst +0 -0
  123. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/function.rst +0 -0
  124. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/method.rst +0 -0
  125. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/module.rst +0 -0
  126. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/package.rst +0 -0
  127. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/autoapi/python/property.rst +0 -0
  128. {barecat-0.2.7 → barecat-0.3.0}/docs/_templates/copyright.html +0 -0
  129. {barecat-0.2.7 → barecat-0.3.0}/docs/abbrev_long.bib +0 -0
  130. {barecat-0.2.7 → barecat-0.3.0}/docs/conf.py +0 -0
  131. {barecat-0.2.7 → barecat-0.3.0}/docs/make.bat +0 -0
  132. {barecat-0.2.7 → barecat-0.3.0}/docs/references.bib +0 -0
  133. {barecat-0.2.7 → barecat-0.3.0}/docs/requirements.txt +0 -0
  134. {barecat-0.2.7 → barecat-0.3.0}/figure.png +0 -0
  135. {barecat-0.2.7 → barecat-0.3.0}/setup.cfg +0 -0
  136. {barecat-0.2.7/src/barecat/core → barecat-0.3.0/src/barecat/cli}/__init__.py +0 -0
  137. {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/dependency_links.txt +0 -0
  138. {barecat-0.2.7 → barecat-0.3.0}/src/barecat.egg-info/top_level.txt +0 -0
@@ -0,0 +1,38 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ test:
11
+ name: Test Python ${{ matrix.python-version }}
12
+ runs-on: ubuntu-latest
13
+ strategy:
14
+ matrix:
15
+ python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
16
+
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ with:
20
+ fetch-depth: 0
21
+
22
+ - name: Set up Python ${{ matrix.python-version }}
23
+ uses: actions/setup-python@v5
24
+ with:
25
+ python-version: ${{ matrix.python-version }}
26
+
27
+ - name: Install dependencies
28
+ run: |
29
+ python -m pip install --upgrade pip
30
+ pip install -e ".[dev]"
31
+
32
+ - name: Run tests with coverage
33
+ run: pytest -v --cov=barecat --cov-report=xml
34
+
35
+ - name: Lint with ruff
36
+ run: |
37
+ ruff check .
38
+ ruff format --check .
@@ -3,6 +3,9 @@ __pycache__/
3
3
  *.py[cod]
4
4
  *$py.class
5
5
 
6
+ # setuptools-scm generated
7
+ src/barecat/_version.py
8
+
6
9
  # C extensions
7
10
  *.so
8
11
 
@@ -159,4 +162,4 @@ cython_debug/
159
162
  # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
160
163
  # and can be added to the global gitignore or merged into this file. For a more nuclear
161
164
  # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
- #.idea/
165
+ .idea/
@@ -0,0 +1,17 @@
1
+ # Run `pre-commit install` to set up hooks
2
+ # Run `pre-commit autoupdate` to update hook versions
3
+ repos:
4
+ - repo: https://github.com/astral-sh/ruff-pre-commit
5
+ rev: v0.8.0
6
+ hooks:
7
+ - id: ruff
8
+ args: [--fix]
9
+ - id: ruff-format
10
+
11
+ - repo: https://github.com/pre-commit/pre-commit-hooks
12
+ rev: v5.0.0
13
+ hooks:
14
+ - id: trailing-whitespace
15
+ - id: end-of-file-fixer
16
+ - id: check-yaml
17
+ - id: check-added-large-files
@@ -0,0 +1,4 @@
1
+ include src/barecat/sql/*.sql
2
+ include src/barecat/man/*.1
3
+ include src/barecat/completions/*.bash
4
+ include src/barecat/completions/*.zsh
barecat-0.3.0/Makefile ADDED
@@ -0,0 +1,22 @@
1
+ .PHONY: install test lint format docs clean
2
+
3
+ install:
4
+ pip install -e ".[dev]"
5
+
6
+ test:
7
+ pytest -v --cov=barecat
8
+
9
+ lint:
10
+ ruff check .
11
+ ruff format --check .
12
+
13
+ format:
14
+ ruff check --fix .
15
+ ruff format .
16
+
17
+ docs:
18
+ cd docs && make html
19
+
20
+ clean:
21
+ rm -rf build dist *.egg-info .pytest_cache .ruff_cache .coverage
22
+ find . -type d -name __pycache__ -exec rm -rf {} +
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: barecat
3
- Version: 0.2.7
3
+ Version: 0.3.0
4
4
  Summary: Scalable archive format for storing millions of small files with random access and SQLite indexing.
5
5
  Author-email: István Sárándi <istvan.sarandi@gmail.com>
6
6
  License: MIT License
@@ -41,7 +41,11 @@ Classifier: Topic :: System :: Filesystems
41
41
  Classifier: License :: OSI Approved :: MIT License
42
42
  Classifier: Programming Language :: Python
43
43
  Classifier: Programming Language :: Python :: 3
44
- Classifier: Programming Language :: Cython
44
+ Classifier: Programming Language :: Python :: 3.9
45
+ Classifier: Programming Language :: Python :: 3.10
46
+ Classifier: Programming Language :: Python :: 3.11
47
+ Classifier: Programming Language :: Python :: 3.12
48
+ Classifier: Programming Language :: Python :: 3.13
45
49
  Classifier: Operating System :: POSIX :: Linux
46
50
  Requires-Python: >=3.9
47
51
  Description-Content-Type: text/markdown
@@ -49,14 +53,42 @@ License-File: LICENSE
49
53
  Requires-Dist: multiprocessing-utils
50
54
  Requires-Dist: tqdm
51
55
  Requires-Dist: crc32c
56
+ Provides-Extra: test
57
+ Requires-Dist: pytest>=7.0; extra == "test"
58
+ Requires-Dist: pytest-cov; extra == "test"
59
+ Provides-Extra: lint
60
+ Requires-Dist: ruff; extra == "lint"
61
+ Requires-Dist: pre-commit; extra == "lint"
62
+ Provides-Extra: docs
63
+ Requires-Dist: sphinx; extra == "docs"
64
+ Requires-Dist: sphinxcontrib-bibtex; extra == "docs"
65
+ Requires-Dist: sphinx-autoapi; extra == "docs"
66
+ Requires-Dist: sphinx-autobuild; extra == "docs"
67
+ Requires-Dist: sphinx-autodoc-typehints; extra == "docs"
68
+ Requires-Dist: sphinxcontrib-prettyspecialmethods; extra == "docs"
69
+ Requires-Dist: sphinx-autodoc-napoleon-typehints; extra == "docs"
70
+ Requires-Dist: sphinx-codeautolink; extra == "docs"
71
+ Requires-Dist: sphinx-rtd-theme; extra == "docs"
72
+ Requires-Dist: pydata-sphinx-theme; extra == "docs"
73
+ Requires-Dist: sphinxcontrib-napoleon; extra == "docs"
74
+ Requires-Dist: Cython; extra == "docs"
75
+ Requires-Dist: numpy; extra == "docs"
76
+ Requires-Dist: setuptools-scm; extra == "docs"
77
+ Requires-Dist: toml; extra == "docs"
78
+ Provides-Extra: dev
79
+ Requires-Dist: barecat[docs,lint,test]; extra == "dev"
80
+ Provides-Extra: mount
81
+ Requires-Dist: barecat-mount; extra == "mount"
52
82
  Dynamic: license-file
53
83
 
54
84
  # Barecat
55
85
 
56
- **[Full API Reference Docs](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)**
86
+ **[Documentation](https://barecat.readthedocs.io)** |
87
+ **[API Reference](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)** |
88
+ **[GitHub](https://github.com/isarandi/barecat)**
57
89
 
58
90
  Barecat (**bare** con**cat**enation) is a highly scalable, simple aggregate storage format for
59
- storing many (tens of millions and more) small files, with focus on fast random access and
91
+ storing many (tens of millions and more) small files, with focus on fast random access and
60
92
  minimal overhead.
61
93
 
62
94
  Barecat can be thought of as a simple filesystem, or as something akin to an indexed tarball, or a
@@ -69,16 +101,28 @@ experience (`listdir`, `walk`, `glob`, etc).
69
101
 
70
102
  Internally, all the data is simply concatenated one after another into one or more data shard files.
71
103
  Additionally, an index is maintained in an SQLite database, which stores the shard number, the offset
72
- and the size of each inner file (as well as a checksum, and further filesystem-like metadata
104
+ and the size of each inner file (as well as a checksum, and further filesystem-like metadata
73
105
  like modification time). Barecat also maintains aggregate statistics for each directory, such as the
74
106
  total number of files and total file size.
75
107
 
76
-
77
108
  ![Architecture](./figure.png)
78
109
 
79
110
  As you can see, the Barecat format is very simple. Readers/writers are easy to write in any language, since
80
111
  SQLite is a widely-supported format.
81
112
 
113
+ ## Installation
114
+
115
+ ```bash
116
+ pip install barecat
117
+ ```
118
+
119
+ Optional extras:
120
+
121
+ ```bash
122
+ pip install barecat[mount] # FUSE mount support
123
+ ```
124
+
125
+ Requires Python 3.9+.
82
126
 
83
127
  ## Background
84
128
 
@@ -105,77 +149,126 @@ statistics (total file size, number of files) for each directory.
105
149
  The index is stored in a separate SQLite database file, which itself does not need to be loaded
106
150
  entirely into memory. Ideal for storing training image data for deep learning jobs.
107
151
  - **Sharding**: To make it easier to move the data around or to distribute it across multiple
108
- storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
152
+ storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
109
153
  The shards do not have to be concatenated to be used, the library will keep all shard files open
110
154
  and load data from the appropriate one during normal operations.
111
155
  - **Fast browsing**: The SQLite database contains an index for the parent directories, allowing
112
156
  fast listing of directory contents and aggregate statistics (total file size, number of files).
113
157
  - **Intuitive API**: Familiar filesystem-like API, as well as a dictionary-like one.
114
- - **Mountable**: The archive can be efficiently mounted in readonly or read-write mode.
158
+ - **Mountable**: The archive can be efficiently mounted in readonly or read-write mode via FUSE.
115
159
  - **Simple storage format**: The files are simply concatenated after each other and the index contains
116
160
  the offsets and sizes of each file. There is no header format to understand. The index can be
117
161
  dumped into any format with simple SQL queries.
118
162
 
119
- ## Command line interface
163
+ ## Command Line Interface
120
164
 
121
- To create a Barecat archive, use the `barecat-create` or `barecat-create-recursive` commands, which
122
- are automatically installed executables with the pip package.
165
+ Barecat provides a unified `barecat` command with subcommands:
123
166
 
124
167
  ```bash
125
- barecat-create --file=mydata.barecat --shard-size=100G < path_of_paths.txt
168
+ # Create archive from files and directories
169
+ barecat create mydata.barecat /path/to/images/
170
+
171
+ # Create from find output
172
+ find /data -name '*.jpg' -print0 | barecat create -0 mydata.barecat -T -
173
+
174
+ # Extract archive
175
+ barecat extract mydata.barecat -C /output/
176
+
177
+ # List contents
178
+ barecat list mydata.barecat
179
+ barecat list -l mydata.barecat # Long format with sizes
180
+
181
+ # Add files to an existing archive
182
+ barecat add mydata.barecat newdir/
183
+
184
+ # Print a file to stdout
185
+ barecat cat mydata.barecat path/to/file.txt
186
+
187
+ # Convert from/to tar or zip
188
+ barecat convert data.tar.gz data.barecat
189
+
190
+ # Interactive shell
191
+ barecat shell mydata.barecat
126
192
 
127
- find dirname -name '*.jpg' -print0 | barecat-create --null --file=mydata.barecat --shard-size=100G
193
+ # Merge multiple archives
194
+ barecat merge -o combined.barecat archive1.barecat archive2.barecat
128
195
 
129
- barecat-create-recursive dir1 dir2 dir3 --file=mydata.barecat --shard-size=100G
196
+ # Verify integrity
197
+ barecat verify mydata.barecat
198
+
199
+ # Defragment after deletions
200
+ barecat defrag mydata.barecat
201
+
202
+ # Mount via FUSE (requires barecat[mount])
203
+ barecat mount mydata.barecat /mnt/mydata
130
204
  ```
131
205
 
132
206
  This may yield the following files:
133
207
 
134
208
  ```
135
- mydata.barecat-shard-00001
136
- mydata.barecat-shard-00002
137
- mydata.barecat-sqlite-index
209
+ mydata.barecat # SQLite index database
210
+ mydata.barecat-shard-00000 # Data shard 0
211
+ mydata.barecat-shard-00001 # Data shard 1
138
212
  ```
139
213
 
140
- The files can be extracted out again. Unix-like permissions, modification times, owner info are
214
+ The files can be extracted back out. Unix-like permissions, modification times, and owner info are
141
215
  preserved.
142
216
 
143
- ```bash
144
- barecat-extract --file=mydata.barecat --target-directory=targetdir/
145
- ```
217
+ See the [documentation](https://barecat.readthedocs.io) for the full CLI reference.
146
218
 
147
219
  ## Python API
148
220
 
149
221
  ```python
150
-
151
222
  import barecat
152
223
 
224
+ # Write
153
225
  with barecat.Barecat('mydata.barecat', readonly=False) as bc:
154
- bc['path/to/file/as/stored.jpg'] = binary_file_data
155
- bc.add_by_path('path/to/file/on/disk.jpg')
156
-
157
- with open('path', 'rb') as f:
158
- bc.add('path/to/file/on/disk.jpg', fileobj=f)
159
-
226
+ bc['path/to/file.jpg'] = binary_file_data
227
+ bc.add_by_path('path/to/file/on/disk.jpg')
228
+
229
+ with open('local_file.bin', 'rb') as f:
230
+ bc.add('path/in/archive.bin', fileobj=f)
231
+
232
+ # Read
160
233
  with barecat.Barecat('mydata.barecat') as bc:
161
- binary_file_data = bc['path/to/file.jpg']
162
- entrynames = bc.listdir('path/to')
163
- for root, dirs, files in bc.walk('path/to/something'):
164
- print(root, dirs, files)
165
-
166
- paths = bc.glob('path/to/**/*.jpg', recursive=True)
167
-
168
- with bc.open('path/to/file.jpg', 'rb') as f:
169
- data = f.read(123)
234
+ binary_file_data = bc['path/to/file.jpg']
235
+ entrynames = bc.listdir('path/to')
236
+
237
+ for root, dirs, files in bc.walk('path/to/something'):
238
+ print(root, dirs, files)
239
+
240
+ paths = bc.glob('path/to/**/*.jpg', recursive=True)
241
+
242
+ with bc.open('path/to/file.jpg', 'rb') as f:
243
+ data = f.read(123)
170
244
  ```
171
245
 
172
- ## Image viewer
246
+ ### Automatic Encoding/Decoding
173
247
 
174
- Barecat comes with a simple image viewer that can be used to browse the contents of a Barecat
175
- archive.
248
+ Wrap the archive with `DecodedView` for automatic serialization based on file extension:
176
249
 
177
- ```bash
178
- barecat-image-viewer mydata.barecat
250
+ ```python
251
+ from barecat import Barecat, DecodedView
252
+ import numpy as np
253
+
254
+ with Barecat('data.barecat', readonly=False) as bc:
255
+ dec = DecodedView(bc)
256
+ dec['config.json'] = {'learning_rate': 0.001}
257
+ dec['weights.npy'] = np.random.randn(100, 100)
258
+
259
+ with Barecat('data.barecat') as bc:
260
+ dec = DecodedView(bc)
261
+ config = dec['config.json'] # Returns dict
262
+ weights = dec['weights.npy'] # Returns numpy array
179
263
  ```
180
264
 
181
-
265
+ Supported formats: `.json`, `.pkl`, `.npy`, `.npz`, `.msgpack`, images (`.png`, `.jpg`, etc.),
266
+ with stackable compression (`.gz`, `.bz2`, `.xz`).
267
+
268
+ ## Documentation
269
+
270
+ Full documentation is available at [barecat.readthedocs.io](https://barecat.readthedocs.io).
271
+
272
+ ## License
273
+
274
+ MIT
@@ -1,9 +1,11 @@
1
1
  # Barecat
2
2
 
3
- **[Full API Reference Docs](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)**
3
+ **[Documentation](https://barecat.readthedocs.io)** |
4
+ **[API Reference](https://istvansarandi.com/docs/barecat/api/barecat/Barecat.html)** |
5
+ **[GitHub](https://github.com/isarandi/barecat)**
4
6
 
5
7
  Barecat (**bare** con**cat**enation) is a highly scalable, simple aggregate storage format for
6
- storing many (tens of millions and more) small files, with focus on fast random access and
8
+ storing many (tens of millions and more) small files, with focus on fast random access and
7
9
  minimal overhead.
8
10
 
9
11
  Barecat can be thought of as a simple filesystem, or as something akin to an indexed tarball, or a
@@ -16,16 +18,28 @@ experience (`listdir`, `walk`, `glob`, etc).
16
18
 
17
19
  Internally, all the data is simply concatenated one after another into one or more data shard files.
18
20
  Additionally, an index is maintained in an SQLite database, which stores the shard number, the offset
19
- and the size of each inner file (as well as a checksum, and further filesystem-like metadata
21
+ and the size of each inner file (as well as a checksum, and further filesystem-like metadata
20
22
  like modification time). Barecat also maintains aggregate statistics for each directory, such as the
21
23
  total number of files and total file size.
22
24
 
23
-
24
25
  ![Architecture](./figure.png)
25
26
 
26
27
  As you can see, the Barecat format is very simple. Readers/writers are easy to write in any language, since
27
28
  SQLite is a widely-supported format.
28
29
 
30
+ ## Installation
31
+
32
+ ```bash
33
+ pip install barecat
34
+ ```
35
+
36
+ Optional extras:
37
+
38
+ ```bash
39
+ pip install barecat[mount] # FUSE mount support
40
+ ```
41
+
42
+ Requires Python 3.9+.
29
43
 
30
44
  ## Background
31
45
 
@@ -52,77 +66,126 @@ statistics (total file size, number of files) for each directory.
52
66
  The index is stored in a separate SQLite database file, which itself does not need to be loaded
53
67
  entirely into memory. Ideal for storing training image data for deep learning jobs.
54
68
  - **Sharding**: To make it easier to move the data around or to distribute it across multiple
55
- storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
69
+ storage devices, the archive can be split into multiple files of equal size (shards, or volumes).
56
70
  The shards do not have to be concatenated to be used, the library will keep all shard files open
57
71
  and load data from the appropriate one during normal operations.
58
72
  - **Fast browsing**: The SQLite database contains an index for the parent directories, allowing
59
73
  fast listing of directory contents and aggregate statistics (total file size, number of files).
60
74
  - **Intuitive API**: Familiar filesystem-like API, as well as a dictionary-like one.
61
- - **Mountable**: The archive can be efficiently mounted in readonly or read-write mode.
75
+ - **Mountable**: The archive can be efficiently mounted in readonly or read-write mode via FUSE.
62
76
  - **Simple storage format**: The files are simply concatenated after each other and the index contains
63
77
  the offsets and sizes of each file. There is no header format to understand. The index can be
64
78
  dumped into any format with simple SQL queries.
65
79
 
66
- ## Command line interface
80
+ ## Command Line Interface
67
81
 
68
- To create a Barecat archive, use the `barecat-create` or `barecat-create-recursive` commands, which
69
- are automatically installed executables with the pip package.
82
+ Barecat provides a unified `barecat` command with subcommands:
70
83
 
71
84
  ```bash
72
- barecat-create --file=mydata.barecat --shard-size=100G < path_of_paths.txt
85
+ # Create archive from files and directories
86
+ barecat create mydata.barecat /path/to/images/
87
+
88
+ # Create from find output
89
+ find /data -name '*.jpg' -print0 | barecat create -0 mydata.barecat -T -
90
+
91
+ # Extract archive
92
+ barecat extract mydata.barecat -C /output/
93
+
94
+ # List contents
95
+ barecat list mydata.barecat
96
+ barecat list -l mydata.barecat # Long format with sizes
97
+
98
+ # Add files to an existing archive
99
+ barecat add mydata.barecat newdir/
100
+
101
+ # Print a file to stdout
102
+ barecat cat mydata.barecat path/to/file.txt
103
+
104
+ # Convert from/to tar or zip
105
+ barecat convert data.tar.gz data.barecat
106
+
107
+ # Interactive shell
108
+ barecat shell mydata.barecat
73
109
 
74
- find dirname -name '*.jpg' -print0 | barecat-create --null --file=mydata.barecat --shard-size=100G
110
+ # Merge multiple archives
111
+ barecat merge -o combined.barecat archive1.barecat archive2.barecat
75
112
 
76
- barecat-create-recursive dir1 dir2 dir3 --file=mydata.barecat --shard-size=100G
113
+ # Verify integrity
114
+ barecat verify mydata.barecat
115
+
116
+ # Defragment after deletions
117
+ barecat defrag mydata.barecat
118
+
119
+ # Mount via FUSE (requires barecat[mount])
120
+ barecat mount mydata.barecat /mnt/mydata
77
121
  ```
78
122
 
79
123
  This may yield the following files:
80
124
 
81
125
  ```
82
- mydata.barecat-shard-00001
83
- mydata.barecat-shard-00002
84
- mydata.barecat-sqlite-index
126
+ mydata.barecat # SQLite index database
127
+ mydata.barecat-shard-00000 # Data shard 0
128
+ mydata.barecat-shard-00001 # Data shard 1
85
129
  ```
86
130
 
87
- The files can be extracted out again. Unix-like permissions, modification times, owner info are
131
+ The files can be extracted back out. Unix-like permissions, modification times, and owner info are
88
132
  preserved.
89
133
 
90
- ```bash
91
- barecat-extract --file=mydata.barecat --target-directory=targetdir/
92
- ```
134
+ See the [documentation](https://barecat.readthedocs.io) for the full CLI reference.
93
135
 
94
136
  ## Python API
95
137
 
96
138
  ```python
97
-
98
139
  import barecat
99
140
 
141
+ # Write
100
142
  with barecat.Barecat('mydata.barecat', readonly=False) as bc:
101
- bc['path/to/file/as/stored.jpg'] = binary_file_data
102
- bc.add_by_path('path/to/file/on/disk.jpg')
103
-
104
- with open('path', 'rb') as f:
105
- bc.add('path/to/file/on/disk.jpg', fileobj=f)
106
-
143
+ bc['path/to/file.jpg'] = binary_file_data
144
+ bc.add_by_path('path/to/file/on/disk.jpg')
145
+
146
+ with open('local_file.bin', 'rb') as f:
147
+ bc.add('path/in/archive.bin', fileobj=f)
148
+
149
+ # Read
107
150
  with barecat.Barecat('mydata.barecat') as bc:
108
- binary_file_data = bc['path/to/file.jpg']
109
- entrynames = bc.listdir('path/to')
110
- for root, dirs, files in bc.walk('path/to/something'):
111
- print(root, dirs, files)
112
-
113
- paths = bc.glob('path/to/**/*.jpg', recursive=True)
114
-
115
- with bc.open('path/to/file.jpg', 'rb') as f:
116
- data = f.read(123)
151
+ binary_file_data = bc['path/to/file.jpg']
152
+ entrynames = bc.listdir('path/to')
153
+
154
+ for root, dirs, files in bc.walk('path/to/something'):
155
+ print(root, dirs, files)
156
+
157
+ paths = bc.glob('path/to/**/*.jpg', recursive=True)
158
+
159
+ with bc.open('path/to/file.jpg', 'rb') as f:
160
+ data = f.read(123)
117
161
  ```
118
162
 
119
- ## Image viewer
163
+ ### Automatic Encoding/Decoding
120
164
 
121
- Barecat comes with a simple image viewer that can be used to browse the contents of a Barecat
122
- archive.
165
+ Wrap the archive with `DecodedView` for automatic serialization based on file extension:
123
166
 
124
- ```bash
125
- barecat-image-viewer mydata.barecat
167
+ ```python
168
+ from barecat import Barecat, DecodedView
169
+ import numpy as np
170
+
171
+ with Barecat('data.barecat', readonly=False) as bc:
172
+ dec = DecodedView(bc)
173
+ dec['config.json'] = {'learning_rate': 0.001}
174
+ dec['weights.npy'] = np.random.randn(100, 100)
175
+
176
+ with Barecat('data.barecat') as bc:
177
+ dec = DecodedView(bc)
178
+ config = dec['config.json'] # Returns dict
179
+ weights = dec['weights.npy'] # Returns numpy array
126
180
  ```
127
181
 
128
-
182
+ Supported formats: `.json`, `.pkl`, `.npy`, `.npz`, `.msgpack`, images (`.png`, `.jpg`, etc.),
183
+ with stackable compression (`.gz`, `.bz2`, `.xz`).
184
+
185
+ ## Documentation
186
+
187
+ Full documentation is available at [barecat.readthedocs.io](https://barecat.readthedocs.io).
188
+
189
+ ## License
190
+
191
+ MIT