itar 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itar-0.4.2/PKG-INFO +67 -0
- itar-0.4.2/README.md +55 -0
- {itar-0.4.0 → itar-0.4.2}/pyproject.toml +6 -8
- {itar-0.4.0 → itar-0.4.2}/src/itar/index.py +14 -11
- {itar-0.4.0 → itar-0.4.2}/src/itar/indexed_tar_file.py +26 -0
- {itar-0.4.0 → itar-0.4.2}/src/itar/utils.py +5 -0
- itar-0.4.0/PKG-INFO +0 -90
- itar-0.4.0/README.md +0 -77
- {itar-0.4.0 → itar-0.4.2}/src/itar/__init__.py +0 -0
- {itar-0.4.0 → itar-0.4.2}/src/itar/cli.py +0 -0
- {itar-0.4.0 → itar-0.4.2}/src/itar/py.typed +0 -0
itar-0.4.2/PKG-INFO
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: itar
|
|
3
|
+
Version: 0.4.2
|
|
4
|
+
Summary: tar file index for constant-time member access
|
|
5
|
+
Author: Karim Abou Zeid
|
|
6
|
+
Author-email: Karim Abou Zeid <contact@ka.codes>
|
|
7
|
+
Requires-Dist: msgpack>=1.1.1
|
|
8
|
+
Requires-Dist: tqdm>=4.67.1
|
|
9
|
+
Requires-Dist: humanize>=4.12.3
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# itar
|
|
14
|
+
|
|
15
|
+
[](https://pypi.python.org/pypi/itar)
|
|
16
|
+
[](https://kabouzeid.github.io/itar/)
|
|
17
|
+
|
|
18
|
+
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
19
|
+
|
|
20
|
+
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
21
|
+
|
|
22
|
+
## Quickstart
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install itar
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Single tarball
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
echo "Hello world!" > hello.txt
|
|
32
|
+
tar cf hello.tar hello.txt # regular tarball
|
|
33
|
+
|
|
34
|
+
itar index create hello.itar # indexes hello.tar
|
|
35
|
+
itar index list hello.itar # list indexed members
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import itar
|
|
40
|
+
|
|
41
|
+
with itar.open("hello.itar") as archive:
|
|
42
|
+
print(archive["hello.txt"].read())
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Sharded tarballs
|
|
46
|
+
|
|
47
|
+
Give each shard a zero-padded suffix before building the index:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
tar cf photos-0.tar wedding/ # shard 0
|
|
51
|
+
tar cf photos-1.tar vacation/ # shard 1
|
|
52
|
+
|
|
53
|
+
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
54
|
+
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import itar
|
|
59
|
+
|
|
60
|
+
with itar.open("photos.itar") as photos:
|
|
61
|
+
assert "wedding/cake.jpg" in photos
|
|
62
|
+
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Docs
|
|
66
|
+
|
|
67
|
+
Full CLI, API, and format details live in the [documentation site](https://kabouzeid.github.io/itar).
|
itar-0.4.2/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# itar
|
|
2
|
+
|
|
3
|
+
[](https://pypi.python.org/pypi/itar)
|
|
4
|
+
[](https://kabouzeid.github.io/itar/)
|
|
5
|
+
|
|
6
|
+
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
7
|
+
|
|
8
|
+
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
9
|
+
|
|
10
|
+
## Quickstart
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install itar
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
### Single tarball
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
echo "Hello world!" > hello.txt
|
|
20
|
+
tar cf hello.tar hello.txt # regular tarball
|
|
21
|
+
|
|
22
|
+
itar index create hello.itar # indexes hello.tar
|
|
23
|
+
itar index list hello.itar # list indexed members
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
import itar
|
|
28
|
+
|
|
29
|
+
with itar.open("hello.itar") as archive:
|
|
30
|
+
print(archive["hello.txt"].read())
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Sharded tarballs
|
|
34
|
+
|
|
35
|
+
Give each shard a zero-padded suffix before building the index:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
tar cf photos-0.tar wedding/ # shard 0
|
|
39
|
+
tar cf photos-1.tar vacation/ # shard 1
|
|
40
|
+
|
|
41
|
+
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
42
|
+
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import itar
|
|
47
|
+
|
|
48
|
+
with itar.open("photos.itar") as photos:
|
|
49
|
+
assert "wedding/cake.jpg" in photos
|
|
50
|
+
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Docs
|
|
54
|
+
|
|
55
|
+
Full CLI, API, and format details live in the [documentation site](https://kabouzeid.github.io/itar).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "itar"
|
|
3
|
-
version = "0.4.
|
|
4
|
-
description = "
|
|
3
|
+
version = "0.4.2"
|
|
4
|
+
description = "tar file index for constant-time member access"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
7
7
|
{ name = "Karim Abou Zeid", email = "contact@ka.codes" }
|
|
@@ -9,23 +9,21 @@ authors = [
|
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
dependencies = [
|
|
11
11
|
"msgpack>=1.1.1",
|
|
12
|
+
"tqdm>=4.67.1",
|
|
13
|
+
"humanize>=4.12.3",
|
|
12
14
|
]
|
|
13
15
|
|
|
14
16
|
[dependency-groups]
|
|
15
17
|
dev = [
|
|
16
18
|
"pytest>=8.4.0",
|
|
17
19
|
"ruff>=0.11.13",
|
|
20
|
+
"zensical>=0.0.10",
|
|
21
|
+
"mkdocstrings-python>=2.0.1",
|
|
18
22
|
]
|
|
19
23
|
|
|
20
24
|
[project.scripts]
|
|
21
25
|
itar = "itar.cli:main"
|
|
22
26
|
|
|
23
|
-
[project.optional-dependencies]
|
|
24
|
-
cli = [
|
|
25
|
-
"tqdm>=4.67.1",
|
|
26
|
-
"humanize>=4.12.3",
|
|
27
|
-
]
|
|
28
|
-
|
|
29
27
|
[build-system]
|
|
30
28
|
requires = ["uv_build>=0.9.0,<0.10.0"]
|
|
31
29
|
build-backend = "uv_build"
|
|
@@ -46,16 +46,19 @@ class IndexLayout:
|
|
|
46
46
|
return candidates
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
class DefaultResolver:
|
|
50
|
+
"""Resolve shard indices to paths next to an existing index file."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, layout: IndexLayout):
|
|
53
|
+
self.layout = layout
|
|
54
|
+
|
|
55
|
+
def __call__(self, shard_idx: int | None) -> Path:
|
|
51
56
|
if shard_idx is None:
|
|
52
|
-
return layout.single_tar()
|
|
57
|
+
return self.layout.single_tar()
|
|
53
58
|
|
|
54
|
-
shard_paths = layout.discover_shards()
|
|
59
|
+
shard_paths = self.layout.discover_shards()
|
|
55
60
|
return shard_paths[shard_idx]
|
|
56
61
|
|
|
57
|
-
return resolve
|
|
58
|
-
|
|
59
62
|
|
|
60
63
|
def _build_index_from_fileobjs(
|
|
61
64
|
file_objs: Iterable[IO[bytes]],
|
|
@@ -132,7 +135,7 @@ def open(
|
|
|
132
135
|
open_fn: Callable[[str | os.PathLike], IO[bytes]] | None = None,
|
|
133
136
|
buffered_file_reader: bool = True,
|
|
134
137
|
) -> IndexedTarFile:
|
|
135
|
-
"""Open an
|
|
138
|
+
"""Open an ``IndexedTarFile`` using an on-disk index file."""
|
|
136
139
|
|
|
137
140
|
path = Path(path)
|
|
138
141
|
index = load(path)
|
|
@@ -141,7 +144,7 @@ def open(
|
|
|
141
144
|
if shards is not None:
|
|
142
145
|
resolved_shards = shards
|
|
143
146
|
else:
|
|
144
|
-
resolved_shards =
|
|
147
|
+
resolved_shards = DefaultResolver(layout)
|
|
145
148
|
return IndexedTarFile(
|
|
146
149
|
resolved_shards,
|
|
147
150
|
index=index,
|
|
@@ -164,10 +167,10 @@ def create(
|
|
|
164
167
|
|
|
165
168
|
|
|
166
169
|
__all__ = [
|
|
167
|
-
"IndexLayout",
|
|
168
|
-
"build",
|
|
169
170
|
"create",
|
|
170
|
-
"load",
|
|
171
171
|
"open",
|
|
172
|
+
"build",
|
|
172
173
|
"dump",
|
|
174
|
+
"load",
|
|
175
|
+
"IndexLayout",
|
|
173
176
|
]
|
|
@@ -20,6 +20,29 @@ ShardResolver = Callable[[int | None], Shard]
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
class IndexedTarFile(Mapping):
|
|
23
|
+
"""
|
|
24
|
+
Read-only mapping that serves members out of indexed tar shards.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
shards: Shard sources (single tar, list of tars, or resolver callable).
|
|
28
|
+
index: Precomputed index mapping member names to offsets.
|
|
29
|
+
open_fn: Optional callable to open paths; defaults to a thread-safe file reader.
|
|
30
|
+
buffered_file_reader: Wrap member streams in a buffered reader when True.
|
|
31
|
+
|
|
32
|
+
Use ``itar.open`` to construct this class. It supports the mapping protocol
|
|
33
|
+
for read access (``archive["path/to/file"]``) and should be used as a
|
|
34
|
+
context manager to close any open file handles.
|
|
35
|
+
|
|
36
|
+
Example:
|
|
37
|
+
```python
|
|
38
|
+
import itar
|
|
39
|
+
|
|
40
|
+
with itar.open("photos.itar") as archive:
|
|
41
|
+
jpg = archive["vacation/sunrise.jpg"].read()
|
|
42
|
+
assert "wedding/cake.jpg" in archive
|
|
43
|
+
```
|
|
44
|
+
"""
|
|
45
|
+
|
|
23
46
|
def __init__(
|
|
24
47
|
self,
|
|
25
48
|
shards: list[Shard] | Shard | ShardResolver,
|
|
@@ -69,6 +92,7 @@ class IndexedTarFile(Mapping):
|
|
|
69
92
|
return source
|
|
70
93
|
|
|
71
94
|
def file(self, name: str) -> IO[bytes]:
|
|
95
|
+
"""Return a readable file-like object for an indexed member."""
|
|
72
96
|
shard_idx, member = self._index[name]
|
|
73
97
|
_, offset_data, size = member
|
|
74
98
|
if isinstance(size, str):
|
|
@@ -78,11 +102,13 @@ class IndexedTarFile(Mapping):
|
|
|
78
102
|
)
|
|
79
103
|
|
|
80
104
|
def info(self, name: str) -> TarInfo:
|
|
105
|
+
"""Return the ``TarInfo`` for an indexed member without reading data."""
|
|
81
106
|
shard_idx, member = self._index[name]
|
|
82
107
|
offset, _, _ = member
|
|
83
108
|
return tar_file_info(offset, self._ensure_shard(shard_idx))
|
|
84
109
|
|
|
85
110
|
def check_tar_index(self, names: list[str] | None = None):
|
|
111
|
+
"""Validate stored offsets for the given names (or all members)."""
|
|
86
112
|
for name in names if names is not None else self:
|
|
87
113
|
shard_idx, member = self._index[name]
|
|
88
114
|
check_tar_index(name, member, self._ensure_shard(shard_idx))
|
|
@@ -11,6 +11,7 @@ MemberRecord = tuple[int, int, int | str] # (offset, offset_data, size | linkna
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def tar_file_info(offset: int, file_obj: IO[bytes]) -> TarInfo:
|
|
14
|
+
"""Return a ``TarInfo`` for the member starting at ``offset``."""
|
|
14
15
|
file_obj.seek(offset)
|
|
15
16
|
return TarInfo.fromtarfile(
|
|
16
17
|
# want to avoid creating a new TarFile instance (potentially slow)
|
|
@@ -43,6 +44,7 @@ def tarinfo2member(tarinfo: TarInfo) -> MemberRecord:
|
|
|
43
44
|
def build_tar_index(
|
|
44
45
|
tar: str | os.PathLike | IO[bytes] | TarFile,
|
|
45
46
|
) -> dict[str, MemberRecord]:
|
|
47
|
+
"""Collect offsets and sizes for all files and links in a tar archive."""
|
|
46
48
|
if isinstance(tar, str | os.PathLike):
|
|
47
49
|
tar = tarfile.open(tar, "r:")
|
|
48
50
|
elif isinstance(tar, TarFile):
|
|
@@ -77,6 +79,7 @@ def check_tar_index(
|
|
|
77
79
|
tar_offset: MemberRecord,
|
|
78
80
|
file_obj: IO[bytes],
|
|
79
81
|
):
|
|
82
|
+
"""Confirm that ``tar_offset`` matches the member on disk."""
|
|
80
83
|
offset, offset_data, size = tar_offset
|
|
81
84
|
info = tar_file_info(offset, file_obj)
|
|
82
85
|
if (
|
|
@@ -165,6 +168,8 @@ class ThreadSafeFileIO(io.RawIOBase):
|
|
|
165
168
|
|
|
166
169
|
|
|
167
170
|
class TarFileSectionIO(io.RawIOBase):
|
|
171
|
+
"""A read-only view over a byte range inside a larger file object."""
|
|
172
|
+
|
|
168
173
|
def __init__(self, fileobj: BinaryIO, offset: int, size: int):
|
|
169
174
|
self._fileobj = fileobj
|
|
170
175
|
self._start = offset
|
itar-0.4.0/PKG-INFO
DELETED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.3
|
|
2
|
-
Name: itar
|
|
3
|
-
Version: 0.4.0
|
|
4
|
-
Summary: Add your description here
|
|
5
|
-
Author: Karim Abou Zeid
|
|
6
|
-
Author-email: Karim Abou Zeid <contact@ka.codes>
|
|
7
|
-
Requires-Dist: msgpack>=1.1.1
|
|
8
|
-
Requires-Dist: tqdm>=4.67.1 ; extra == 'cli'
|
|
9
|
-
Requires-Dist: humanize>=4.12.3 ; extra == 'cli'
|
|
10
|
-
Requires-Python: >=3.10
|
|
11
|
-
Provides-Extra: cli
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
|
|
14
|
-
# itar
|
|
15
|
-
|
|
16
|
-
[](https://pypi.python.org/pypi/itar)
|
|
17
|
-
|
|
18
|
-
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
19
|
-
|
|
20
|
-
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
21
|
-
|
|
22
|
-
## Quickstart (single tarball)
|
|
23
|
-
|
|
24
|
-
```bash
|
|
25
|
-
echo "Hello world!" > hello.txt
|
|
26
|
-
tar cf hello.tar hello.txt # regular tarball
|
|
27
|
-
|
|
28
|
-
itar index create hello.itar # indexes hello.tar
|
|
29
|
-
itar index list hello.itar # list indexed members
|
|
30
|
-
```
|
|
31
|
-
|
|
32
|
-
```python
|
|
33
|
-
import itar
|
|
34
|
-
|
|
35
|
-
with itar.open("hello.itar") as archive:
|
|
36
|
-
print(archive["hello.txt"].read())
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
## Quickstart (sharded tarballs)
|
|
40
|
-
|
|
41
|
-
Give each shard a zero-padded suffix before building the index:
|
|
42
|
-
|
|
43
|
-
```bash
|
|
44
|
-
tar cf photos-0.tar wedding/ # shard 0
|
|
45
|
-
tar cf photos-1.tar vacation/ # shard 1
|
|
46
|
-
|
|
47
|
-
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
48
|
-
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
```python
|
|
52
|
-
import itar
|
|
53
|
-
|
|
54
|
-
with itar.open("photos.itar") as photos:
|
|
55
|
-
assert "wedding/cake.jpg" in photos
|
|
56
|
-
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
## CLI reference
|
|
60
|
-
|
|
61
|
-
| Command | Purpose |
|
|
62
|
-
| --- | --- |
|
|
63
|
-
| `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
|
|
64
|
-
| `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
|
|
65
|
-
| `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
|
|
66
|
-
| `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
|
|
67
|
-
|
|
68
|
-
## Python helpers
|
|
69
|
-
|
|
70
|
-
- `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
|
|
71
|
-
- `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
|
|
72
|
-
- `itar.index.dump(index, path)`: serialize an index you built elsewhere.
|
|
73
|
-
- `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
|
|
74
|
-
- `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
|
|
75
|
-
|
|
76
|
-
## `itar` File Format
|
|
77
|
-
An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
|
|
78
|
-
```python
|
|
79
|
-
{
|
|
80
|
-
"path/to/member1.jpg": [ # file name
|
|
81
|
-
null, # either null or shard index (0-based)
|
|
82
|
-
[
|
|
83
|
-
2048, # metadata byte offset
|
|
84
|
-
2560, # data byte offset
|
|
85
|
-
1048576, # file length in bytes
|
|
86
|
-
],
|
|
87
|
-
],
|
|
88
|
-
...
|
|
89
|
-
}
|
|
90
|
-
```
|
itar-0.4.0/README.md
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
# itar
|
|
2
|
-
|
|
3
|
-
[](https://pypi.python.org/pypi/itar)
|
|
4
|
-
|
|
5
|
-
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
6
|
-
|
|
7
|
-
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
8
|
-
|
|
9
|
-
## Quickstart (single tarball)
|
|
10
|
-
|
|
11
|
-
```bash
|
|
12
|
-
echo "Hello world!" > hello.txt
|
|
13
|
-
tar cf hello.tar hello.txt # regular tarball
|
|
14
|
-
|
|
15
|
-
itar index create hello.itar # indexes hello.tar
|
|
16
|
-
itar index list hello.itar # list indexed members
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
```python
|
|
20
|
-
import itar
|
|
21
|
-
|
|
22
|
-
with itar.open("hello.itar") as archive:
|
|
23
|
-
print(archive["hello.txt"].read())
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## Quickstart (sharded tarballs)
|
|
27
|
-
|
|
28
|
-
Give each shard a zero-padded suffix before building the index:
|
|
29
|
-
|
|
30
|
-
```bash
|
|
31
|
-
tar cf photos-0.tar wedding/ # shard 0
|
|
32
|
-
tar cf photos-1.tar vacation/ # shard 1
|
|
33
|
-
|
|
34
|
-
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
35
|
-
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
```python
|
|
39
|
-
import itar
|
|
40
|
-
|
|
41
|
-
with itar.open("photos.itar") as photos:
|
|
42
|
-
assert "wedding/cake.jpg" in photos
|
|
43
|
-
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
## CLI reference
|
|
47
|
-
|
|
48
|
-
| Command | Purpose |
|
|
49
|
-
| --- | --- |
|
|
50
|
-
| `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
|
|
51
|
-
| `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
|
|
52
|
-
| `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
|
|
53
|
-
| `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
|
|
54
|
-
|
|
55
|
-
## Python helpers
|
|
56
|
-
|
|
57
|
-
- `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
|
|
58
|
-
- `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
|
|
59
|
-
- `itar.index.dump(index, path)`: serialize an index you built elsewhere.
|
|
60
|
-
- `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
|
|
61
|
-
- `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
|
|
62
|
-
|
|
63
|
-
## `itar` File Format
|
|
64
|
-
An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
|
|
65
|
-
```python
|
|
66
|
-
{
|
|
67
|
-
"path/to/member1.jpg": [ # file name
|
|
68
|
-
null, # either null or shard index (0-based)
|
|
69
|
-
[
|
|
70
|
-
2048, # metadata byte offset
|
|
71
|
-
2560, # data byte offset
|
|
72
|
-
1048576, # file length in bytes
|
|
73
|
-
],
|
|
74
|
-
],
|
|
75
|
-
...
|
|
76
|
-
}
|
|
77
|
-
```
|
|
File without changes
|
|
File without changes
|
|
File without changes
|