itar 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {itar-0.2.0 → itar-0.4.0}/PKG-INFO +30 -17
- itar-0.4.0/README.md +77 -0
- {itar-0.2.0 → itar-0.4.0}/pyproject.toml +1 -1
- {itar-0.2.0 → itar-0.4.0}/src/itar/cli.py +91 -43
- {itar-0.2.0 → itar-0.4.0}/src/itar/index.py +40 -34
- {itar-0.2.0 → itar-0.4.0}/src/itar/indexed_tar_file.py +46 -29
- {itar-0.2.0 → itar-0.4.0}/src/itar/utils.py +2 -3
- itar-0.2.0/README.md +0 -64
- {itar-0.2.0 → itar-0.4.0}/src/itar/__init__.py +0 -0
- {itar-0.2.0 → itar-0.4.0}/src/itar/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: itar
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Add your description here
|
|
5
5
|
Author: Karim Abou Zeid
|
|
6
6
|
Author-email: Karim Abou Zeid <contact@ka.codes>
|
|
@@ -13,14 +13,19 @@ Description-Content-Type: text/markdown
|
|
|
13
13
|
|
|
14
14
|
# itar
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
[](https://pypi.python.org/pypi/itar)
|
|
17
|
+
|
|
18
|
+
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
19
|
+
|
|
20
|
+
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
17
21
|
|
|
18
22
|
## Quickstart (single tarball)
|
|
19
23
|
|
|
20
24
|
```bash
|
|
21
25
|
echo "Hello world!" > hello.txt
|
|
22
26
|
tar cf hello.tar hello.txt # regular tarball
|
|
23
|
-
|
|
27
|
+
|
|
28
|
+
itar index create hello.itar # indexes hello.tar
|
|
24
29
|
itar index list hello.itar # list indexed members
|
|
25
30
|
```
|
|
26
31
|
|
|
@@ -29,11 +34,6 @@ import itar
|
|
|
29
34
|
|
|
30
35
|
with itar.open("hello.itar") as archive:
|
|
31
36
|
print(archive["hello.txt"].read())
|
|
32
|
-
|
|
33
|
-
# If you just need the index dictionary without opening handles:
|
|
34
|
-
index = itar.index.build("hello.tar")
|
|
35
|
-
# Write the index file to disk:
|
|
36
|
-
itar.index.save("hello.itar", num_shards=None, index=index)
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
## Quickstart (sharded tarballs)
|
|
@@ -43,7 +43,8 @@ Give each shard a zero-padded suffix before building the index:
|
|
|
43
43
|
```bash
|
|
44
44
|
tar cf photos-0.tar wedding/ # shard 0
|
|
45
45
|
tar cf photos-1.tar vacation/ # shard 1
|
|
46
|
-
|
|
46
|
+
|
|
47
|
+
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
47
48
|
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
48
49
|
```
|
|
49
50
|
|
|
@@ -53,25 +54,37 @@ import itar
|
|
|
53
54
|
with itar.open("photos.itar") as photos:
|
|
54
55
|
assert "wedding/cake.jpg" in photos
|
|
55
56
|
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
56
|
-
|
|
57
|
-
index = itar.index.build(["photos-0.tar", "photos-1.tar"])
|
|
58
|
-
itar.index.create("photos.itar", ["photos-0.tar", "photos-1.tar"])
|
|
59
|
-
|
|
60
|
-
num_shards, stored_index = itar.index.load("photos.itar")
|
|
61
57
|
```
|
|
62
58
|
|
|
63
59
|
## CLI reference
|
|
64
60
|
|
|
65
61
|
| Command | Purpose |
|
|
66
62
|
| --- | --- |
|
|
67
|
-
| `itar index
|
|
63
|
+
| `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
|
|
68
64
|
| `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
|
|
69
65
|
| `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
|
|
66
|
+
| `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
|
|
70
67
|
|
|
71
68
|
## Python helpers
|
|
72
69
|
|
|
73
70
|
- `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
|
|
74
71
|
- `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
|
|
75
|
-
- `itar.index.
|
|
76
|
-
- `itar.index.load(path) ->
|
|
72
|
+
- `itar.index.dump(index, path)`: serialize an index you built elsewhere.
|
|
73
|
+
- `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
|
|
77
74
|
- `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
|
|
75
|
+
|
|
76
|
+
## `itar` File Format
|
|
77
|
+
An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
|
|
78
|
+
```python
|
|
79
|
+
{
|
|
80
|
+
"path/to/member1.jpg": [ # file name
|
|
81
|
+
null, # either null or shard index (0-based)
|
|
82
|
+
[
|
|
83
|
+
2048, # metadata byte offset
|
|
84
|
+
2560, # data byte offset
|
|
85
|
+
1048576, # file length in bytes
|
|
86
|
+
],
|
|
87
|
+
],
|
|
88
|
+
...
|
|
89
|
+
}
|
|
90
|
+
```
|
itar-0.4.0/README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# itar
|
|
2
|
+
|
|
3
|
+
[](https://pypi.python.org/pypi/itar)
|
|
4
|
+
|
|
5
|
+
`itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
|
|
6
|
+
|
|
7
|
+
Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
|
|
8
|
+
|
|
9
|
+
## Quickstart (single tarball)
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
echo "Hello world!" > hello.txt
|
|
13
|
+
tar cf hello.tar hello.txt # regular tarball
|
|
14
|
+
|
|
15
|
+
itar index create hello.itar # indexes hello.tar
|
|
16
|
+
itar index list hello.itar # list indexed members
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
import itar
|
|
21
|
+
|
|
22
|
+
with itar.open("hello.itar") as archive:
|
|
23
|
+
print(archive["hello.txt"].read())
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Quickstart (sharded tarballs)
|
|
27
|
+
|
|
28
|
+
Give each shard a zero-padded suffix before building the index:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
tar cf photos-0.tar wedding/ # shard 0
|
|
32
|
+
tar cf photos-1.tar vacation/ # shard 1
|
|
33
|
+
|
|
34
|
+
itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
35
|
+
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import itar
|
|
40
|
+
|
|
41
|
+
with itar.open("photos.itar") as photos:
|
|
42
|
+
assert "wedding/cake.jpg" in photos
|
|
43
|
+
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## CLI reference
|
|
47
|
+
|
|
48
|
+
| Command | Purpose |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
|
|
51
|
+
| `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
|
|
52
|
+
| `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
|
|
53
|
+
| `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
|
|
54
|
+
|
|
55
|
+
## Python helpers
|
|
56
|
+
|
|
57
|
+
- `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
|
|
58
|
+
- `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
|
|
59
|
+
- `itar.index.dump(index, path)`: serialize an index you built elsewhere.
|
|
60
|
+
- `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
|
|
61
|
+
- `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
|
|
62
|
+
|
|
63
|
+
## `itar` File Format
|
|
64
|
+
An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
|
|
65
|
+
```python
|
|
66
|
+
{
|
|
67
|
+
"path/to/member1.jpg": [ # file name
|
|
68
|
+
null, # either null or shard index (0-based)
|
|
69
|
+
[
|
|
70
|
+
2048, # metadata byte offset
|
|
71
|
+
2560, # data byte offset
|
|
72
|
+
1048576, # file length in bytes
|
|
73
|
+
],
|
|
74
|
+
],
|
|
75
|
+
...
|
|
76
|
+
}
|
|
77
|
+
```
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import argparse
|
|
2
|
-
import re
|
|
3
2
|
import sys
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
|
|
@@ -12,15 +11,36 @@ def main() -> None:
|
|
|
12
11
|
subparsers = parser.add_subparsers(dest="command")
|
|
13
12
|
subparsers.required = True
|
|
14
13
|
|
|
14
|
+
cat_parser = subparsers.add_parser(
|
|
15
|
+
"cat", help="Stream a member from an index to stdout"
|
|
16
|
+
)
|
|
17
|
+
cat_parser.add_argument("index", type=Path, help="Path to the .itar index file")
|
|
18
|
+
cat_parser.add_argument("member", help="Member path to stream")
|
|
19
|
+
cat_parser.set_defaults(func=_cmd_cat)
|
|
20
|
+
|
|
21
|
+
ls_parser = subparsers.add_parser(
|
|
22
|
+
"ls", aliases=["list"], help="List members recorded in an index"
|
|
23
|
+
)
|
|
24
|
+
ls_parser.add_argument("index", type=Path, help="Path to the .itar index file")
|
|
25
|
+
ls_parser.add_argument(
|
|
26
|
+
"-l", "--long", action="store_true", help="Show shard and offset details"
|
|
27
|
+
)
|
|
28
|
+
ls_parser.add_argument(
|
|
29
|
+
"--bytes",
|
|
30
|
+
action="store_true",
|
|
31
|
+
help="Render sizes as number of bytes",
|
|
32
|
+
)
|
|
33
|
+
ls_parser.set_defaults(func=_cmd_ls)
|
|
34
|
+
|
|
15
35
|
index_parser = subparsers.add_parser("index", help="Manage .itar index files")
|
|
16
36
|
index_subparsers = index_parser.add_subparsers(dest="index_command")
|
|
17
37
|
index_subparsers.required = True
|
|
18
38
|
|
|
19
|
-
|
|
20
|
-
"
|
|
39
|
+
create_parser = index_subparsers.add_parser(
|
|
40
|
+
"create", help="Create an index file from one or more TAR shards"
|
|
21
41
|
)
|
|
22
|
-
|
|
23
|
-
|
|
42
|
+
create_parser.add_argument("index", type=Path, help="Destination .itar index file")
|
|
43
|
+
create_parser.add_argument(
|
|
24
44
|
"--shards",
|
|
25
45
|
type=Path,
|
|
26
46
|
nargs="+",
|
|
@@ -29,20 +49,20 @@ def main() -> None:
|
|
|
29
49
|
"Explicit shard files to index. Use when you want to bypass auto-discovery."
|
|
30
50
|
),
|
|
31
51
|
)
|
|
32
|
-
|
|
52
|
+
create_parser.add_argument(
|
|
33
53
|
"--single",
|
|
34
54
|
type=Path,
|
|
35
55
|
dest="single_tar",
|
|
36
56
|
metavar="TAR",
|
|
37
57
|
help="Path to a single TAR archive. Ignored if --shards is provided.",
|
|
38
58
|
)
|
|
39
|
-
|
|
59
|
+
create_parser.add_argument(
|
|
40
60
|
"--no-progress",
|
|
41
61
|
dest="progress",
|
|
42
62
|
action="store_false",
|
|
43
63
|
help="Disable the indexing progress bar",
|
|
44
64
|
)
|
|
45
|
-
|
|
65
|
+
create_parser.set_defaults(progress=True, func=_cmd_index_create)
|
|
46
66
|
|
|
47
67
|
check_parser = index_subparsers.add_parser(
|
|
48
68
|
"check", help="Verify every member recorded in an index file"
|
|
@@ -64,10 +84,9 @@ def main() -> None:
|
|
|
64
84
|
"-l", "--long", action="store_true", help="Show shard and offset details"
|
|
65
85
|
)
|
|
66
86
|
list_parser.add_argument(
|
|
67
|
-
"
|
|
68
|
-
"--human-readable",
|
|
87
|
+
"--bytes",
|
|
69
88
|
action="store_true",
|
|
70
|
-
help="Render sizes
|
|
89
|
+
help="Render sizes as number of bytes",
|
|
71
90
|
)
|
|
72
91
|
list_parser.set_defaults(func=_cmd_index_list)
|
|
73
92
|
|
|
@@ -83,7 +102,7 @@ class CLIError(RuntimeError):
|
|
|
83
102
|
"""Raised for recoverable CLI errors."""
|
|
84
103
|
|
|
85
104
|
|
|
86
|
-
def
|
|
105
|
+
def _resolve_shards_for_create(
|
|
87
106
|
index_path: Path,
|
|
88
107
|
explicit_shards: list[Path] | None,
|
|
89
108
|
single_tar: Path | None,
|
|
@@ -96,17 +115,10 @@ def _resolve_shards_for_build(
|
|
|
96
115
|
single_tar = Path(single_tar)
|
|
97
116
|
if not single_tar.is_file():
|
|
98
117
|
raise CLIError(f"Single tar not found: {single_tar}")
|
|
99
|
-
return single_tar,
|
|
100
|
-
|
|
101
|
-
stem = index_path.stem
|
|
102
|
-
pattern = re.compile(rf"^{re.escape(stem)}-\d+\.tar$")
|
|
103
|
-
indexed_shards = [
|
|
104
|
-
candidate
|
|
105
|
-
for candidate in index_path.parent.glob(f"{index_path.stem}-*.tar")
|
|
106
|
-
if candidate.is_file() and pattern.match(candidate.name)
|
|
107
|
-
]
|
|
108
|
-
indexed_shards.sort()
|
|
118
|
+
return single_tar, 1
|
|
119
|
+
|
|
109
120
|
layout = index.IndexLayout(index_path)
|
|
121
|
+
indexed_shards = layout.discover_shards()
|
|
110
122
|
single_tar_candidate = layout.single_tar()
|
|
111
123
|
has_single = single_tar_candidate.is_file()
|
|
112
124
|
has_indexed = len(indexed_shards) > 0
|
|
@@ -119,11 +131,11 @@ def _resolve_shards_for_build(
|
|
|
119
131
|
|
|
120
132
|
if has_single:
|
|
121
133
|
shards = single_tar_candidate
|
|
122
|
-
|
|
134
|
+
shard_count = 1
|
|
123
135
|
else:
|
|
124
136
|
shards = indexed_shards
|
|
125
|
-
|
|
126
|
-
if
|
|
137
|
+
shard_count = len(shards)
|
|
138
|
+
if shard_count < 1:
|
|
127
139
|
raise CLIError(
|
|
128
140
|
f"No shards found for {index_path}.\n"
|
|
129
141
|
f"Please create shard files first. Expected pattern: "
|
|
@@ -131,23 +143,42 @@ def _resolve_shards_for_build(
|
|
|
131
143
|
f"or '{index_path.stem}.tar' for a single shard."
|
|
132
144
|
)
|
|
133
145
|
|
|
134
|
-
expected = layout.shards(
|
|
146
|
+
expected = layout.shards(shard_count)
|
|
135
147
|
assert shards == expected, (
|
|
136
148
|
f"Shards do not match expected names: {shards} != {expected}"
|
|
137
149
|
)
|
|
138
150
|
|
|
139
|
-
return shards,
|
|
151
|
+
return shards, shard_count
|
|
152
|
+
|
|
140
153
|
|
|
154
|
+
def _cmd_cat(args) -> None:
|
|
155
|
+
try:
|
|
156
|
+
with index.open(args.index) as archive:
|
|
157
|
+
try:
|
|
158
|
+
source = archive.file(args.member)
|
|
159
|
+
except KeyError as exc: # pragma: no cover - defensive
|
|
160
|
+
raise CLIError(f"Member not found: {args.member}") from exc
|
|
141
161
|
|
|
142
|
-
|
|
162
|
+
try:
|
|
163
|
+
while True:
|
|
164
|
+
chunk = source.read(64 * 1024)
|
|
165
|
+
if not chunk:
|
|
166
|
+
break
|
|
167
|
+
sys.stdout.buffer.write(chunk)
|
|
168
|
+
sys.stdout.buffer.flush()
|
|
169
|
+
finally:
|
|
170
|
+
source.close()
|
|
171
|
+
except FileNotFoundError as exc: # pragma: no cover - defensive
|
|
172
|
+
raise CLIError(str(exc)) from exc
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _cmd_index_create(args) -> None:
|
|
143
176
|
index_path = Path(args.index)
|
|
144
|
-
shards,
|
|
177
|
+
shards, shard_count = _resolve_shards_for_create(
|
|
145
178
|
index_path, args.shards, args.single_tar
|
|
146
179
|
)
|
|
147
180
|
index.create(index_path, shards, progress_bar=args.progress)
|
|
148
|
-
print(
|
|
149
|
-
f"Wrote index to {index_path} with {1 if num_shards is None else num_shards} shard(s)."
|
|
150
|
-
)
|
|
181
|
+
print(f"Wrote index to {index_path} with {shard_count} shard(s).")
|
|
151
182
|
|
|
152
183
|
|
|
153
184
|
def _cmd_index_check(args) -> None:
|
|
@@ -169,23 +200,40 @@ def _cmd_index_check(args) -> None:
|
|
|
169
200
|
sys.exit(1)
|
|
170
201
|
|
|
171
202
|
|
|
203
|
+
def _cmd_ls(args) -> None:
|
|
204
|
+
current_index = index.load(args.index)
|
|
205
|
+
if args.long:
|
|
206
|
+
max_size = 0
|
|
207
|
+
lines = []
|
|
208
|
+
for member, (shard_idx, (offset, offset_data, size)) in current_index.items():
|
|
209
|
+
if not args.bytes:
|
|
210
|
+
from humanize import naturalsize
|
|
211
|
+
|
|
212
|
+
size = naturalsize(size, gnu=True)
|
|
213
|
+
else:
|
|
214
|
+
size = str(size)
|
|
215
|
+
max_size = max(max_size, len(size))
|
|
216
|
+
lines.append((size, member))
|
|
217
|
+
for line in [f"{size:>{max_size}} {member}" for size, member in lines]:
|
|
218
|
+
print(line)
|
|
219
|
+
else:
|
|
220
|
+
for member in current_index:
|
|
221
|
+
print(member)
|
|
222
|
+
|
|
223
|
+
|
|
172
224
|
def _cmd_index_list(args) -> None:
|
|
173
|
-
|
|
225
|
+
current_index = index.load(args.index)
|
|
174
226
|
if args.long:
|
|
175
227
|
for member, (shard_idx, (offset, offset_data, size)) in current_index.items():
|
|
176
|
-
if args.
|
|
228
|
+
if not args.bytes:
|
|
177
229
|
from humanize import naturalsize
|
|
178
230
|
|
|
179
231
|
size = naturalsize(size, gnu=True)
|
|
232
|
+
shard_repr = "-" if shard_idx is None else str(shard_idx)
|
|
180
233
|
print(
|
|
181
|
-
f"{member:<40} {
|
|
234
|
+
f"{member:<40} {shard_repr:>5} {offset:>12} {offset_data:>12} {size:>10}"
|
|
182
235
|
)
|
|
183
236
|
print(f"{'NAME':<40} {'SHARD':>5} {'OFFSET':>12} {'OFF_DATA':>12} {'SIZE':>10}")
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
print(member)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if __name__ == "__main__":
|
|
191
|
-
main()
|
|
237
|
+
else:
|
|
238
|
+
for member in current_index:
|
|
239
|
+
print(member)
|
|
@@ -5,12 +5,7 @@ from collections.abc import Iterable
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import IO, Callable
|
|
7
7
|
|
|
8
|
-
from .indexed_tar_file import
|
|
9
|
-
IndexedTarFile,
|
|
10
|
-
IndexedTarIndex,
|
|
11
|
-
Shard,
|
|
12
|
-
_normalize_shards,
|
|
13
|
-
)
|
|
8
|
+
from .indexed_tar_file import IndexedTarFile, IndexedTarIndex, Shard, ShardResolver
|
|
14
9
|
from .utils import build_tar_index
|
|
15
10
|
|
|
16
11
|
|
|
@@ -51,10 +46,22 @@ class IndexLayout:
|
|
|
51
46
|
return candidates
|
|
52
47
|
|
|
53
48
|
|
|
49
|
+
def _make_default_resolver(layout: IndexLayout) -> ShardResolver:
|
|
50
|
+
def resolve(shard_idx: int | None) -> Path:
|
|
51
|
+
if shard_idx is None:
|
|
52
|
+
return layout.single_tar()
|
|
53
|
+
|
|
54
|
+
shard_paths = layout.discover_shards()
|
|
55
|
+
return shard_paths[shard_idx]
|
|
56
|
+
|
|
57
|
+
return resolve
|
|
58
|
+
|
|
59
|
+
|
|
54
60
|
def _build_index_from_fileobjs(
|
|
55
61
|
file_objs: Iterable[IO[bytes]],
|
|
56
62
|
*,
|
|
57
63
|
progress_bar: bool,
|
|
64
|
+
use_shard_indices: bool,
|
|
58
65
|
) -> IndexedTarIndex:
|
|
59
66
|
iterator = file_objs
|
|
60
67
|
if progress_bar:
|
|
@@ -63,7 +70,7 @@ def _build_index_from_fileobjs(
|
|
|
63
70
|
iterator = tqdm(file_objs, desc="Building index", unit="shard")
|
|
64
71
|
|
|
65
72
|
return {
|
|
66
|
-
name: (i, member)
|
|
73
|
+
name: ((i if use_shard_indices else None), member)
|
|
67
74
|
for i, file_obj in enumerate(iterator)
|
|
68
75
|
for name, member in build_tar_index(file_obj).items()
|
|
69
76
|
}
|
|
@@ -75,50 +82,48 @@ def build(
|
|
|
75
82
|
progress_bar: bool = False,
|
|
76
83
|
) -> IndexedTarIndex:
|
|
77
84
|
"""Build an index mapping without instantiating ``IndexedTarFile``."""
|
|
85
|
+
is_sharded = isinstance(shards, list)
|
|
86
|
+
if not is_sharded:
|
|
87
|
+
shards = [shards]
|
|
88
|
+
needs_open = [isinstance(s, (str, os.PathLike)) for s in shards]
|
|
78
89
|
|
|
79
|
-
_, shard_list, needs_open = _normalize_shards(shards)
|
|
80
90
|
file_objs: list[IO[bytes]] = [
|
|
81
91
|
builtins.open(tar, "rb") if needs else tar
|
|
82
|
-
for tar, needs in zip(
|
|
92
|
+
for tar, needs in zip(shards, needs_open, strict=True)
|
|
83
93
|
]
|
|
84
94
|
try:
|
|
85
|
-
return _build_index_from_fileobjs(
|
|
95
|
+
return _build_index_from_fileobjs(
|
|
96
|
+
file_objs,
|
|
97
|
+
progress_bar=progress_bar,
|
|
98
|
+
use_shard_indices=is_sharded,
|
|
99
|
+
)
|
|
86
100
|
finally:
|
|
87
101
|
for needs, file_obj in zip(needs_open, file_objs, strict=True):
|
|
88
102
|
if needs:
|
|
89
103
|
file_obj.close()
|
|
90
104
|
|
|
91
105
|
|
|
92
|
-
def load(path: str | os.PathLike) ->
|
|
93
|
-
"""Load an index dictionary
|
|
106
|
+
def load(path: str | os.PathLike) -> IndexedTarIndex:
|
|
107
|
+
"""Load an index dictionary from a saved ``.itar`` index file."""
|
|
94
108
|
|
|
95
109
|
import msgpack
|
|
96
110
|
|
|
97
111
|
path = Path(path)
|
|
98
112
|
with builtins.open(path, "rb") as f:
|
|
99
|
-
|
|
100
|
-
return num_shards, index
|
|
113
|
+
return msgpack.load(f)
|
|
101
114
|
|
|
102
115
|
|
|
103
|
-
def
|
|
104
|
-
path: str | os.PathLike,
|
|
105
|
-
num_shards: int | None,
|
|
116
|
+
def dump(
|
|
106
117
|
index: IndexedTarIndex,
|
|
118
|
+
path: str | os.PathLike,
|
|
107
119
|
) -> None:
|
|
108
|
-
"""Persist ``
|
|
120
|
+
"""Persist ``index`` to disk in msgpack format."""
|
|
109
121
|
|
|
110
122
|
import msgpack
|
|
111
123
|
|
|
112
124
|
path = Path(path)
|
|
113
125
|
with builtins.open(path, "wb") as f:
|
|
114
|
-
msgpack.dump(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
def _infer_default_shards(path: Path, num_shards: int | None) -> list[Shard] | Shard:
|
|
118
|
-
layout = IndexLayout(path)
|
|
119
|
-
if num_shards is None:
|
|
120
|
-
return layout.single_tar()
|
|
121
|
-
return layout.shards(num_shards)
|
|
126
|
+
msgpack.dump(index, f)
|
|
122
127
|
|
|
123
128
|
|
|
124
129
|
def open(
|
|
@@ -130,10 +135,13 @@ def open(
|
|
|
130
135
|
"""Open an :class:`IndexedTarFile` using an on-disk index file."""
|
|
131
136
|
|
|
132
137
|
path = Path(path)
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
138
|
+
index = load(path)
|
|
139
|
+
layout = IndexLayout(path)
|
|
140
|
+
resolved_shards: list[Shard] | Shard | ShardResolver
|
|
141
|
+
if shards is not None:
|
|
142
|
+
resolved_shards = shards
|
|
143
|
+
else:
|
|
144
|
+
resolved_shards = _make_default_resolver(layout)
|
|
137
145
|
return IndexedTarFile(
|
|
138
146
|
resolved_shards,
|
|
139
147
|
index=index,
|
|
@@ -151,9 +159,7 @@ def create(
|
|
|
151
159
|
"""Build an index for ``shards`` and save it to ``path``."""
|
|
152
160
|
|
|
153
161
|
index = build(shards, progress_bar=progress_bar)
|
|
154
|
-
|
|
155
|
-
num_shards = len(normalized) if is_sharded else None
|
|
156
|
-
save(path, num_shards, index)
|
|
162
|
+
dump(index, path)
|
|
157
163
|
return index
|
|
158
164
|
|
|
159
165
|
|
|
@@ -163,5 +169,5 @@ __all__ = [
|
|
|
163
169
|
"create",
|
|
164
170
|
"load",
|
|
165
171
|
"open",
|
|
166
|
-
"
|
|
172
|
+
"dump",
|
|
167
173
|
]
|
|
@@ -12,23 +12,17 @@ from .utils import (
|
|
|
12
12
|
tar_file_info,
|
|
13
13
|
)
|
|
14
14
|
|
|
15
|
-
IndexedTarIndex = dict[
|
|
15
|
+
IndexedTarIndex = dict[
|
|
16
|
+
str, (int | None, MemberRecord)
|
|
17
|
+
] # fname -> (shard_idx, MemberRecord)
|
|
16
18
|
Shard = str | os.PathLike | IO[bytes]
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def _normalize_shards(
|
|
20
|
-
shards: list[Shard] | Shard,
|
|
21
|
-
) -> tuple[bool, list[Shard], list[bool]]:
|
|
22
|
-
is_sharded = isinstance(shards, list)
|
|
23
|
-
normalized = list(shards) if is_sharded else [shards]
|
|
24
|
-
needs_open = [isinstance(s, (str, os.PathLike)) for s in normalized]
|
|
25
|
-
return is_sharded, normalized, needs_open
|
|
19
|
+
ShardResolver = Callable[[int | None], Shard]
|
|
26
20
|
|
|
27
21
|
|
|
28
22
|
class IndexedTarFile(Mapping):
|
|
29
23
|
def __init__(
|
|
30
24
|
self,
|
|
31
|
-
shards: list[Shard] | Shard,
|
|
25
|
+
shards: list[Shard] | Shard | ShardResolver,
|
|
32
26
|
index: IndexedTarIndex,
|
|
33
27
|
open_fn: Callable[[str | os.PathLike], IO[bytes]] = None,
|
|
34
28
|
buffered_file_reader: bool = True,
|
|
@@ -36,43 +30,66 @@ class IndexedTarFile(Mapping):
|
|
|
36
30
|
if index is None:
|
|
37
31
|
raise ValueError("index must be provided")
|
|
38
32
|
|
|
39
|
-
_, shards, self._needs_open = _normalize_shards(shards)
|
|
40
33
|
self._file_reader = BufferedReader if buffered_file_reader else lambda x: x
|
|
41
|
-
|
|
34
|
+
self._open_fn = (
|
|
42
35
|
open_fn or ThreadSafeFileIO
|
|
43
36
|
) # In our benchmarks, `ThreadSafeFileIO` is even faster than `partial(open, mode="rb", buffering=0)`. Likely due to `pread` being fewer syscalls than `seek` + `read`.
|
|
44
|
-
self._shard_file_objs: list[IO[bytes]] = [
|
|
45
|
-
open_fn(tar) if needs_open else tar
|
|
46
|
-
for tar, needs_open in zip(shards, self._needs_open, strict=True)
|
|
47
|
-
]
|
|
48
37
|
self._index = index
|
|
49
38
|
|
|
39
|
+
self._resolver: ShardResolver
|
|
40
|
+
self._handles: dict[int | None, IO[bytes]] = {}
|
|
41
|
+
self._closable: set[int | None] = set()
|
|
42
|
+
|
|
43
|
+
if callable(shards):
|
|
44
|
+
self._resolver = shards # type: ignore[assignment]
|
|
45
|
+
else:
|
|
46
|
+
sources = (
|
|
47
|
+
{idx: shard for idx, shard in enumerate(shards)}
|
|
48
|
+
if isinstance(shards, list)
|
|
49
|
+
else {None: shards}
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def resolver(idx: int | None) -> Shard:
|
|
53
|
+
return sources[idx]
|
|
54
|
+
|
|
55
|
+
self._resolver = resolver
|
|
56
|
+
|
|
57
|
+
def _ensure_shard(self, shard_idx: int | None) -> IO[bytes]:
|
|
58
|
+
if shard_idx in self._handles:
|
|
59
|
+
return self._handles[shard_idx]
|
|
60
|
+
|
|
61
|
+
source = self._resolver(shard_idx)
|
|
62
|
+
if isinstance(source, (str, os.PathLike)):
|
|
63
|
+
handle = self._open_fn(source)
|
|
64
|
+
self._handles[shard_idx] = handle
|
|
65
|
+
self._closable.add(shard_idx)
|
|
66
|
+
return handle
|
|
67
|
+
|
|
68
|
+
self._handles[shard_idx] = source
|
|
69
|
+
return source
|
|
70
|
+
|
|
50
71
|
def file(self, name: str) -> IO[bytes]:
|
|
51
|
-
|
|
72
|
+
shard_idx, member = self._index[name]
|
|
52
73
|
_, offset_data, size = member
|
|
53
74
|
if isinstance(size, str):
|
|
54
75
|
return self.file(size) # symlink or hard link
|
|
55
76
|
return self._file_reader(
|
|
56
|
-
TarFileSectionIO(self.
|
|
77
|
+
TarFileSectionIO(self._ensure_shard(shard_idx), offset_data, size)
|
|
57
78
|
)
|
|
58
79
|
|
|
59
80
|
def info(self, name: str) -> TarInfo:
|
|
60
|
-
|
|
81
|
+
shard_idx, member = self._index[name]
|
|
61
82
|
offset, _, _ = member
|
|
62
|
-
return tar_file_info(offset, self.
|
|
83
|
+
return tar_file_info(offset, self._ensure_shard(shard_idx))
|
|
63
84
|
|
|
64
85
|
def check_tar_index(self, names: list[str] | None = None):
|
|
65
86
|
for name in names if names is not None else self:
|
|
66
|
-
|
|
67
|
-
check_tar_index(name, member, self.
|
|
87
|
+
shard_idx, member = self._index[name]
|
|
88
|
+
check_tar_index(name, member, self._ensure_shard(shard_idx))
|
|
68
89
|
|
|
69
90
|
def close(self):
|
|
70
|
-
for
|
|
71
|
-
self.
|
|
72
|
-
):
|
|
73
|
-
if needed_open:
|
|
74
|
-
# only close what we opened
|
|
75
|
-
file_obj.close()
|
|
91
|
+
for key in self._closable:
|
|
92
|
+
self._handles[key].close()
|
|
76
93
|
|
|
77
94
|
def __enter__(self):
|
|
78
95
|
return self
|
|
@@ -105,7 +105,6 @@ class ThreadSafeFileIO(io.RawIOBase):
|
|
|
105
105
|
self._path = str(path)
|
|
106
106
|
self._fd = os.open(path, os.O_RDONLY)
|
|
107
107
|
self._local = threading.local()
|
|
108
|
-
self._closed = False
|
|
109
108
|
|
|
110
109
|
def _get_pos(self) -> int:
|
|
111
110
|
return getattr(self._local, "pos", 0)
|
|
@@ -147,9 +146,9 @@ class ThreadSafeFileIO(io.RawIOBase):
|
|
|
147
146
|
return False
|
|
148
147
|
|
|
149
148
|
def close(self) -> None:
|
|
150
|
-
if not self.
|
|
149
|
+
if not self.closed and hasattr(self, "_fd"):
|
|
151
150
|
os.close(self._fd)
|
|
152
|
-
|
|
151
|
+
super().close()
|
|
153
152
|
|
|
154
153
|
def fileno(self) -> int:
|
|
155
154
|
return self._fd
|
itar-0.2.0/README.md
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
# itar
|
|
2
|
-
|
|
3
|
-
`itar` builds constant-time indexes for one or more TAR shards so you can seek directly to a member without extracting the archive. The project ships a small CLI (`itar`) and a Python helper (`IndexedTarFile`) plus a module-level `itar.open()` convenience that mirrors `tarfile.open()`.
|
|
4
|
-
|
|
5
|
-
## Quickstart (single tarball)
|
|
6
|
-
|
|
7
|
-
```bash
|
|
8
|
-
echo "Hello world!" > hello.txt
|
|
9
|
-
tar cf hello.tar hello.txt # regular tarball
|
|
10
|
-
itar index build hello.itar # indexes hello.tar
|
|
11
|
-
itar index list hello.itar # list indexed members
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
```python
|
|
15
|
-
import itar
|
|
16
|
-
|
|
17
|
-
with itar.open("hello.itar") as archive:
|
|
18
|
-
print(archive["hello.txt"].read())
|
|
19
|
-
|
|
20
|
-
# If you just need the index dictionary without opening handles:
|
|
21
|
-
index = itar.index.build("hello.tar")
|
|
22
|
-
# Write the index file to disk:
|
|
23
|
-
itar.index.save("hello.itar", num_shards=None, index=index)
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## Quickstart (sharded tarballs)
|
|
27
|
-
|
|
28
|
-
Give each shard a zero-padded suffix before building the index:
|
|
29
|
-
|
|
30
|
-
```bash
|
|
31
|
-
tar cf photos-0.tar wedding/ # shard 0
|
|
32
|
-
tar cf photos-1.tar vacation/ # shard 1
|
|
33
|
-
itar index build photos.itar # discovers photos-0.tar, photos-1.tar, ...
|
|
34
|
-
itar index list -l photos.itar # shard index, offsets, byte sizes
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
```python
|
|
38
|
-
import itar
|
|
39
|
-
|
|
40
|
-
with itar.open("photos.itar") as photos:
|
|
41
|
-
assert "wedding/cake.jpg" in photos
|
|
42
|
-
img_bytes = photos["vacation/sunrise.jpg"].read()
|
|
43
|
-
|
|
44
|
-
index = itar.index.build(["photos-0.tar", "photos-1.tar"])
|
|
45
|
-
itar.index.create("photos.itar", ["photos-0.tar", "photos-1.tar"])
|
|
46
|
-
|
|
47
|
-
num_shards, stored_index = itar.index.load("photos.itar")
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
## CLI reference
|
|
51
|
-
|
|
52
|
-
| Command | Purpose |
|
|
53
|
-
| --- | --- |
|
|
54
|
-
| `itar index build <archive>.itar [--single TAR | --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
|
|
55
|
-
| `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
|
|
56
|
-
| `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
|
|
57
|
-
|
|
58
|
-
## Python helpers
|
|
59
|
-
|
|
60
|
-
- `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
|
|
61
|
-
- `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
|
|
62
|
-
- `itar.index.save(path, num_shards, index)`: serialize an index you built elsewhere.
|
|
63
|
-
- `itar.index.load(path) -> (num_shards, index)`: load the msgpack index without opening shards.
|
|
64
|
-
- `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
|
|
File without changes
|
|
File without changes
|