itar 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
itar-0.4.2/PKG-INFO ADDED
@@ -0,0 +1,67 @@
1
+ Metadata-Version: 2.3
2
+ Name: itar
3
+ Version: 0.4.2
4
+ Summary: tar file index for constant-time member access
5
+ Author: Karim Abou Zeid
6
+ Author-email: Karim Abou Zeid <contact@ka.codes>
7
+ Requires-Dist: msgpack>=1.1.1
8
+ Requires-Dist: tqdm>=4.67.1
9
+ Requires-Dist: humanize>=4.12.3
10
+ Requires-Python: >=3.10
11
+ Description-Content-Type: text/markdown
12
+
13
+ # itar
14
+
15
+ [![PyPI version](https://img.shields.io/pypi/v/itar.svg)](https://pypi.python.org/pypi/itar)
16
+ [![docs](https://img.shields.io/badge/docs-brightgreen.svg)](https://kabouzeid.github.io/itar/)
17
+
18
+ `itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
19
+
20
+ Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
21
+
22
+ ## Quickstart
23
+
24
+ ```bash
25
+ pip install itar
26
+ ```
27
+
28
+ ### Single tarball
29
+
30
+ ```bash
31
+ echo "Hello world!" > hello.txt
32
+ tar cf hello.tar hello.txt # regular tarball
33
+
34
+ itar index create hello.itar # indexes hello.tar
35
+ itar index list hello.itar # list indexed members
36
+ ```
37
+
38
+ ```python
39
+ import itar
40
+
41
+ with itar.open("hello.itar") as archive:
42
+ print(archive["hello.txt"].read())
43
+ ```
44
+
45
+ ### Sharded tarballs
46
+
47
+ Give each shard a zero-padded suffix before building the index:
48
+
49
+ ```bash
50
+ tar cf photos-0.tar wedding/ # shard 0
51
+ tar cf photos-1.tar vacation/ # shard 1
52
+
53
+ itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
54
+ itar index list -l photos.itar # shard index, offsets, byte sizes
55
+ ```
56
+
57
+ ```python
58
+ import itar
59
+
60
+ with itar.open("photos.itar") as photos:
61
+ assert "wedding/cake.jpg" in photos
62
+ img_bytes = photos["vacation/sunrise.jpg"].read()
63
+ ```
64
+
65
+ ## Docs
66
+
67
+ Full CLI, API, and format details live in the [documentation site](https://kabouzeid.github.io/itar).
itar-0.4.2/README.md ADDED
@@ -0,0 +1,55 @@
1
+ # itar
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/itar.svg)](https://pypi.python.org/pypi/itar)
4
+ [![docs](https://img.shields.io/badge/docs-brightgreen.svg)](https://kabouzeid.github.io/itar/)
5
+
6
+ `itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
7
+
8
+ Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
9
+
10
+ ## Quickstart
11
+
12
+ ```bash
13
+ pip install itar
14
+ ```
15
+
16
+ ### Single tarball
17
+
18
+ ```bash
19
+ echo "Hello world!" > hello.txt
20
+ tar cf hello.tar hello.txt # regular tarball
21
+
22
+ itar index create hello.itar # indexes hello.tar
23
+ itar index list hello.itar # list indexed members
24
+ ```
25
+
26
+ ```python
27
+ import itar
28
+
29
+ with itar.open("hello.itar") as archive:
30
+ print(archive["hello.txt"].read())
31
+ ```
32
+
33
+ ### Sharded tarballs
34
+
35
+ Give each shard a zero-padded suffix before building the index:
36
+
37
+ ```bash
38
+ tar cf photos-0.tar wedding/ # shard 0
39
+ tar cf photos-1.tar vacation/ # shard 1
40
+
41
+ itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
42
+ itar index list -l photos.itar # shard index, offsets, byte sizes
43
+ ```
44
+
45
+ ```python
46
+ import itar
47
+
48
+ with itar.open("photos.itar") as photos:
49
+ assert "wedding/cake.jpg" in photos
50
+ img_bytes = photos["vacation/sunrise.jpg"].read()
51
+ ```
52
+
53
+ ## Docs
54
+
55
+ Full CLI, API, and format details live in the [documentation site](https://kabouzeid.github.io/itar).
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "itar"
3
- version = "0.4.0"
4
- description = "Add your description here"
3
+ version = "0.4.2"
4
+ description = "tar file index for constant-time member access"
5
5
  readme = "README.md"
6
6
  authors = [
7
7
  { name = "Karim Abou Zeid", email = "contact@ka.codes" }
@@ -9,23 +9,21 @@ authors = [
9
9
  requires-python = ">=3.10"
10
10
  dependencies = [
11
11
  "msgpack>=1.1.1",
12
+ "tqdm>=4.67.1",
13
+ "humanize>=4.12.3",
12
14
  ]
13
15
 
14
16
  [dependency-groups]
15
17
  dev = [
16
18
  "pytest>=8.4.0",
17
19
  "ruff>=0.11.13",
20
+ "zensical>=0.0.10",
21
+ "mkdocstrings-python>=2.0.1",
18
22
  ]
19
23
 
20
24
  [project.scripts]
21
25
  itar = "itar.cli:main"
22
26
 
23
- [project.optional-dependencies]
24
- cli = [
25
- "tqdm>=4.67.1",
26
- "humanize>=4.12.3",
27
- ]
28
-
29
27
  [build-system]
30
28
  requires = ["uv_build>=0.9.0,<0.10.0"]
31
29
  build-backend = "uv_build"
@@ -46,16 +46,19 @@ class IndexLayout:
46
46
  return candidates
47
47
 
48
48
 
49
- def _make_default_resolver(layout: IndexLayout) -> ShardResolver:
50
- def resolve(shard_idx: int | None) -> Path:
49
+ class DefaultResolver:
50
+ """Resolve shard indices to paths next to an existing index file."""
51
+
52
+ def __init__(self, layout: IndexLayout):
53
+ self.layout = layout
54
+
55
+ def __call__(self, shard_idx: int | None) -> Path:
51
56
  if shard_idx is None:
52
- return layout.single_tar()
57
+ return self.layout.single_tar()
53
58
 
54
- shard_paths = layout.discover_shards()
59
+ shard_paths = self.layout.discover_shards()
55
60
  return shard_paths[shard_idx]
56
61
 
57
- return resolve
58
-
59
62
 
60
63
  def _build_index_from_fileobjs(
61
64
  file_objs: Iterable[IO[bytes]],
@@ -132,7 +135,7 @@ def open(
132
135
  open_fn: Callable[[str | os.PathLike], IO[bytes]] | None = None,
133
136
  buffered_file_reader: bool = True,
134
137
  ) -> IndexedTarFile:
135
- """Open an :class:`IndexedTarFile` using an on-disk index file."""
138
+ """Open an ``IndexedTarFile`` using an on-disk index file."""
136
139
 
137
140
  path = Path(path)
138
141
  index = load(path)
@@ -141,7 +144,7 @@ def open(
141
144
  if shards is not None:
142
145
  resolved_shards = shards
143
146
  else:
144
- resolved_shards = _make_default_resolver(layout)
147
+ resolved_shards = DefaultResolver(layout)
145
148
  return IndexedTarFile(
146
149
  resolved_shards,
147
150
  index=index,
@@ -164,10 +167,10 @@ def create(
164
167
 
165
168
 
166
169
  __all__ = [
167
- "IndexLayout",
168
- "build",
169
170
  "create",
170
- "load",
171
171
  "open",
172
+ "build",
172
173
  "dump",
174
+ "load",
175
+ "IndexLayout",
173
176
  ]
@@ -20,6 +20,29 @@ ShardResolver = Callable[[int | None], Shard]
20
20
 
21
21
 
22
22
  class IndexedTarFile(Mapping):
23
+ """
24
+ Read-only mapping that serves members out of indexed tar shards.
25
+
26
+ Args:
27
+ shards: Shard sources (single tar, list of tars, or resolver callable).
28
+ index: Precomputed index mapping member names to offsets.
29
+ open_fn: Optional callable to open paths; defaults to a thread-safe file reader.
30
+ buffered_file_reader: Wrap member streams in a buffered reader when True.
31
+
32
+ Use ``itar.open`` to construct this class. It supports the mapping protocol
33
+ for read access (``archive["path/to/file"]``) and should be used as a
34
+ context manager to close any open file handles.
35
+
36
+ Example:
37
+ ```python
38
+ import itar
39
+
40
+ with itar.open("photos.itar") as archive:
41
+ jpg = archive["vacation/sunrise.jpg"].read()
42
+ assert "wedding/cake.jpg" in archive
43
+ ```
44
+ """
45
+
23
46
  def __init__(
24
47
  self,
25
48
  shards: list[Shard] | Shard | ShardResolver,
@@ -69,6 +92,7 @@ class IndexedTarFile(Mapping):
69
92
  return source
70
93
 
71
94
  def file(self, name: str) -> IO[bytes]:
95
+ """Return a readable file-like object for an indexed member."""
72
96
  shard_idx, member = self._index[name]
73
97
  _, offset_data, size = member
74
98
  if isinstance(size, str):
@@ -78,11 +102,13 @@ class IndexedTarFile(Mapping):
78
102
  )
79
103
 
80
104
  def info(self, name: str) -> TarInfo:
105
+ """Return the ``TarInfo`` for an indexed member without reading data."""
81
106
  shard_idx, member = self._index[name]
82
107
  offset, _, _ = member
83
108
  return tar_file_info(offset, self._ensure_shard(shard_idx))
84
109
 
85
110
  def check_tar_index(self, names: list[str] | None = None):
111
+ """Validate stored offsets for the given names (or all members)."""
86
112
  for name in names if names is not None else self:
87
113
  shard_idx, member = self._index[name]
88
114
  check_tar_index(name, member, self._ensure_shard(shard_idx))
@@ -11,6 +11,7 @@ MemberRecord = tuple[int, int, int | str] # (offset, offset_data, size | linkna
11
11
 
12
12
 
13
13
  def tar_file_info(offset: int, file_obj: IO[bytes]) -> TarInfo:
14
+ """Return a ``TarInfo`` for the member starting at ``offset``."""
14
15
  file_obj.seek(offset)
15
16
  return TarInfo.fromtarfile(
16
17
  # want to avoid creating a new TarFile instance (potentially slow)
@@ -43,6 +44,7 @@ def tarinfo2member(tarinfo: TarInfo) -> MemberRecord:
43
44
  def build_tar_index(
44
45
  tar: str | os.PathLike | IO[bytes] | TarFile,
45
46
  ) -> dict[str, MemberRecord]:
47
+ """Collect offsets and sizes for all files and links in a tar archive."""
46
48
  if isinstance(tar, str | os.PathLike):
47
49
  tar = tarfile.open(tar, "r:")
48
50
  elif isinstance(tar, TarFile):
@@ -77,6 +79,7 @@ def check_tar_index(
77
79
  tar_offset: MemberRecord,
78
80
  file_obj: IO[bytes],
79
81
  ):
82
+ """Confirm that ``tar_offset`` matches the member on disk."""
80
83
  offset, offset_data, size = tar_offset
81
84
  info = tar_file_info(offset, file_obj)
82
85
  if (
@@ -165,6 +168,8 @@ class ThreadSafeFileIO(io.RawIOBase):
165
168
 
166
169
 
167
170
  class TarFileSectionIO(io.RawIOBase):
171
+ """A read-only view over a byte range inside a larger file object."""
172
+
168
173
  def __init__(self, fileobj: BinaryIO, offset: int, size: int):
169
174
  self._fileobj = fileobj
170
175
  self._start = offset
itar-0.4.0/PKG-INFO DELETED
@@ -1,90 +0,0 @@
1
- Metadata-Version: 2.3
2
- Name: itar
3
- Version: 0.4.0
4
- Summary: Add your description here
5
- Author: Karim Abou Zeid
6
- Author-email: Karim Abou Zeid <contact@ka.codes>
7
- Requires-Dist: msgpack>=1.1.1
8
- Requires-Dist: tqdm>=4.67.1 ; extra == 'cli'
9
- Requires-Dist: humanize>=4.12.3 ; extra == 'cli'
10
- Requires-Python: >=3.10
11
- Provides-Extra: cli
12
- Description-Content-Type: text/markdown
13
-
14
- # itar
15
-
16
- [![image](https://img.shields.io/pypi/v/itar.svg)](https://pypi.python.org/pypi/itar)
17
-
18
- `itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
19
-
20
- Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
21
-
22
- ## Quickstart (single tarball)
23
-
24
- ```bash
25
- echo "Hello world!" > hello.txt
26
- tar cf hello.tar hello.txt # regular tarball
27
-
28
- itar index create hello.itar # indexes hello.tar
29
- itar index list hello.itar # list indexed members
30
- ```
31
-
32
- ```python
33
- import itar
34
-
35
- with itar.open("hello.itar") as archive:
36
- print(archive["hello.txt"].read())
37
- ```
38
-
39
- ## Quickstart (sharded tarballs)
40
-
41
- Give each shard a zero-padded suffix before building the index:
42
-
43
- ```bash
44
- tar cf photos-0.tar wedding/ # shard 0
45
- tar cf photos-1.tar vacation/ # shard 1
46
-
47
- itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
48
- itar index list -l photos.itar # shard index, offsets, byte sizes
49
- ```
50
-
51
- ```python
52
- import itar
53
-
54
- with itar.open("photos.itar") as photos:
55
- assert "wedding/cake.jpg" in photos
56
- img_bytes = photos["vacation/sunrise.jpg"].read()
57
- ```
58
-
59
- ## CLI reference
60
-
61
- | Command | Purpose |
62
- | --- | --- |
63
- | `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
64
- | `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
65
- | `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
66
- | `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
67
-
68
- ## Python helpers
69
-
70
- - `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
71
- - `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
72
- - `itar.index.dump(index, path)`: serialize an index you built elsewhere.
73
- - `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
74
- - `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
75
-
76
- ## `itar` File Format
77
- An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
78
- ```python
79
- {
80
- "path/to/member1.jpg": [ # file name
81
- null, # either null or shard index (0-based)
82
- [
83
- 2048, # metadata byte offset
84
- 2560, # data byte offset
85
- 1048576, # file length in bytes
86
- ],
87
- ],
88
- ...
89
- }
90
- ```
itar-0.4.0/README.md DELETED
@@ -1,77 +0,0 @@
1
- # itar
2
-
3
- [![image](https://img.shields.io/pypi/v/itar.svg)](https://pypi.python.org/pypi/itar)
4
-
5
- `itar` builds constant‑time indexes over one or more tar file shards, enabling direct, random access to members without extracting the archives. It ships a lightweight CLI (`itar`) and a Python API.
6
-
7
- Designed for large datasets and deep‑learning pipelines, it supports single or sharded tar archives with thread‑safe access for concurrent reads.
8
-
9
- ## Quickstart (single tarball)
10
-
11
- ```bash
12
- echo "Hello world!" > hello.txt
13
- tar cf hello.tar hello.txt # regular tarball
14
-
15
- itar index create hello.itar # indexes hello.tar
16
- itar index list hello.itar # list indexed members
17
- ```
18
-
19
- ```python
20
- import itar
21
-
22
- with itar.open("hello.itar") as archive:
23
- print(archive["hello.txt"].read())
24
- ```
25
-
26
- ## Quickstart (sharded tarballs)
27
-
28
- Give each shard a zero-padded suffix before building the index:
29
-
30
- ```bash
31
- tar cf photos-0.tar wedding/ # shard 0
32
- tar cf photos-1.tar vacation/ # shard 1
33
-
34
- itar index create photos.itar # discovers photos-0.tar, photos-1.tar, ...
35
- itar index list -l photos.itar # shard index, offsets, byte sizes
36
- ```
37
-
38
- ```python
39
- import itar
40
-
41
- with itar.open("photos.itar") as photos:
42
- assert "wedding/cake.jpg" in photos
43
- img_bytes = photos["vacation/sunrise.jpg"].read()
44
- ```
45
-
46
- ## CLI reference
47
-
48
- | Command | Purpose |
49
- | --- | --- |
50
- | `itar index create <archive>.itar [--single TAR \| --shards shard0.tar shard1.tar ...]` | Indexes a single archive or an explicit set of shards. With no flags, shards are auto-discovered next to `<archive>.itar`. |
51
- | `itar index list <archive>.itar` | Lists members. Use `-l` for shard/offset info and `-H` for human-readable sizes. |
52
- | `itar index check <archive>.itar` | Validates recorded entries; add `--member NAME` to focus on specific files. |
53
- | `itar cat <archive>.itar <member>` | Streams a member’s bytes to stdout. |
54
-
55
- ## Python helpers
56
-
57
- - `itar.index.build(shards, progress_bar=False) -> dict`: construct an index mapping for paths, file objects, or buffers.
58
- - `itar.index.create("archive.itar", shards)`: convenience wrapper that builds + saves an index file.
59
- - `itar.index.dump(index, path)`: serialize an index you built elsewhere.
60
- - `itar.index.load(path) -> dict`: load the msgpack index without opening shards.
61
- - `itar.open(path, *, shards=None, open_fn=None) -> IndexedTarFile`: attach shard handles using an existing index file.
62
-
63
- ## `itar` File Format
64
- An `itar` index file is a simple [MessagePack](https://msgpack.org/) dictionary mapping member paths to metadata:
65
- ```python
66
- {
67
- "path/to/member1.jpg": [ # file name
68
- null, # either null or shard index (0-based)
69
- [
70
- 2048, # metadata byte offset
71
- 2560, # data byte offset
72
- 1048576, # file length in bytes
73
- ],
74
- ],
75
- ...
76
- }
77
- ```
File without changes
File without changes
File without changes