untar 0.1.0__tar.gz

untar-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,78 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+        include:
+          - os: macos-latest
+            python-version: "3.14"
+          - os: windows-latest
+            python-version: "3.14"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e '.[dev]'
+      - run: python tests/vendor/fetch_cpython_testdata.py cpython-testdata
+      - run: pytest -rs
+        env:
+          UNTAR_CPYTHON_TESTDATA: cpython-testdata
+
+  wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
+        with:
+          python-version: "3.14"
+      - run: pip install build
+      - run: python -m build --wheel
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: wheel
+          path: dist/
+
+  # Spack still supports Python 3.6, for which hatchling and hypothesis are too new: test the
+  # wheel built above, which also checks that old pip installs it.
+  test-old:
+    needs: wheel
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.6", "3.7"]
+    runs-on: ubuntu-latest
+    container: python:${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        with:
+          name: wheel
+          path: dist/
+      - run: pip install pytest dist/*.whl
+      - run: python tests/vendor/fetch_cpython_testdata.py cpython-testdata
+      - run: rm untar.py && pytest -rs
+        env:
+          UNTAR_CPYTHON_TESTDATA: cpython-testdata
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
+        with:
+          python-version: "3.14"
+      - run: pip install -e '.[dev]'
+      - run: ruff check
+      - run: ruff format --diff
+      - run: mypy

untar-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
+        with:
+          python-version: "3.14"
+      - run: pip install build twine
+      - run: python -m build
+      - run: twine check --strict dist/*
+      - name: The tag matches the version
+        run: test "${GITHUB_REF_NAME}" = "v$(python -c 'import untar; print(untar.__version__)')"
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2

untar-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+.ruff_cache/
+.mypy_cache/
+.pytest_cache/
+.hypothesis/
+*.egg-info/
+dist/
+.venv/
+cpython-testdata/

untar-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Harmen Stoppels
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

untar-0.1.0/PKG-INFO

@@ -0,0 +1,80 @@
+Metadata-Version: 2.5
+Name: untar
+Version: 0.1.0
+Summary: Fast, safe, streaming extraction of tar archives that distribute software
+Project-URL: Homepage, https://github.com/haampie/untar
+Project-URL: Repository, https://github.com/haampie/untar
+Author-email: Harmen Stoppels <me@harmenstoppels.nl>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: archive,extract,tar
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: System :: Archiving
+Requires-Python: >=3.6
+Provides-Extra: dev
+Requires-Dist: hypothesis; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# untar
+
+Fast, safe, streaming extraction of tar archives that distribute software, in a single pure
+Python module without dependencies, for Python 3.6 and later.
+
+```python
+import untar
+
+# Extract an archive, from a path or any readable binary stream
+untar.extract("pkg.tar.gz", "/opt/pkg")
+
+# Extract the dir that contains `.meta`, wherever it is in the archive
+root = untar.extract("pkg.tar.gz", "/opt/pkg", root_marker=".meta")
+```
+
+## Scope
+
+untar is for installing software from archives, not for restoring backups. Owners, groups,
+timestamps and permissions recorded in an archive describe the machine it was made on; on
+another machine they are meaningless at best and a liability at worst. So untar does not
+restore them: files are created with `file_mode` (without executable bits where the archive
+has none for the owner), dirs with `dir_mode`, optionally with group `gid`, and only regular
+files, dirs, symlinks and hardlinks are extracted.
+
+## Features
+
+- **One pass.** The archive is read front to back once, without seeking, from a file, pipe or
+  network stream. Gzip is detected from the first bytes.
+- **Less work per member.** Only the header fields that extraction needs are parsed (ustar,
+  pax and GNU long names), and a file costs an `open`, `write`s and a `close`.
+- **A root only known from the contents.** With `root_marker`, the parent of the shallowest
+  member with that name is extracted, like `tar --strip-components` with a count that is only
+  known after the last member.
+- **No path length limits.** Entries are created relative to open dir file descriptors, so no
+  path longer than one name is passed to the operating system.
+
+## Safety
+
+Nothing is written outside the destination, whatever the archive contains. Absolute names and
+`..` are rejected, dirs are opened with `O_NOFOLLOW` so nothing is written through a symlink,
+and files are created with `O_EXCL | O_NOFOLLOW` so existing entries are never replaced.
+Symlinks may point anywhere, but untar never follows them. On Windows, which has no dir file
+descriptors, these checks are path based and not safe against concurrent changes.
+
+Archives that tar implementations read differently are rejected, so that no member can hide
+from one of them: numbers must be plain digits, a member's name and size come from one header
+only, and global pax headers cannot set them. The gzip checksum is always verified.
+
+## Development
+
+```console
+$ pip install -e '.[dev]'
+$ python tests/vendor/fetch_cpython_testdata.py cpython-testdata
+$ UNTAR_CPYTHON_TESTDATA=cpython-testdata pytest
+$ ruff check && ruff format --diff && mypy
+```
+
+`tests/vendor/test_cpython.py` is adapted from CPython's `test_tarfile.py` and is under the PSF
+License Agreement included in it. The rest of untar is under the MIT license.

untar-0.1.0/README.md

@@ -0,0 +1,59 @@
+# untar
+
+Fast, safe, streaming extraction of tar archives that distribute software, in a single pure
+Python module without dependencies, for Python 3.6 and later.
+
+```python
+import untar
+
+# Extract an archive, from a path or any readable binary stream
+untar.extract("pkg.tar.gz", "/opt/pkg")
+
+# Extract the dir that contains `.meta`, wherever it is in the archive
+root = untar.extract("pkg.tar.gz", "/opt/pkg", root_marker=".meta")
+```
+
+## Scope
+
+untar is for installing software from archives, not for restoring backups. Owners, groups,
+timestamps and permissions recorded in an archive describe the machine it was made on; on
+another machine they are meaningless at best and a liability at worst. So untar does not
+restore them: files are created with `file_mode` (without executable bits where the archive
+has none for the owner), dirs with `dir_mode`, optionally with group `gid`, and only regular
+files, dirs, symlinks and hardlinks are extracted.
+
+## Features
+
+- **One pass.** The archive is read front to back once, without seeking, from a file, pipe or
+  network stream. Gzip is detected from the first bytes.
+- **Less work per member.** Only the header fields that extraction needs are parsed (ustar,
+  pax and GNU long names), and a file costs an `open`, `write`s and a `close`.
+- **A root only known from the contents.** With `root_marker`, the parent of the shallowest
+  member with that name is extracted, like `tar --strip-components` with a count that is only
+  known after the last member.
+- **No path length limits.** Entries are created relative to open dir file descriptors, so no
+  path longer than one name is passed to the operating system.
+
+## Safety
+
+Nothing is written outside the destination, whatever the archive contains. Absolute names and
+`..` are rejected, dirs are opened with `O_NOFOLLOW` so nothing is written through a symlink,
+and files are created with `O_EXCL | O_NOFOLLOW` so existing entries are never replaced.
+Symlinks may point anywhere, but untar never follows them. On Windows, which has no dir file
+descriptors, these checks are path based and not safe against concurrent changes.
+
+Archives that tar implementations read differently are rejected, so that no member can hide
+from one of them: numbers must be plain digits, a member's name and size come from one header
+only, and global pax headers cannot set them. The gzip checksum is always verified.
+
+## Development
+
+```console
+$ pip install -e '.[dev]'
+$ python tests/vendor/fetch_cpython_testdata.py cpython-testdata
+$ UNTAR_CPYTHON_TESTDATA=cpython-testdata pytest
+$ ruff check && ruff format --diff && mypy
+```
+
+`tests/vendor/test_cpython.py` is adapted from CPython's `test_tarfile.py` and is under the PSF
+License Agreement included in it. The rest of untar is under the MIT license.

untar-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[project]
+name = "untar"
+description = "Fast, safe, streaming extraction of tar archives that distribute software"
+readme = "README.md"
+requires-python = ">=3.6"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Harmen Stoppels", email = "me@harmenstoppels.nl" }]
+keywords = ["tar", "archive", "extract"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Topic :: System :: Archiving",
+]
+dependencies = []
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/haampie/untar"
+Repository = "https://github.com/haampie/untar"
+
+[project.optional-dependencies]
+dev = ["pytest", "hypothesis", "ruff", "mypy"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "untar.py"
+
+[tool.hatch.build.targets.wheel]
+only-include = ["untar.py"]
+
+# The tests adapted from CPython are under the PSF license; keep distributions MIT only
+[tool.hatch.build.targets.sdist]
+exclude = ["tests/vendor"]
+
+[tool.mypy]
+files = ["untar.py"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+# The oldest target ruff supports; requires-python is lower
+target-version = "py37"
+# from __future__ import annotations needs Python 3.7
+lint.ignore = ["FA100"]

untar-0.1.0/tests/test_fuzz.py

@@ -0,0 +1,217 @@
+"""Structure-aware fuzzing: one header field at a time gets an odd value, and the checksum is
+recomputed, so that mutations get past the checksum. Whatever the archive, reading must make
+progress and raise only untar.Error, and extracting must raise only untar.Error or OSError and
+change nothing outside the destination."""
+
+import errno
+import gzip
+import io
+import itertools
+import os
+import random
+import shutil
+import tarfile
+
+import pytest
+from test_untar import SAMPLE, make_tar, needs_symlinks, snapshot
+
+import untar
+
+FIELDS = {
+    "name": (0, 100),
+    "mode": (100, 108),
+    "size": (124, 136),
+    "type": (156, 157),
+    "linkname": (157, 257),
+    "magic": (257, 265),
+    "prefix": (345, 500),
+}
+
+NUMBERS = [
+    b"-1777",
+    b"+17",
+    b"0o17",
+    b"0_17",
+    b"1 7",
+    b"",
+    b" " * 11,
+    b"0",
+    b"1",
+    b"777",
+    b"77777777777",
+    b"\x80" + bytes(10) + b"\x10",
+    b"\x80" + b"\xff" * 11,
+    b"\xff" * 12,
+    b"\xd9\xa3",
+]
+
+VALUES = {
+    "name": [
+        b"",
+        b".",
+        b"..",
+        b"/abs",
+        b"a/../b",
+        b"a//b",
+        b"./a",
+        b"a/",
+        b"a/.",
+        b"\xff\xfe",
+    ],
+    "mode": NUMBERS,
+    "size": NUMBERS,
+    "type": [bytes([t]) for t in b"012345675xgLKSXZ\0"],
+    "linkname": [b"", b".", b"..", b"/etc", b"pkg", b"../pkg", b"pkg/bin/app", b"\0x"],
+    "magic": [b"ustar\x0000", b"ustar  \0", b"ustar\0\0\0", bytes(8)],
+    "prefix": [b"", b"pkg", b"/", b"..", b"a/b"],
+}
+
+RECORDS = [
+    b"5 abc\n",
+    b"6 =abc\n",
+    b"10 size=0\n",
+    b"13 size=-512\n",
+    b"11 size=1_0\n",
+    b"13 path=a/..\n",
+    b"12 path=a\0b\n",
+    b"9 path=\n",
+    b"16 linkpath=/etc\n",
+    b"+9 path=x\n",
+    b"99 path=x\n",
+    b"25 GNU.sparse.size=1000\n",
+]
+
+
+def archives():
+    gnu = make_tar(SAMPLE, tarfile.GNU_FORMAT)
+    pax = make_tar(SAMPLE, tarfile.PAX_FORMAT)
+    out = io.BytesIO()
+    with tarfile.open(fileobj=out, mode="w", format=tarfile.PAX_FORMAT) as tar:
+        tar.pax_headers = {"comment": "global"}
+        for name in ("a", "b/c"):
+            info = tarfile.TarInfo(name)
+            info.size = 3
+            tar.addfile(info, io.BytesIO(b"abc"))
+    return [gnu, pax, out.getvalue()]
+
+
+def headers(archive: bytes):
+    """Offsets of the blocks that are headers."""
+    offsets = []
+    for offset in range(0, len(archive), 512):
+        block = archive[offset : offset + 512]
+        try:
+            if block != bytes(512) and untar._checksum_ok(block):
+                offsets.append(offset)
+        except untar.Error:
+            pass
+    return offsets
+
+
+def checksum(block: bytearray) -> None:
+    block[148:156] = b" " * 8
+    block[148:156] = b"%06o\0 " % sum(block)
+
+
+def mutate(archive: bytes, rng: random.Random) -> bytes:
+    data = bytearray(archive)
+    for _ in range(rng.choice([1, 1, 2, 3])):
+        offset = rng.choice(headers(archive))
+        if data[offset + 156 : offset + 157] in (b"x", b"g") and rng.random() < 0.3:
+            # Replace the pax records, keeping the header's size
+            records = b"".join(rng.sample(RECORDS, rng.randint(1, 3)))
+            try:
+                size = untar._int(bytes(data[offset + 124 : offset + 136]))
+            except untar.Error:
+                continue
+            data[offset + 512 : offset + 512 + size] = records[:size].ljust(size, b"\0")
+            continue
+        field = rng.choice(list(FIELDS))
+        start, end = FIELDS[field]
+        value = rng.choice(VALUES[field])
+        if rng.random() < 0.1:
+            value = bytes(rng.randrange(256) for _ in range(end - start))
+        data[offset + start : offset + end] = value[: end - start].ljust(
+            end - start, b"\0"
+        )
+        block = data[offset : offset + 512]
+        checksum(block)
+        data[offset : offset + 512] = block
+    return bytes(data)
+
+
+def check_reader(archive: bytes) -> None:
+    """The reader ends within the number of blocks, and raises only untar.Error."""
+    limit = len(archive) // 512 + 1
+    reader = untar.TarReader(io.BytesIO(archive))
+    try:
+        count = 0
+        for _ in itertools.islice(reader, limit + 1):
+            for _ in reader.chunks():
+                pass
+            count += 1
+        assert count <= limit, "the reader does not make progress"
+    except untar.Error:
+        pass
+
+
+def check_extract(archive: bytes, outer) -> None:
+    """Extracting raises only untar.Error or OSError, and changes nothing outside dest."""
+    flag = outer / "flag"
+    dest = outer / "dest"
+    try:
+        untar.extract(io.BytesIO(archive), str(dest))
+    except (untar.Error, OSError):
+        pass
+    assert sorted(os.listdir(outer)) == ["dest", "flag"]
+    assert flag.read_bytes() == b"flag"
+    assert not [n for n in os.listdir(dest) if n.startswith(".untar-")]
+    shutil.rmtree(dest)
+
+
+@needs_symlinks
+@pytest.mark.parametrize("links", ["supported", "unsupported"])
+def test_fuzz_fields(tmp_path, monkeypatch, links):
+    if links == "unsupported":
+
+        def fail(*args, **kwargs):
+            raise OSError(errno.EPERM, "links are not supported")
+
+        monkeypatch.setattr(os, "symlink", fail)
+        monkeypatch.setattr(os, "link", fail)
+    outer = tmp_path / "outer"
+    outer.mkdir()
+    (outer / "flag").write_bytes(b"flag")
+    rng = random.Random(f"fields-{links}")
+    bases = archives()
+    for _ in range(int(os.environ.get("UNTAR_FUZZ", "600"))):
+        archive = mutate(rng.choice(bases), rng)
+        check_reader(archive)
+        check_extract(archive, outer)
+
+
+@needs_symlinks
+def test_fuzz_gzip(tmp_path):
+    """Truncated or corrupted gzip fails, or extracts exactly the original tree."""
+    plain = make_tar(SAMPLE)
+    untar.extract(io.BytesIO(plain), str(tmp_path / "expected"))
+    expected = snapshot(tmp_path / "expected")
+    compressed = gzip.compress(plain)
+    rng = random.Random("gzip")
+    outcomes = {"error": 0, "same": 0}
+    for i in range(300):
+        data = bytearray(compressed)
+        if rng.random() < 0.3:
+            data = data[: rng.randrange(len(data))]
+        else:
+            for _ in range(rng.randint(1, 4)):
+                data[rng.randrange(len(data))] ^= 1 << rng.randrange(8)
+        dest = tmp_path / str(i)
+        try:
+            untar.extract(io.BytesIO(bytes(data)), str(dest))
+        except untar.Error:
+            outcomes["error"] += 1
+            continue
+        assert snapshot(dest) == expected
+        outcomes["same"] += 1
+    assert outcomes["error"] > 0

untar-0.1.0/tests/test_property.py

@@ -0,0 +1,79 @@
+"""Property test against tarfile, which needs hypothesis."""
+
+import io
+import os
+import tarfile
+
+import pytest
+from test_untar import make_tar, needs_symlinks, snapshot, tarfile_extract
+
+import untar
+
+hypothesis = pytest.importorskip("hypothesis")
+st = hypothesis.strategies
+
+names = st.sampled_from(["a", "b", "lib", ".meta", "x" * 120, "ü", "-x"])
+
+
+@st.composite
+def archives(draw):
+    """Random valid archives: a tree in random order, possibly without some dir entries."""
+    paths = draw(
+        st.lists(st.lists(names, min_size=1, max_size=4), min_size=1, max_size=12)
+    )
+    members, files, seen = [], [], set()
+    for parts in paths:
+        for i in range(1, len(parts)):
+            seen.add("/".join(parts[:i]))
+        name = "/".join(parts)
+        if name in seen:
+            continue
+        seen.add(name)
+        kind = draw(st.sampled_from("fffsh" if files else "ffs"))
+        if kind == "f":
+            members.append(
+                (
+                    name,
+                    "f",
+                    draw(st.binary(max_size=2000)),
+                    draw(st.sampled_from([0o644, 0o755])),
+                )
+            )
+            files.append(name)
+        elif kind == "s":
+            members.append(
+                (name, "s", draw(st.sampled_from(["a", "../b", "/abs", "x/y"])), 0o777)
+            )
+        else:
+            # like tar, a hardlink records the mode of the inode it shares
+            target = draw(st.sampled_from(files))
+            mode = next(m[3] for m in members if m[0] == target)
+            members.append((name, "h", target, mode))
+    dirs = sorted(seen - {m[0] for m in members})
+    members += [(d, "d", None, 0o755) for d in dirs if draw(st.booleans())]
+    # no member below a non-dir, and hardlinks after their targets
+    nondirs = {m[0] for m in members if m[1] != "d"}
+    members = [m for m in members if not any(m[0].startswith(n + "/") for n in nondirs)]
+    placed, result = set(), []
+    for m in draw(st.permutations(members)):
+        if m[1] == "h" and m[2] not in placed:
+            continue
+        placed.add(m[0])
+        result.append(m)
+    return result, draw(st.sampled_from([tarfile.GNU_FORMAT, tarfile.PAX_FORMAT]))
+
+
+@needs_symlinks
+@hypothesis.settings(
+    max_examples=int(os.environ.get("UNTAR_EXAMPLES", "300")),
+    deadline=None,
+    suppress_health_check=list(hypothesis.HealthCheck),
+)
+@hypothesis.given(archive=archives())
+def test_property_same_tree_as_tarfile(tmp_path_factory, archive):
+    members, format = archive
+    tmp = tmp_path_factory.mktemp("p")
+    data = make_tar(members, format)
+    tarfile_extract(data, tmp / "expected")
+    untar.extract(io.BytesIO(data), str(tmp / "actual"))
+    assert snapshot(tmp / "actual") == snapshot(tmp / "expected")