ec2-byhand 0.1.0__py3-none-any.whl

diskread/__init__.py

@@ -0,0 +1,23 @@
+"""diskread: pure-Python, read-only parsers for disks, partitions, LVM and filesystems.
+
+This package must not depend on AWS or on ec2_byhand, so it can be tested
+against local image files and split out later.
+
+diskread: ディスク・パーティション・LVM・ファイルシステムを読み取り専用で解析する純 Python の
+パーサー群。AWS にも ec2_byhand にも依存しない（ローカルのイメージファイルでテストでき、将来切り
+出せるように）。
+"""
+
+from __future__ import annotations
+
+from diskread.blockdev import BlockDevice, BytesDevice, FileDevice, SliceDevice
+from diskread.errors import DiskReadError, UnsupportedError
+
+__all__ = [
+    "BlockDevice",
+    "BytesDevice",
+    "DiskReadError",
+    "FileDevice",
+    "SliceDevice",
+    "UnsupportedError",
+]

diskread/blockdev.py

@@ -0,0 +1,187 @@
+"""The "disk" abstraction every parser reads through.
+
+A block device is just a fixed-size byte range with random access. Image
+files, partitions, LVM logical volumes and EBS snapshots all implement it,
+so parsers never care where the bytes come from.
+
+全パーサーが読み取りに使う「ディスク」の抽象。ブロックデバイスはランダムアクセスできる固定長の
+バイト列にすぎず、イメージファイル・パーティション・LVM の論理ボリューム・EBS スナップショット
+のどれも同じ形で扱える。
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from collections.abc import Iterator
+from pathlib import Path
+from typing import BinaryIO, Protocol, runtime_checkable
+
+from diskread.errors import DiskReadError
+
+
+@runtime_checkable
+class BlockDevice(Protocol):
+    """The read-only, random-access byte range every parser consumes.
+
+    全パーサーが読む、読み取り専用でランダムアクセスできるバイト範囲。
+    """
+
+    @property
+    def size(self) -> int:
+        """Total size in bytes.
+
+        全体のサイズ（バイト）。
+        """
+        ...
+
+    def read_at(self, offset: int, length: int) -> bytes:
+        """Read exactly `length` bytes at `offset`.
+
+        Raises DiskReadError if the range extends past the end of the device.
+
+        `offset` から `length` バイトちょうどを読む。範囲がデバイスの端を越えると
+        DiskReadError。
+        """
+        ...
+
+
+@runtime_checkable
+class ReadAhead(Protocol):
+    """A block device that can start fetching what it is told will be read next.
+
+    次に読まれると告げられた範囲を、先に取り寄せ始められるブロックデバイス。
+    """
+
+    def will_read(self, offset: int, length: int) -> None:
+        """Note that [offset, offset + length) is to be read, after what was noted before.
+
+        [offset, offset + length) を、これまでに告げた範囲の後に読む予定だと伝える。
+        """
+        ...
+
+
+def will_read(dev: BlockDevice, offset: int = 0, length: int | None = None) -> None:
+    """Tell `dev` that [offset, offset + length) (by default, to the end) is to be read
+    next, after the ranges told before, so that a device that reads over a network can
+    fetch it ahead. Only a hint: nothing is read here, devices that cannot fetch ahead
+    ignore it, and it never fails: a range the device does not have (damaged metadata
+    that points past its end) is left to the read, which meets the error and reports
+    it. Tell only what will be read, since what a device fetches ahead and nobody reads
+    is fetched for nothing.
+
+    `dev` に [offset, offset + length)（既定は末尾まで）を、先に告げた範囲の次に読むと伝え、
+    ネットワーク越しに読むデバイスが先読みできるようにする。ヒントにすぎず、ここでは何も読まな
+    い。先読みできないデバイスは無視する。失敗もしない: デバイスに無い範囲（壊れたメタデータが
+    末尾の先を指している）は読み出しに任せ、そこでエラーになって報告される。先読みして誰も読ま
+    なければ無駄になるので、本当に読む範囲だけを告げる。
+    """
+    if isinstance(dev, ReadAhead):
+        with contextlib.suppress(DiskReadError):
+            dev.will_read(offset, dev.size - offset if length is None else length)
+
+
+def check_range(dev_size: int, offset: int, length: int) -> None:
+    """Raise DiskReadError unless [offset, offset + length) fits in dev_size.
+
+    [offset, offset + length) が dev_size に収まらなければ DiskReadError。
+    """
+    if offset < 0 or length < 0 or offset + length > dev_size:
+        raise DiskReadError(f"read out of range: offset={offset} length={length} size={dev_size}")
+
+
+def read_chunks(dev: BlockDevice, chunk_size: int = 1 << 20) -> Iterator[bytes]:
+    """The device's bytes in order, a chunk at a time, for files too large to hold in memory.
+
+    デバイスのバイト列を先頭から少しずつ返す（メモリに載せきれない大きなファイルのため）。
+    """
+    for offset in range(0, dev.size, chunk_size):
+        yield dev.read_at(offset, min(chunk_size, dev.size - offset))
+
+
+class FileDevice:
+    """A block device backed by a local image file.
+
+    ローカルのイメージファイルを背にしたブロックデバイス。
+    """
+
+    def __init__(self, path: str | os.PathLike[str]) -> None:
+        # Stays open until close() (or __exit__), so no `with` here.
+        self._f: BinaryIO = Path(path).open("rb")  # noqa: SIM115
+        self._size = os.fstat(self._f.fileno()).st_size
+
+    @property
+    def size(self) -> int:
+        return self._size
+
+    def read_at(self, offset: int, length: int) -> bytes:
+        check_range(self._size, offset, length)
+        self._f.seek(offset)
+        data = self._f.read(length)
+        if len(data) != length:
+            raise DiskReadError(f"short read at offset {offset}")
+        return data
+
+    def close(self) -> None:
+        """Close the underlying image file.
+
+        下のイメージファイルを閉じる。
+        """
+        self._f.close()
+
+    def __enter__(self) -> FileDevice:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+class BytesDevice:
+    """A block device held in memory. Mostly useful for tests.
+
+    メモリ上のブロックデバイス。主にテスト用。
+    """
+
+    def __init__(self, data: bytes | bytearray) -> None:
+        self._data = bytes(data)
+
+    @property
+    def size(self) -> int:
+        return len(self._data)
+
+    def read_at(self, offset: int, length: int) -> bytes:
+        check_range(len(self._data), offset, length)
+        return self._data[offset : offset + length]
+
+
+class SliceDevice:
+    """A sub-range of another block device, e.g. a partition.
+
+    別のブロックデバイスの部分範囲。パーティションなど。
+    """
+
+    def __init__(self, parent: BlockDevice, offset: int, size: int) -> None:
+        check_range(parent.size, offset, size)
+        self._parent = parent
+        self._offset = offset
+        self._size = size
+
+    @property
+    def size(self) -> int:
+        return self._size
+
+    @property
+    def offset(self) -> int:
+        """Start of this slice within the parent device, in bytes.
+
+        親デバイス内でのこのスライスの開始位置（バイト）。
+        """
+        return self._offset
+
+    def read_at(self, offset: int, length: int) -> bytes:
+        check_range(self._size, offset, length)
+        return self._parent.read_at(self._offset + offset, length)
+
+    def will_read(self, offset: int, length: int) -> None:
+        check_range(self._size, offset, length)  # not what lies beside it on the parent
+        will_read(self._parent, self._offset + offset, length)

diskread/crc32c.py

@@ -0,0 +1,72 @@
+"""CRC-32C (Castagnoli), as used by ext4 metadata_csum and XFS v5.
+
+`crc32c_raw` is the bare register update with no pre/post inversion, like
+the kernel's crc32c_le(): ext4 chains it and stores the raw value. XFS
+starts from ~0 and stores the inverted result, which is the standard
+CRC-32C that `crc32c` returns.
+
+Pure Python, so the inner loop matters: this is slicing-by-8 (8 lookup
+tables, one struct-decoded 8-byte step per iteration), several times
+faster than the classic one-table byte loop.
+
+ext4 metadata_csum と XFS v5 が使う CRC-32C（Castagnoli）。`crc32c_raw` は前後の反転なしの素の
+更新（カーネルの crc32c_le() 相当）で、`crc32c` は標準の CRC-32C。純 Python なので内側のループ
+が要で、slicing-by-8 により古典的な 1 テーブル方式の数倍速い。
+"""
+
+from __future__ import annotations
+
+import struct
+
+_POLY = 0x82F63B78  # reflected 0x1EDC6F41
+
+
+def _tables() -> list[list[int]]:
+    """Build the eight slicing-by-8 lookup tables."""
+    t0 = []
+    for i in range(256):
+        c = i
+        for _ in range(8):
+            c = (c >> 1) ^ _POLY if c & 1 else c >> 1
+        t0.append(c)
+    tables = [t0]
+    for _ in range(7):
+        prev = tables[-1]
+        tables.append([t0[c & 0xFF] ^ (c >> 8) for c in prev])
+    return tables
+
+
+_T0, _T1, _T2, _T3, _T4, _T5, _T6, _T7 = _tables()
+
+
+def crc32c_raw(crc: int, data: bytes) -> int:
+    """Advance the bare CRC register over `data` (no pre/post inversion).
+
+    前後の反転なしの素の CRC レジスタを `data` のぶん進める。
+    """
+    t0, t1, t2, t3 = _T0, _T1, _T2, _T3
+    t4, t5, t6, t7 = _T4, _T5, _T6, _T7
+    head = len(data) & ~7
+    for lo, hi in struct.iter_unpack("<II", data[:head]):
+        lo ^= crc
+        crc = (
+            t7[lo & 0xFF]
+            ^ t6[(lo >> 8) & 0xFF]
+            ^ t5[(lo >> 16) & 0xFF]
+            ^ t4[lo >> 24]
+            ^ t3[hi & 0xFF]
+            ^ t2[(hi >> 8) & 0xFF]
+            ^ t1[(hi >> 16) & 0xFF]
+            ^ t0[hi >> 24]
+        )
+    for b in data[head:]:
+        crc = t0[(crc ^ b) & 0xFF] ^ (crc >> 8)
+    return crc
+
+
+def crc32c(data: bytes) -> int:
+    """The standard CRC-32C of `data`.
+
+    `data` の標準の CRC-32C。
+    """
+    return crc32c_raw(0xFFFFFFFF, data) ^ 0xFFFFFFFF

diskread/errors.py

@@ -0,0 +1,50 @@
+"""Errors every diskread parser raises.
+
+diskread の全パーサーが投げるエラー。
+"""
+
+from __future__ import annotations
+
+import struct
+from collections.abc import Callable
+from functools import wraps
+
+# What parsing truncated or nonsensical bytes raises when no explicit check
+# caught it first: short struct.unpack_from, indexing past the end, bad UTF-8,
+# dividing by a size that reads as zero (ArithmeticError, with OverflowError)...
+MALFORMED_DATA_ERRORS = (struct.error, LookupError, ValueError, ArithmeticError, RecursionError)
+
+
+class DiskReadError(Exception):
+    """The on-disk data is malformed, truncated, or fails a checksum.
+
+    ディスク上のデータが壊れている・切れている・チェックサムが合わない。
+    """
+
+
+class UnsupportedError(DiskReadError):
+    """The on-disk data is valid but uses a feature diskread does not handle.
+
+    データ自体は正しいが、diskread が扱わない機能を使っている。
+    """
+
+
+def malformed_as_disk_read_error[**P, R](func: Callable[P, R]) -> Callable[P, R]:
+    """Make `func`, which parses on-disk bytes, report malformed data as DiskReadError.
+
+    Callers only handle DiskReadError, so that one bad inode costs one
+    warning. Every check a parser could miss must not become a traceback.
+
+    ディスク上のバイト列を解析する `func` が、壊れたデータを DiskReadError として報告するように
+    する。呼び出し側は DiskReadError だけを扱うので、inode が 1 つ壊れていても警告 1 件で済む。
+    パーサーの検査漏れがトレースバックにならないようにするためのもの。
+    """
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        try:
+            return func(*args, **kwargs)
+        except MALFORMED_DATA_ERRORS as err:
+            raise DiskReadError(f"malformed on-disk data: {err!r}") from err
+
+    return wrapper

diskread/fs/__init__.py

@@ -0,0 +1,328 @@
+"""Filesystem-independent types and helpers shared by the ext4, XFS and FAT readers.
+
+各ファイルシステムのリーダーが共有する、ファイルシステム非依存の型とヘルパー。
+"""
+
+from __future__ import annotations
+
+import contextlib
+import stat as _stat
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Protocol, runtime_checkable
+
+from diskread.blockdev import BlockDevice
+from diskread.errors import DiskReadError, malformed_as_disk_read_error
+
+
+class FileType(StrEnum):
+    """An inode's type as a short lowercase name ("reg", "dir", ...).
+
+    inode の種別（「reg」「dir」などの短い小文字名）。
+    """
+
+    REG = "reg"
+    DIR = "dir"
+    LNK = "lnk"
+    CHR = "chr"
+    BLK = "blk"
+    FIFO = "fifo"
+    SOCK = "sock"
+
+
+_TYPES = {
+    _stat.S_IFREG: FileType.REG,
+    _stat.S_IFDIR: FileType.DIR,
+    _stat.S_IFLNK: FileType.LNK,
+    _stat.S_IFCHR: FileType.CHR,
+    _stat.S_IFBLK: FileType.BLK,
+    _stat.S_IFIFO: FileType.FIFO,
+    _stat.S_IFSOCK: FileType.SOCK,
+}
+
+
+def file_type(mode: int) -> FileType:
+    """The FileType in `mode`'s S_IFMT bits; DiskReadError if unknown.
+
+    `mode` の S_IFMT ビットが表す FileType。未知なら DiskReadError。
+    """
+    try:
+        return _TYPES[_stat.S_IFMT(mode)]
+    except KeyError:
+        raise DiskReadError(f"unknown file type in mode {mode:#o}") from None
+
+
+@dataclass(frozen=True, slots=True)  # slots: one is kept for every path that is walked
+class Stat:
+    """What stat(2) would report for an inode, as Linux computes it.
+
+    その inode について stat(2) が返す内容（Linux の計算に合わせる）。
+    """
+
+    ino: int
+    mode: int  # including the S_IFMT type bits
+    uid: int
+    gid: int
+    size: int
+    nlink: int
+    blocks: int  # in 512-byte units, like st_blocks
+    atime_ns: int
+    mtime_ns: int
+    ctime_ns: int
+    btime_ns: int | None  # None when the filesystem does not record it
+    rdev: tuple[int, int] | None  # (major, minor) for device nodes
+    _type: FileType | None = field(default=None, init=False, repr=False, compare=False)
+
+    @property
+    def type(self) -> FileType:
+        """The FileType encoded in `mode`. Asked for on every path step of every lookup:
+        worked out once and kept.
+
+        `mode` が表す FileType。パス解決の各段で毎回聞かれるので、1 度求めて覚えておく。
+        """
+        found = self._type
+        if found is None:
+            found = file_type(self.mode)
+            object.__setattr__(self, "_type", found)  # frozen: this is only a memo
+        return found
+
+
+class Filesystem(Protocol):
+    """What every filesystem reader provides; walk() and lookup() build on it.
+
+    各ファイルシステムリーダーが提供する読み取り口。walk() と lookup() はこの上に載る。
+    """
+
+    fs_type: str  # "ext4" (also for ext2 / ext3), "xfs" or "vfat", as detect_fs names them
+
+    @property
+    def is_dirty(self) -> bool:
+        """Whether it may hold changes only in its journal or log, which are not replayed.
+
+        変更がジャーナルやログにしか無い可能性があるか（それらは再生しない）。
+        """
+        ...
+
+    @property
+    def root_ino(self) -> int:
+        """The root directory's inode number.
+
+        ルートディレクトリの inode 番号。
+        """
+        ...
+
+    def stat(self, ino: int) -> Stat:
+        """What stat(2) would report for the inode.
+
+        その inode の stat(2) 相当の情報。
+        """
+        ...
+
+    def listdir(self, ino: int) -> list[tuple[bytes, int]]:
+        """Directory entries as (name, inode), without "." and "..".
+
+        ディレクトリの (名前, inode) の一覧。「.」と「..」は含まない。
+        """
+        ...
+
+    def open(self, ino: int) -> BlockDevice:
+        """The contents of a regular file (holes read as zeros).
+
+        通常ファイルの内容（穴はゼロとして読める）。
+        """
+        ...
+
+    def readlink(self, ino: int) -> bytes:
+        """The symlink's target, as raw bytes.
+
+        シンボリックリンクのリンク先（生のバイト列）。
+        """
+        ...
+
+    def xattrs(self, ino: int) -> dict[str, bytes]:
+        """Extended attributes as getxattr(2) would return them.
+
+        getxattr(2) が返す形の拡張属性。
+        """
+        ...
+
+
+def reads_disk_metadata[T](cls: type[T]) -> type[T]:
+    """Class decorator for a filesystem reader: whatever its Filesystem methods
+    make of malformed metadata comes out as DiskReadError.
+
+    ファイルシステムリーダーのクラスデコレータ: Filesystem のメソッドが壊れたメタデータで何を起
+    こしても DiskReadError として出す。
+    """
+    # will_list too, where a reader has it: it parses the same metadata as listdir.
+    for name in ("stat", "listdir", "open", "readlink", "xattrs", "will_list"):
+        if hasattr(cls, name):
+            setattr(cls, name, malformed_as_disk_read_error(getattr(cls, name)))
+    return cls
+
+
+def valid_name(name: bytes) -> bool:
+    """Whether a directory entry's name is one a path can hold: not empty, without "/"
+    or NUL, and not "." or "..". A disk can hold any bytes; a name that is none would
+    lead a path somewhere else than under its directory.
+
+    ディレクトリエントリの名前が、パスに入れられるものか: 空でなく、「/」も NUL も含まず、
+    「.」「..」でもない。ディスクにはどんなバイト列もあり得て、名前でないものを繋ぐと、パスが
+    そのディレクトリの下ではない場所を指してしまう。
+    """
+    return bool(name) and b"/" not in name and b"\0" not in name and name not in (b".", b"..")
+
+
+INODE_CACHE = 1 << 18  # inodes a filesystem keeps the bytes of (262,144)
+PARSED_INODES = 1 << 13  # and those it keeps parsed too (8,192): the ones in use
+
+
+@runtime_checkable
+class ListsAhead(Protocol):
+    """A filesystem that can tell its disk where a directory's entries are.
+
+    ディレクトリの中身がどこにあるかをディスクに告げられるファイルシステム。
+    """
+
+    def will_list(self, ino: int) -> None:
+        """Tell the disk (will_read) the blocks listdir(ino) will read, nothing more.
+
+        listdir(ino) が読むブロックだけをディスクに告げる（will_read）。
+        """
+        ...
+
+
+def will_list(fs: Filesystem, ino: int) -> None:
+    """Tell the disk under `fs` where directory `ino`'s entries are, when it can say:
+    a hint only, nothing is read but the directory's own metadata, and an error is
+    left to listdir, which meets it again.
+
+    `fs` の下のディスクに、ディレクトリ `ino` の中身の位置を告げる（告げられる場合）。ヒント
+    にすぎず、読むのはディレクトリ自身のメタデータだけ。エラーは listdir に任せる。
+    """
+    if isinstance(fs, ListsAhead):
+        with contextlib.suppress(DiskReadError):
+            fs.will_list(ino)
+
+
+def walk(
+    fs: Filesystem,
+    on_error: Callable[[bytes, DiskReadError], None] | None = None,
+    prune: Callable[[bytes], bool] | None = None,
+    listdir: Callable[[int], list[tuple[bytes, int]]] | None = None,
+    stat: Callable[[int], Stat] | None = None,
+    tell: Callable[[int], None] | None = None,
+) -> Iterator[tuple[bytes, int, Stat]]:
+    """Yield (path, inode, stat) for every reachable file, root first, a level of
+    directories at a time.
+
+    Paths are absolute byte strings ("/", "/etc", "/etc/passwd"). Each
+    directory is entered once even if corrupt metadata creates a cycle.
+
+    By default the first DiskReadError aborts the walk. With `on_error`,
+    the error is reported with the path it concerns and that entry (or
+    that directory's contents) is skipped, so one bad inode in a snapshot
+    of a live system does not hide the rest of the tree.
+
+    Entries whose path satisfies `prune` are neither yielded nor entered
+    (used to hide what lies under a mountpoint). `listdir` and `stat`
+    replace the filesystem's own calls, so a caller can route them through
+    its caches (as SystemSource.walk does, to avoid re-reading the same
+    metadata when it resolves paths after the walk).
+
+    Breadth first: every directory of a level is told to the disk (`tell`, by
+    default will_list) before any is listed, so that a disk read over a network
+    fetches them all at once. Depth first, the next directory is known only once
+    the one before is read, and each waited for its blocks (nine tenths of what an
+    analysis over the EBS direct APIs waited for). With `listdir` given, `tell` is
+    the caller's: only it knows which directories its listdir will read.
+
+    到達できる全ファイルを (path, inode, stat) で、ルートから順に yield する。パスは絶対のバイ
+    ト列。既定では最初の DiskReadError で中断し、`on_error` を渡すと該当パスを添えて報告してそ
+    の項目（またはそのディレクトリの中身）だけを飛ばす。`prune` に合うパスは出さず、中にも入ら
+    ない。`listdir` と `stat` を渡すとファイルシステム直呼びの代わりに使う（SystemSource.walk
+    が自分のキャッシュ経由にして、walk 後のパス解決で同じメタデータを読み直さないため）。
+    """
+
+    def fail(path: bytes, err: DiskReadError) -> None:
+        if on_error is None:
+            raise err
+        on_error(path, err)
+
+    if tell is None and listdir is None:
+        tell = lambda ino: will_list(fs, ino)  # noqa: E731
+    listdir = fs.listdir if listdir is None else listdir
+    stat = fs.stat if stat is None else stat
+    root = fs.root_ino
+    try:
+        root_stat = stat(root)
+    except DiskReadError as err:  # a volume whose root is damaged: nothing of it, said once
+        fail(b"/", err)
+        return
+    yield b"/", root, root_stat
+    seen_dirs = {root}
+    level: list[tuple[bytes, int]] = [(b"", root)]
+    while level:
+        if tell is not None:
+            for _, dir_ino in level:
+                tell(dir_ino)
+        below: list[tuple[bytes, int]] = []
+        for prefix, dir_ino in level:
+            try:
+                entries = listdir(dir_ino)
+            except DiskReadError as err:
+                fail(prefix or b"/", err)
+                continue
+            for name, ino in entries:
+                path = prefix + b"/" + name
+                if not valid_name(name):
+                    fail(path, DiskReadError(f"invalid name {name!r} in directory inode {dir_ino}"))
+                    continue
+                if prune is not None and prune(path):
+                    continue
+                try:
+                    st = stat(ino)
+                except DiskReadError as err:
+                    fail(path, err)
+                    continue
+                if st.type is FileType.DIR and ino in seen_dirs:
+                    fail(path, DiskReadError(f"directory inode {ino} reached twice"))
+                    continue
+                yield path, ino, st
+                if st.type is FileType.DIR:
+                    seen_dirs.add(ino)
+                    below.append((path, ino))
+        level = below
+
+
+def lookup(fs: Filesystem, path: bytes | str) -> int:
+    """Resolve an absolute path to an inode without following symlinks.
+
+    絶対パスを、シンボリックリンクを辿らずに inode に解決する。
+    """
+    if isinstance(path, str):
+        path = path.encode("utf-8", "surrogateescape")
+    ino = fs.root_ino
+    for part in path.split(b"/"):
+        if not part:
+            continue
+        for name, child in fs.listdir(ino):
+            if name == part:
+                ino = child
+                break
+        else:
+            raise FileNotFoundError(path)
+    return ino
+
+
+__all__ = [
+    "FileType",
+    "Filesystem",
+    "Stat",
+    "file_type",
+    "lookup",
+    "reads_disk_metadata",
+    "valid_name",
+    "walk",
+]