filetx 0.1.0__py3-none-any.whl

filetx/__init__.py

@@ -0,0 +1,53 @@
+"""Transactional filesystem operations.
+
+Batch moves, copies and deletes that either all commit or all roll back::
+
+    from filetx import Transaction
+
+    with Transaction(journal="release.jsonl") as tx:
+        tx.mkdir("dist", exist_ok=True)
+        tx.move("build/app.bin", "dist/app.bin")
+        tx.delete("build", missing_ok=True)
+
+If any step fails, every completed step is undone and the tree is left as it
+was found. If the process is killed part-way through, ``filetx undo
+release.jsonl`` finishes the rollback from the journal.
+
+See :class:`Transaction` for the API, :mod:`filetx.ops` for how reversibility
+is implemented, and :mod:`filetx.journal` for the on-disk format.
+"""
+
+from .errors import (
+    FiletxError,
+    JournalError,
+    OperationError,
+    RollbackError,
+    TransactionError,
+)
+from .journal import JOURNAL_VERSION, Journal, JournalFile, read_journal
+from .ops import STAGE_PREFIX, Copy, Delete, Mkdir, Move, Operation
+from .transaction import Recovery, Transaction, recover
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "JOURNAL_VERSION",
+    "STAGE_PREFIX",
+    "Copy",
+    "Delete",
+    "FiletxError",
+    "Journal",
+    "JournalError",
+    "JournalFile",
+    "Mkdir",
+    "Move",
+    "Operation",
+    "OperationError",
+    "Recovery",
+    "RollbackError",
+    "Transaction",
+    "TransactionError",
+    "__version__",
+    "read_journal",
+    "recover",
+]

filetx/cli.py

@@ -0,0 +1,109 @@
+"""Command-line interface.
+
+The library is the product; the CLI exists for the one job you cannot do from
+inside the crashed process -- finishing an interrupted transaction after the
+fact::
+
+    filetx inspect release.jsonl    # what was this transaction doing?
+    filetx undo release.jsonl       # put it back
+
+Exit codes: ``0`` success, ``1`` recovery failed or the journal is unusable,
+``2`` bad arguments (argparse's own convention).
+"""
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from . import __version__
+from .errors import FiletxError, RollbackError
+from .journal import read_journal
+from .transaction import recover
+
+__all__ = ["main"]
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="filetx",
+        description="Inspect and recover filetx transaction journals.",
+    )
+    parser.add_argument("--version", action="version", version=f"filetx {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    undo = sub.add_parser("undo", help="finish an interrupted transaction from its journal")
+    undo.add_argument("journal", help="path to the journal file")
+    undo.add_argument(
+        "--no-fsync",
+        action="store_true",
+        help="skip fsync when settling the journal (faster, less durable)",
+    )
+
+    inspect = sub.add_parser("inspect", help="show a journal's plan and status")
+    inspect.add_argument("journal", help="path to the journal file")
+
+    return parser
+
+
+def _cmd_inspect(args: argparse.Namespace) -> int:
+    journal = read_journal(args.journal)
+    status = journal.terminal or "INTERRUPTED"
+    print(f"transaction {journal.txid}  journal v{journal.version}  status: {status}")
+    print(f"{len(journal.plan)} operation(s) planned:")
+    for index, op in enumerate(journal.plan):
+        if index in journal.completed:
+            mark = "done"
+        elif index in journal.attempted:
+            mark = "PARTIAL"
+        else:
+            mark = "-"
+        print(f"  [{index:>3}] {mark:<8} {op.describe()}")
+    if not journal.settled:
+        print("\nThis transaction never settled. Run `filetx undo` to roll it back.")
+    return 0
+
+
+def _cmd_undo(args: argparse.Namespace) -> int:
+    result = recover(args.journal, fsync=not args.no_fsync)
+
+    if result.action == "rolled-back":
+        print(f"rolled back transaction {result.txid}: {len(result.reverted)} operation(s) undone")
+    elif result.action == "purged":
+        print(f"transaction {result.txid} had committed; cleaned up {len(result.purged)} entry(s)")
+    else:
+        print(f"transaction {result.txid} was already settled; nothing to do")
+
+    if result.leftover:
+        print(
+            f"\n{len(result.leftover)} staged entry(s) were kept rather than deleted, because "
+            f"their original paths are occupied again and these may be the only copies left:"
+        )
+        for path in result.leftover:
+            print(f"  {path}")
+        print("Review them, then remove them by hand.")
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point. Returns the process exit code."""
+    args = _build_parser().parse_args(argv)
+    handlers = {"inspect": _cmd_inspect, "undo": _cmd_undo}
+    try:
+        return handlers[args.command](args)
+    except RollbackError as exc:
+        print(f"filetx: {exc}", file=sys.stderr)
+        for index, failure in exc.failures:
+            print(f"  operation {index}: {failure}", file=sys.stderr)
+        print(
+            "The journal was left unsettled. Fix the underlying problem and run "
+            "`filetx undo` again.",
+            file=sys.stderr,
+        )
+        return 1
+    except FiletxError as exc:
+        print(f"filetx: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

filetx/errors.py

@@ -0,0 +1,72 @@
+"""Exception hierarchy.
+
+The distinction that matters is :class:`TransactionError` versus
+:class:`RollbackError`. The first means "your change did not happen, and the
+tree is as you left it" -- recoverable, and the caller can retry. The second
+means "your change did not happen and undoing it *also* failed" -- the tree is
+in an indeterminate state and a human has to look at it. Collapsing those two
+into one exception type would be the single most dangerous thing this library
+could do, so they are separate and ``RollbackError`` carries the failures.
+"""
+
+__all__ = [
+    "FiletxError",
+    "JournalError",
+    "OperationError",
+    "RollbackError",
+    "TransactionError",
+]
+
+
+class FiletxError(Exception):
+    """Base class for every error raised by filetx."""
+
+
+class OperationError(FiletxError):
+    """A single operation could not be applied.
+
+    Raised during :meth:`filetx.Transaction.commit` and normally wrapped in a
+    :class:`TransactionError` after the rollback has completed.
+    """
+
+
+class TransactionError(FiletxError):
+    """A commit failed and was rolled back successfully.
+
+    The filesystem is back in its pre-commit state. The underlying cause is
+    available as ``__cause__`` and as :attr:`cause`.
+    """
+
+    def __init__(self, message: str, *, index: int, cause: BaseException) -> None:
+        super().__init__(message)
+        self.index = index
+        """Position in the plan of the operation that failed."""
+        self.cause = cause
+        """The exception that caused the failure."""
+
+
+class RollbackError(FiletxError):
+    """A commit failed *and* the rollback failed.
+
+    The filesystem is in an indeterminate state: some operations were undone
+    and some were not. :attr:`failures` lists what could not be undone, and
+    :attr:`journal` points at the journal file to hand to ``filetx undo``
+    once the underlying problem is fixed.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        failures: list[tuple[int, BaseException]],
+        journal: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.failures = failures
+        """``(plan index, exception)`` for each operation that could not be undone."""
+        self.journal = journal
+        """Path to the journal file, if the transaction was journalled."""
+
+
+class JournalError(FiletxError):
+    """A journal file is missing, unreadable, or written by an incompatible version."""

filetx/journal.py

@@ -0,0 +1,195 @@
+"""Write-ahead journal.
+
+The journal is what makes a crash survivable. Each record is one JSON object
+on its own line, flushed and ``fsync``-ed *before* the filesystem change it
+describes is attempted. The journal on disk is therefore never behind reality
+-- at worst it records an operation as attempted that did not in fact happen,
+which is the safe direction to be wrong in, because every ``revert`` checks
+the real filesystem before acting.
+
+Record sequence for a transaction::
+
+    {"event": "begin",  "version": 1, "tx": "...", "plan": [...]}
+    {"event": "apply",  "index": 0, "undo": {...}}
+    {"event": "done",   "index": 0}
+    {"event": "apply",  "index": 1, "undo": {...}}
+    {"event": "fail",   "index": 1, "error": "..."}
+    {"event": "rollback"}
+
+A journal whose last record is ``commit`` or ``rollback`` is *settled*.
+Anything else is an interrupted transaction that :func:`filetx.recover` can
+finish, using the undo records already on disk.
+
+JSON Lines rather than a binary format or SQLite: an operator staring at a
+half-finished batch can read it with ``cat``, and appending one short line is
+already atomic enough for records this size.
+"""
+
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import TracebackType
+from typing import Any
+
+from .errors import JournalError
+from .ops import Operation, UndoRecord, operation_from_dict
+
+__all__ = ["JOURNAL_VERSION", "Journal", "JournalFile", "read_journal"]
+
+JOURNAL_VERSION = 1
+"""Bumped only on a breaking change to the record format."""
+
+SETTLED = ("commit", "rollback")
+
+
+class Journal:
+    """Append-only writer for one transaction's journal.
+
+    Args:
+        path: File to append to. Created if missing.
+        fsync: Force each record to stable storage before returning. Correct
+            by default, and substantially slower than buffered writes. Turn it
+            off only when the journal is a debugging aid rather than a
+            recovery mechanism.
+    """
+
+    def __init__(self, path: str | os.PathLike[str], *, fsync: bool = True) -> None:
+        self.path = Path(path)
+        self.fsync = fsync
+        try:
+            self._fh = self.path.open("a", encoding="utf-8", newline="\n")
+        except OSError as exc:
+            raise JournalError(f"cannot open journal {self.path}: {exc}") from exc
+
+    def __enter__(self) -> "Journal":
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Close the underlying file. Safe to call more than once."""
+        if not self._fh.closed:
+            self._fh.close()
+
+    def _write(self, record: dict[str, Any]) -> None:
+        record["ts"] = time.time()
+        self._fh.write(json.dumps(record, sort_keys=True) + "\n")
+        self._fh.flush()
+        if self.fsync:
+            os.fsync(self._fh.fileno())
+
+    def begin(self, txid: str, plan: list[Operation]) -> None:
+        """Record the full plan before any of it is attempted."""
+        self._write(
+            {
+                "event": "begin",
+                "version": JOURNAL_VERSION,
+                "tx": txid,
+                "plan": [op.to_dict() for op in plan],
+            }
+        )
+
+    def apply(self, index: int, undo: UndoRecord) -> None:
+        """Record how to undo an operation, before it is attempted."""
+        self._write({"event": "apply", "index": index, "undo": undo})
+
+    def done(self, index: int) -> None:
+        """Record that an operation completed."""
+        self._write({"event": "done", "index": index})
+
+    def fail(self, index: int, error: str) -> None:
+        """Record that an operation failed."""
+        self._write({"event": "fail", "index": index, "error": error})
+
+    def commit(self) -> None:
+        """Settle the journal: the transaction is permanent."""
+        self._write({"event": "commit"})
+
+    def rollback(self) -> None:
+        """Settle the journal: the transaction was undone."""
+        self._write({"event": "rollback"})
+
+
+@dataclass
+class JournalFile:
+    """A parsed journal."""
+
+    path: Path
+    txid: str
+    version: int
+    plan: list[Operation]
+    attempted: dict[int, UndoRecord] = field(default_factory=dict)
+    """Undo records for every operation that was started, keyed by plan index.
+    An operation is in here whether or not it finished -- that is the point."""
+    completed: set[int] = field(default_factory=set)
+    """Operations that ran to completion."""
+    terminal: str | None = None
+    """``"commit"``, ``"rollback"``, or ``None`` if the transaction never settled."""
+
+    @property
+    def settled(self) -> bool:
+        """True if the transaction reached a definite end."""
+        return self.terminal in SETTLED
+
+
+def read_journal(path: str | os.PathLike[str]) -> JournalFile:
+    """Parse a journal file.
+
+    Raises:
+        JournalError: The file is missing, malformed, has no ``begin`` record,
+            or was written by an incompatible journal version.
+    """
+    path = Path(path)
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise JournalError(f"cannot read journal {path}: {exc}") from exc
+
+    records: list[dict[str, Any]] = []
+    for lineno, line in enumerate(lines, start=1):
+        if not line.strip():
+            continue
+        try:
+            records.append(json.loads(line))
+        except json.JSONDecodeError as exc:
+            # A torn last line means the process died mid-write. Everything
+            # before it is still valid, so keep that and stop.
+            if lineno == len(lines):
+                break
+            raise JournalError(f"{path}:{lineno}: malformed journal record: {exc}") from exc
+
+    if not records or records[0].get("event") != "begin":
+        raise JournalError(f"{path}: no begin record -- not a filetx journal")
+
+    begin = records[0]
+    version = begin.get("version")
+    if version != JOURNAL_VERSION:
+        raise JournalError(
+            f"{path}: journal version {version!r}, this filetx understands {JOURNAL_VERSION}"
+        )
+
+    journal = JournalFile(
+        path=path,
+        txid=begin.get("tx", ""),
+        version=version,
+        plan=[operation_from_dict(d) for d in begin.get("plan", [])],
+    )
+
+    for record in records[1:]:
+        event = record.get("event")
+        if event == "apply":
+            journal.attempted[record["index"]] = record.get("undo", {})
+        elif event == "done":
+            journal.completed.add(record["index"])
+        elif event in SETTLED:
+            journal.terminal = event
+
+    return journal