scitex-storage 0.2.0__py3-none-any.whl

scitex_storage/__init__.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""scitex-storage: research-data storage triage.
+
+Current scope (see README for the full roadmap): a read-only, stat-only
+directory-tree ``scan`` (delegates its walk to ``fd``) that inventories
+the biggest space and inode (file-count) consumers per top-level child of
+a root; ``find_duplicates`` — an explicitly opt-in verb that DOES read
+file contents (delegated to ``fclones``) to report exact-duplicate
+groups; ``images prune`` — rotation for a directory of versioned files
+(e.g. dated SIF builds) that never removes a file any symlink in the
+directory currently references; ``sweep`` — tars an inode-hog directory
+in place (compute-node-only, explicit per-directory confirm required);
+and ``archive``/``restore`` — move-not-delete tiering to nas/nas2 over
+ssh, verified before the local copy is removed, with a manifest
+``restore`` reads back. Every mutating command defaults to a dry-run.
+
+Public names — including ``__version__`` — are exposed via PEP 562
+``__getattr__`` (lazy, on first access) so ``import scitex_storage`` itself
+stays cheap — every CLI invocation and every shell tab-completion pays this
+cost (see the python-api skill's "PEP 562 module __getattr__" section).
+``__version__`` in particular defers even the ``import importlib.metadata``
+statement itself, not just the ``version()`` call — that stdlib import
+alone measured ~280ms in this environment (dist-metadata scanning cost,
+not this package's own code), so doing it eagerly at module top-level
+would defeat the whole point of the lazy design.
+"""
+
+from __future__ import annotations
+
+# Public-name -> source-submodule map. ONE row per public symbol.
+_LAZY_ATTRS: dict[str, str] = {
+    "ChildUsage": "_scan",
+    "MissingSystemDependencyError": "_scan",
+    "RootScan": "_scan",
+    "scan": "_scan",
+    "scan_roots": "_scan",
+    "find_duplicates": "_duplicates",
+    "ApplyResult": "_images",
+    "PruneCandidate": "_images",
+    "PrunePlan": "_images",
+    "SkippedInUse": "_images",
+    "apply_prune": "_images",
+    "plan_prune": "_images",
+    "SweepCandidate": "_sweep",
+    "SweepPlan": "_sweep",
+    "SweepResult": "_sweep",
+    "SweptCandidate": "_sweep",
+    "SweptEntry": "_sweep",
+    "apply_sweep": "_sweep",
+    "plan_sweep": "_sweep",
+    "sweep_status": "_sweep",
+    "ArchiveManifest": "_archive",
+    "ArchivePlan": "_archive",
+    "RestorePlan": "_archive",
+    "apply_archive": "_archive",
+    "apply_restore": "_archive",
+    "plan_archive": "_archive",
+    "plan_restore": "_archive",
+}
+
+
+def __getattr__(name: str):
+    """PEP 562 lazy-loader: import on first access, cache, return."""
+    if name == "__version__":
+        from importlib.metadata import PackageNotFoundError
+        from importlib.metadata import version as _get_version
+
+        try:
+            value = _get_version("scitex-storage")
+        except PackageNotFoundError:  # source checkout without an installed dist
+            value = "0.0.0+local"  # PEP 440 local version segment
+        globals()["__version__"] = value
+        return value
+
+    mod_name = _LAZY_ATTRS.get(name)
+    if mod_name is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    from importlib import import_module
+
+    attr = getattr(import_module(f".{mod_name}", __name__), name)
+    globals()[name] = attr  # cache; subsequent access skips this branch
+    return attr
+
+
+def __dir__() -> list[str]:
+    return sorted(set(_LAZY_ATTRS) | set(globals()))
+
+
+# Literal (not computed) so PA-101's static AST check can see it. Keep in
+# sync with the _LAZY_ATTRS keys above.
+__all__ = [
+    "ChildUsage",
+    "MissingSystemDependencyError",
+    "RootScan",
+    "scan",
+    "scan_roots",
+    "find_duplicates",
+    "ApplyResult",
+    "PruneCandidate",
+    "PrunePlan",
+    "SkippedInUse",
+    "apply_prune",
+    "plan_prune",
+    "SweepCandidate",
+    "SweepPlan",
+    "SweepResult",
+    "SweptCandidate",
+    "SweptEntry",
+    "apply_sweep",
+    "plan_sweep",
+    "sweep_status",
+    "ArchiveManifest",
+    "ArchivePlan",
+    "RestorePlan",
+    "apply_archive",
+    "apply_restore",
+    "plan_archive",
+    "plan_restore",
+    "__version__",
+]
+
+# EOF

scitex_storage/__main__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""``python -m scitex_storage`` entry point — forwards to the CLI."""
+
+from __future__ import annotations
+
+from ._cli import main
+
+if __name__ == "__main__":
+    main()
+
+# EOF

scitex_storage/_archive.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Move-not-delete tiering: archive a directory to nas/nas2, with a manifest + restore.
+
+The MVP verb behind ``scitex-storage archive`` / ``restore``. Built on
+scitex-ssh's ``sync_dir`` rsync-over-ssh primitive (per scitex-ssh's own
+design: it stays transport-only and policy-free — copy-verify-then-remove
+tiering logic belongs here, not in the transport layer).
+
+Design constraints:
+
+* **Copy-verify-then-remove, never delete-then-copy.** :func:`apply_archive`
+  only removes the local source after ``sync_dir`` reports success; a
+  failed sync leaves the source completely untouched and raises loud.
+* **The remote parent must exist before rsync does.** On a destination
+  that's never held archived data before, ``~/scitex-storage-archive/...``
+  simply doesn't exist yet — rsync then fails with "errors selecting
+  input/output files (code 3)". :func:`apply_archive` runs
+  ``mkdir -p`` on the remote parent via ``exec_remote`` before ``sync_dir``.
+  Deliberately NOT ``rsync --mkpath`` (added in rsync 3.2.3; a real nas2
+  target still runs 3.0.7, where ``--mkpath`` is a hard unknown-option
+  error) — ``mkdir -p`` works on any rsync version. Found + verified by
+  scitex-ssh smoke-testing a real archive against real nas/nas2, not a
+  hypothetical (2026-07-11).
+* **Never wrap a leading ``~`` in shell quotes.** Every remote path this
+  module builds a shell command from (the ``mkdir -p`` above, ``rm -rf``
+  in :func:`apply_restore`) goes through :func:`_quote_remote_path`, which
+  leaves a leading ``~/`` unquoted and only quotes the rest — a bare
+  ``shlex.quote(path)`` turns ``~`` into a literal character (tilde-
+  expansion only applies to an unquoted leading ``~``), silently creating
+  a directory named ``~`` instead of resolving ``$HOME``. Also found by
+  scitex-ssh's real-nas2 smoke test: the stray ``~`` directory was sitting
+  in the remote home directory after the naive-quoting version "succeeded"
+  (2026-07-11).
+* **Trailing slash controls "contents of" vs "the directory itself".**
+  rsync's classic gotcha: a source arg with NO trailing slash copies the
+  source directory itself into the destination (nesting one level:
+  ``dest/<source-basename>/...``); WITH a trailing slash it copies the
+  source's *contents* directly into the destination. Both
+  :func:`apply_archive` (push) and :func:`apply_restore` (pull) want
+  "contents of," so the copy SIDE (``local`` for push, ``remote_path`` for
+  pull) always gets a trailing slash appended before being handed to
+  ``sync_dir`` -- otherwise a restore lands two directories deep
+  (``source/<remote-basename>/<source-basename>/...``) with byte-correct
+  but wrongly-placed data, easy to miss if only checksums are checked.
+  scitex-ssh's own docstring is explicit that trailing-slash semantics are
+  the caller's to control; this is that layer. Found by scitex-ssh
+  smoke-testing a real archive+restore round-trip and diffing the actual
+  file paths, not just checksums (2026-07-11).
+* **Manifest before delete.** The manifest is written *before* the local
+  source is removed, so a crash between "sync succeeded" and "manifest
+  written" fails safe (source still present; re-running `archive` is a
+  cheap no-op rsync).
+* **Restore never destroys the archive by default.** :func:`apply_restore`
+  pulls the data back to its original local path; the remote copy is only
+  removed if the caller explicitly opts in via ``delete_remote=True``.
+* **Aliases, not addresses.** ``destination`` is one of the ``~/.ssh/config``
+  aliases ``nas`` / ``nas2`` — scitex-ssh already handles mux discipline per
+  alias; this module never opens its own ssh connections.
+"""
+
+from __future__ import annotations
+
+import json
+import posixpath
+import shlex
+import shutil
+import time
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from scitex_ssh import SSHResult, exec_remote, sync_dir
+
+from ._scan import _measure_dir
+
+DESTINATIONS: tuple[str, ...] = ("nas", "nas2")
+DEFAULT_REMOTE_ROOT = "~/scitex-storage-archive"
+
+# Remote paths this dangerous are never a legitimate archive/restore target
+# -- a real path always has more structure than this after flattening a
+# real local absolute path under DEFAULT_REMOTE_ROOT.
+_UNSAFE_REMOTE_PATHS = {"", "/", "~", "."}
+
+
+def _quote_remote_path(path: str) -> str:
+    """shlex.quote ``path`` for a remote shell without breaking a leading
+    ``~`` home-dir shortcut.
+
+    Shell tilde-expansion only applies to an UNQUOTED (or quote-adjacent)
+    leading ``~`` -- naively wrapping the whole string in single quotes (a
+    bare ``shlex.quote(path)``) turns ``~`` into a literal character, so
+    ``mkdir -p '~/scitex-storage-archive/x'`` creates a bogus directory
+    literally named ``~`` instead of expanding to ``$HOME``. Confirmed via
+    scitex-ssh smoke-testing a real ``mkdir`` on nas2 (2026-07-11) -- the
+    stray ``~`` dir was found sitting in the remote home directory.
+    """
+    if path == "~":
+        return "~"
+    if path.startswith("~/"):
+        return "~/" + shlex.quote(path[2:])
+    return shlex.quote(path)
+
+
+def _as_dir_contents(path: str) -> str:
+    """Append a trailing ``/`` (idempotent) so rsync copies ``path``'s
+    CONTENTS into the destination rather than nesting ``path`` itself one
+    level deeper as a subdirectory -- see the module docstring's trailing-
+    slash bullet."""
+    return path.rstrip("/") + "/"
+
+
+def _manifest_dir() -> Path:
+    """Resolved fresh on every call (not a module-level constant) so tests
+    can sandbox it by setting ``$HOME`` before calling in — matching the
+    pattern already used for ``scan``'s default roots."""
+    return Path("~/.scitex/scitex-storage/runtime/archive-manifests").expanduser()
+
+
+def _default_remote_path(source: Path, remote_root: str = DEFAULT_REMOTE_ROOT) -> str:
+    """Mirror ``source``'s absolute path under ``remote_root``, flattened."""
+    flattened = str(source).lstrip("/")
+    return f"{remote_root.rstrip('/')}/{flattened}"
+
+
+def _manifest_path(source: Path) -> Path:
+    """Deterministic manifest filename for ``source`` (so restore can find it)."""
+    flattened = str(source).lstrip("/").replace("/", "--")
+    return _manifest_dir() / f"{flattened}.json"
+
+
+@dataclass
+class ArchivePlan:
+    """The result of :func:`plan_archive` — never touches the network."""
+
+    source: Path
+    destination: str
+    remote_path: str
+    size_bytes: int
+    file_count: int
+    manifest_path: Path
+
+
+def plan_archive(
+    source: str | Path,
+    destination: str,
+    remote_path: str | None = None,
+) -> ArchivePlan:
+    """Compute (never execute) an archive plan for ``source`` -> ``destination``.
+
+    Read-only: stats ``source`` via the same walk :func:`~scitex_storage._scan.scan`
+    uses, never touches the network. Fail-loud on a missing / non-directory
+    ``source`` or an unrecognised ``destination``.
+    """
+    if destination not in DESTINATIONS:
+        raise ValueError(
+            f"destination must be one of {DESTINATIONS}, got {destination!r}"
+        )
+    source = Path(source).expanduser()
+    if not source.exists():
+        raise FileNotFoundError(f"path does not exist: {source}")
+    if not source.is_dir():
+        raise NotADirectoryError(f"not a directory: {source}")
+    source = source.resolve()
+
+    size, file_count, _newest_mtime, _err = _measure_dir(source)
+    remote = remote_path or _default_remote_path(source)
+    if remote.strip() in _UNSAFE_REMOTE_PATHS:
+        raise ValueError(f"refusing an unsafe remote path: {remote!r}")
+
+    return ArchivePlan(
+        source=source,
+        destination=destination,
+        remote_path=remote,
+        size_bytes=size,
+        file_count=file_count,
+        manifest_path=_manifest_path(source),
+    )
+
+
+@dataclass
+class ArchiveManifest:
+    """Persisted record of one completed archive — the source of truth for restore."""
+
+    source: str
+    destination: str
+    remote_path: str
+    size_bytes: int
+    file_count: int
+    checksummed: bool
+    archived_at: float
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> ArchiveManifest:
+        return cls(**data)
+
+
+def apply_archive(
+    plan: ArchivePlan,
+    *,
+    checksum: bool = True,
+    exclude: tuple[str, ...] = (),
+    runner=None,
+) -> ArchiveManifest:
+    """Execute an archive plan: push, verify, write manifest, THEN remove source.
+
+    ``checksum`` (default on, since this precedes an irreversible local
+    delete) adds rsync's ``--checksum`` flag — reads every byte on both
+    sides instead of rsync's fast mtime+size quick-check. A non-zero rsync
+    exit raises immediately; the source is left completely untouched and no
+    manifest is written. ``runner`` is passed straight through to both
+    ``exec_remote`` (the mkdir) and ``sync_dir`` (a ``subprocess.run``-
+    shaped invoker) — the same real-transport-free test seam scitex-ssh
+    itself exposes.
+    """
+    remote_parent = posixpath.dirname(plan.remote_path)
+    if remote_parent and remote_parent not in _UNSAFE_REMOTE_PATHS:
+        mkdir_result = exec_remote(
+            plan.destination,
+            f"mkdir -p -- {_quote_remote_path(remote_parent)}",
+            runner=runner,
+        )
+        if not mkdir_result.success:
+            raise RuntimeError(
+                f"failed to create remote parent directory "
+                f"{plan.destination}:{remote_parent} "
+                f"(exit {mkdir_result.returncode}) -- source left untouched.\n"
+                f"--- stdout ---\n{mkdir_result.stdout}\n"
+                f"--- stderr ---\n{mkdir_result.stderr}"
+            )
+
+    extra_opts: tuple[str, ...] = ("--checksum",) if checksum else ()
+    result: SSHResult = sync_dir(
+        plan.destination,
+        _as_dir_contents(str(plan.source)),
+        plan.remote_path,
+        direction="push",
+        exclude=exclude,
+        extra_opts=extra_opts,
+        runner=runner,
+    )
+    if not result.success:
+        raise RuntimeError(
+            f"archive sync to {plan.destination}:{plan.remote_path} failed "
+            f"(exit {result.returncode}) -- source left untouched.\n"
+            f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}"
+        )
+
+    manifest = ArchiveManifest(
+        source=str(plan.source),
+        destination=plan.destination,
+        remote_path=plan.remote_path,
+        size_bytes=plan.size_bytes,
+        file_count=plan.file_count,
+        checksummed=checksum,
+        archived_at=time.time(),
+    )
+    plan.manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    plan.manifest_path.write_text(json.dumps(manifest.to_dict(), indent=2))
+
+    shutil.rmtree(plan.source)
+    return manifest
+
+
+@dataclass
+class RestorePlan:
+    """The result of :func:`plan_restore` — never touches the network."""
+
+    manifest: ArchiveManifest
+    manifest_path: Path
+
+
+def plan_restore(source: str | Path) -> RestorePlan:
+    """Load the manifest for ``source`` — read-only, never touches the network.
+
+    ``source`` need not currently exist (it typically doesn't — archiving
+    removed it). Fail-loud if no manifest was ever written for this path.
+    """
+    resolved = Path(source).expanduser().resolve()
+    manifest_path = _manifest_path(resolved)
+    if not manifest_path.exists():
+        raise FileNotFoundError(
+            f"no archive manifest found for {resolved} at {manifest_path} "
+            "-- was this directory ever archived from here?"
+        )
+    manifest = ArchiveManifest.from_dict(json.loads(manifest_path.read_text()))
+    return RestorePlan(manifest=manifest, manifest_path=manifest_path)
+
+
+def apply_restore(
+    plan: RestorePlan, *, delete_remote: bool = False, runner=None
+) -> Path:
+    """Pull the archived directory back to its original local path.
+
+    Verifies via rsync's own exit code; raises loud on failure (nothing is
+    removed remotely in that case, regardless of ``delete_remote``). The
+    remote copy is only removed when ``delete_remote=True`` — off by
+    default, since restoring locally should not destroy the backup unless
+    explicitly asked. ``runner`` is passed straight through to both
+    ``sync_dir`` and ``exec_remote``.
+    """
+    manifest = plan.manifest
+    source = Path(manifest.source)
+    result: SSHResult = sync_dir(
+        manifest.destination,
+        str(source),
+        _as_dir_contents(manifest.remote_path),
+        direction="pull",
+        runner=runner,
+    )
+    if not result.success:
+        raise RuntimeError(
+            f"restore pull from {manifest.destination}:{manifest.remote_path} "
+            f"failed (exit {result.returncode}) -- remote copy untouched.\n"
+            f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}"
+        )
+
+    if delete_remote:
+        if manifest.remote_path.strip() in _UNSAFE_REMOTE_PATHS:
+            raise ValueError(
+                f"refusing to delete an unsafe remote path: {manifest.remote_path!r}"
+            )
+        rm_result = exec_remote(
+            manifest.destination,
+            f"rm -rf -- {_quote_remote_path(manifest.remote_path)}",
+            runner=runner,
+        )
+        if not rm_result.success:
+            raise RuntimeError(
+                f"local restore succeeded, but removing the remote copy at "
+                f"{manifest.destination}:{manifest.remote_path} failed "
+                f"(exit {rm_result.returncode}) -- remote copy still present.\n"
+                f"--- stdout ---\n{rm_result.stdout}\n--- stderr ---\n{rm_result.stderr}"
+            )
+
+    return source
+
+
+# EOF

scitex_storage/_cli/__init__.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""scitex-storage CLI — thin orchestrator; each verb lives in its own submodule."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from .. import __version__
+from ._archive_cmd import archive_cmd, restore_cmd
+from ._compat import spec_group_kwargs
+from ._duplicates_cmd import find_duplicates_cmd
+from ._images_cmd import images_group
+from ._introspect import list_python_apis
+from ._mcp_commands import mcp
+from ._scan_cmd import scan_cmd
+from ._sweep_cmd import sweep_cmd, sweep_status_cmd
+
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
+
+
+def _print_command_help(cmd, prefix: str, parent_ctx) -> None:
+    click.echo(f"\n{'=' * 50}")
+    click.echo(prefix)
+    click.echo("=" * 50)
+    sub_ctx = click.Context(cmd, info_name=prefix.split()[-1], parent=parent_ctx)
+    click.echo(cmd.get_help(sub_ctx))
+    if isinstance(cmd, click.Group):
+        for sub_name, sub_cmd in sorted(cmd.commands.items()):
+            _print_command_help(sub_cmd, f"{prefix} {sub_name}", sub_ctx)
+
+
+@click.group(
+    invoke_without_command=True,
+    context_settings=CONTEXT_SETTINGS,
+    **spec_group_kwargs(
+        summary="Research-data storage triage (scan + duplicates + versioned-image rotation).",
+        description=(
+            "Discovery + rotation layers of a planned storage-tiering tool "
+            "(local SSD -> NAS SSD -> NAS HDD -> offline). `scan` is a "
+            "read-only, stat-only directory walk (via `fd`) that reports "
+            "the biggest space and inode (file-count) consumers per "
+            "top-level child of a root -- no file contents are ever read. "
+            "`find-duplicates` is a separate, explicitly opt-in verb (via "
+            "`fclones`) that DOES read file contents (to hash them) to "
+            "report exact-duplicate groups. `images prune` rotates a "
+            "directory of versioned files (e.g. dated SIF builds), always "
+            "excluding files any symlink in the directory currently "
+            "references. `sweep` tars an inode-hog directory in place "
+            "(many small files -> one tar, one inode), compute-node-only, "
+            "gated on an explicit per-directory confirm. `archive` moves a "
+            "directory to nas/nas2 over ssh (scitex-ssh's sync_dir), "
+            "verifying before removing the local copy and writing a "
+            "manifest `restore` reads back. Every mutating command "
+            "defaults to a dry-run.",
+        ),
+        config_resolution=(
+            "scitex-storage has no configurable state yet — every command "
+            "takes its options on the command line. Runtime state written: "
+            "archive manifests under "
+            "~/.scitex/scitex-storage/runtime/archive-manifests/, one JSON "
+            "file per archived directory, read back by `restore`.",
+        ),
+        version_of="scitex-storage",
+        command_categories=(
+            (
+                "Storage",
+                (
+                    "scan",
+                    "find-duplicates",
+                    "images",
+                    "sweep",
+                    "sweep-status",
+                    "archive",
+                    "restore",
+                ),
+            ),
+            ("Introspection", ("list-python-apis", "mcp")),
+        ),
+    ),
+)
+@click.version_option(
+    __version__,
+    "-V",
+    "--version",
+    prog_name="scitex-storage",
+    message="%(prog)s %(version)s",
+)
+@click.option("--help-recursive", is_flag=True, help="Show help for all commands.")
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    help="Emit structured JSON output (propagates to subcommands that honour it).",
+)
+@click.pass_context
+def main(ctx: click.Context, help_recursive: bool, as_json: bool) -> None:
+    ctx.ensure_object(dict)
+    ctx.obj["as_json"] = as_json
+
+    if help_recursive:
+        click.echo(f"scitex-storage {__version__}")
+        click.echo(main.get_help(ctx))
+        for name, cmd in sorted(main.commands.items()):
+            _print_command_help(cmd, f"scitex-storage {name}", ctx)
+        ctx.exit(0)
+
+    if ctx.invoked_subcommand is None:
+        click.echo(ctx.get_help())
+
+
+main.add_command(scan_cmd)
+main.add_command(find_duplicates_cmd)
+main.add_command(images_group)
+main.add_command(sweep_cmd)
+main.add_command(sweep_status_cmd)
+main.add_command(archive_cmd)
+main.add_command(restore_cmd)
+main.add_command(list_python_apis)
+main.add_command(mcp)
+
+# §1a: install-shell-completion + print-shell-completion (canonical leaves).
+try:
+    from scitex_dev._cli._completion import attach_shell_completion
+
+    attach_shell_completion(main, prog_name="scitex-storage")
+except ImportError:
+    pass
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())
+
+# EOF