awstorage 0.1.0__py3-none-any.whl

awstorage/__init__.py

@@ -0,0 +1,97 @@
+"""awstorage -- every drive on every node, indexed, classified and diffed.
+
+Scan a root, get a SNAPSHOT: every directory tree down to a bounded depth with
+its bytes, file count and newest mtime, plus the largest files. Classify each
+tree (cache / build-temp / model-weights / dataset / backup / repo / logs /
+service-state / media / unknown) and mark whether it is RE-FETCHABLE. Store
+snapshots in a catalog and DIFF them, so "what grew since last week" is a
+query rather than a memory. Turn the classification into PROPOSALS under a
+written policy, and APPLY only what the policy pre-approves or a human
+approved -- dry-run by default, ledgered, and never outside a declared root.
+
+    import awstorage
+    from pathlib import Path
+
+    snap = awstorage.scan(Path("E:/"), max_depth=3, time_budget_s=120)
+    awstorage.classify_snapshot(snap)                # heuristics, offline
+    cat = awstorage.Catalog(Path("inventory.db"))
+    sid = cat.put_snapshot(snap)
+    for row in awstorage.rank(snap)[:20]:
+        print(row["bytes"], row["path"], row["cls"], row["refetchable"])
+    props = awstorage.propose(snap, awstorage.default_policy())
+    awstorage.apply(props[0], roots=[Path("E:/")], dry_run=True)
+
+Three rules it exists to enforce:
+
+**Measure before you delete.** A `du` answers one question once. A snapshot
+answers the same question every week, and the diff between two is the
+question you actually had ("what filled the disk?").
+
+**Re-fetchable is a property, not a guess.** A tree is re-fetchable when a
+known producer can recreate it (a package cache, a build output, a model
+weight that lives on a mirror). The classifier records WHY it decided that,
+and whether a heuristic or a model decided -- so a wrong call is auditable.
+
+**Deleting is a policy, not a mood.** `apply` refuses anything outside the
+declared roots, anything whose fingerprint changed since the scan, and any
+class the policy does not pre-approve unless a human approved that proposal.
+It writes a ledger row either way. A dry run is the default.
+
+The package is stdlib-only and speaks to nothing. Fleet integration (a scanner
+per node, a catalog behind an API, a GUI, an autonomous steward, model-backed
+classification) lives in the platform that imports it -- awstorage works alone.
+"""
+
+from __future__ import annotations
+
+from ._fs import SCHEMA_VERSION, ScanError, scan
+from .catalog import Catalog
+from .classify import (
+    CLASSES,
+    Classifier,
+    HeuristicClassifier,
+    LLMClassifier,
+    classify_snapshot,
+    classify_tree,
+)
+from .diff import diff_snapshots
+from .graph import to_graph
+from .policy import (
+    ApplyRefused,
+    Proposal,
+    apply,
+    default_policy,
+    list_quarantine,
+    propose,
+    purge_quarantine,
+    revert,
+)
+from .report import rank, summarize
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "ScanError",
+    "scan",
+    "Catalog",
+    "CLASSES",
+    "Classifier",
+    "HeuristicClassifier",
+    "LLMClassifier",
+    "classify_snapshot",
+    "classify_tree",
+    "diff_snapshots",
+    "to_graph",
+    "ApplyRefused",
+    "Proposal",
+    "apply",
+    "default_policy",
+    "list_quarantine",
+    "propose",
+    "purge_quarantine",
+    "revert",
+    "rank",
+    "summarize",
+    "__version__",
+]

awstorage/_fs.py

@@ -0,0 +1,222 @@
+"""Bounded, resumable directory scanning -> a snapshot dict.
+
+A snapshot is a plain dict (JSON-serialisable) so it can cross any wire:
+
+    {"schema": 1, "node": "<hostname>", "root": "E:/", "taken_at": "...",
+     "max_depth": 3, "time_budget_s": 120, "truncated": false,
+     "trees": [ {"path": "E:/Caches", "depth": 1, "bytes": 157_000_000_000,
+                 "files": 12000, "dirs": 340, "newest_mtime": 1756000000.0,
+                 "oldest_mtime": 1700000000.0, "fingerprint": "sha1:..."} ],
+     "top_files": [ {"path": ..., "bytes": ..., "mtime": ...} ],
+     "errors": ["E:/System Volume Information: PermissionError"],
+     "elapsed_s": 41.2}
+
+`trees` holds every directory at depth <= max_depth (depth 0 is the root),
+each with the AGGREGATE of everything beneath it -- so a parent's bytes
+already include its children's. Consumers that want exclusive sizes subtract.
+
+The fingerprint is sha1 over (bytes, files, newest_mtime): cheap, and enough
+to notice that a tree changed between the scan and an apply. It is NOT a
+content hash; awstorage never reads file contents.
+
+Bounded on purpose: a scan that runs forever on a 4 TB drive is one nobody
+runs twice. `time_budget_s` stops the walk and marks the snapshot TRUNCATED
+(a real field, printed by every consumer) rather than returning a total that
+looks complete and is not.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import socket
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+SCHEMA_VERSION = 1
+
+# Directories that are never worth descending into: either they lie about size
+# (reparse/junction targets, /proc) or they are the OS's own business.
+DEFAULT_SKIP_NAMES = frozenset(
+    {
+        "$recycle.bin",
+        "system volume information",
+        "proc",
+        "sys",
+        "dev",
+        "run",
+    }
+)
+
+
+class ScanError(Exception):
+    """The scan could not run at all (root missing, unreadable, not a dir)."""
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _norm(p: Path) -> str:
+    # Forward slashes everywhere so the same tree has ONE spelling in the
+    # catalog whether it was scanned from Windows, WSL, or a Linux node.
+    return str(p).replace("\\", "/")
+
+
+def fingerprint(bytes_: int, files: int, newest_mtime: float) -> str:
+    h = hashlib.sha1(f"{bytes_}|{files}|{int(newest_mtime)}".encode())
+    return "sha1:" + h.hexdigest()[:16]
+
+
+def scan(
+    root: Path | str,
+    *,
+    max_depth: int = 3,
+    time_budget_s: float = 300.0,
+    top_files: int = 50,
+    skip_names: Iterable[str] = DEFAULT_SKIP_NAMES,
+    node: str | None = None,
+    follow_symlinks: bool = False,
+) -> dict:
+    """Walk `root` and return a snapshot dict. Never raises past the root check.
+
+    Per-entry errors (permission denied, vanished file) are recorded in
+    `errors` and the walk continues; only an unusable ROOT raises ScanError.
+    """
+    root_p = Path(root)
+    if not root_p.exists():
+        raise ScanError(f"root does not exist: {root}")
+    if not root_p.is_dir():
+        raise ScanError(f"root is not a directory: {root}")
+    skip = {s.lower() for s in skip_names}
+    t0 = time.monotonic()
+    deadline = t0 + float(time_budget_s)
+
+    # Aggregates keyed by the directory path (string), for dirs at depth <= max_depth.
+    agg: dict[str, dict] = {}
+    errors: list[str] = []
+    biggest: list[tuple[int, float, str]] = []  # (bytes, mtime, path) kept small
+    truncated = False
+
+    def account(dir_key: str, size: int, mtime: float, is_file: bool) -> None:
+        a = agg[dir_key]
+        a["bytes"] += size
+        if is_file:
+            a["files"] += 1
+        else:
+            a["dirs"] += 1
+        if mtime > a["newest_mtime"]:
+            a["newest_mtime"] = mtime
+        if mtime and (a["oldest_mtime"] == 0.0 or mtime < a["oldest_mtime"]):
+            a["oldest_mtime"] = mtime
+
+    def ensure(path_str: str, depth: int) -> None:
+        if path_str not in agg:
+            agg[path_str] = {
+                "path": path_str,
+                "depth": depth,
+                "bytes": 0,
+                "files": 0,
+                "dirs": 0,
+                "newest_mtime": 0.0,
+                "oldest_mtime": 0.0,
+                "git": False,
+            }
+
+    root_key = _norm(root_p)
+    ensure(root_key, 0)
+    # Stack of (dir path, depth, ancestors-at-or-below-max-depth keys)
+    stack: list[tuple[str, int, tuple[str, ...]]] = [(str(root_p), 0, (root_key,))]
+
+    while stack:
+        if time.monotonic() >= deadline:
+            truncated = True
+            break
+        d, depth, owners = stack.pop()
+        try:
+            it = os.scandir(d)
+        except OSError as exc:
+            errors.append(f"{_norm(Path(d))}: {type(exc).__name__}")
+            continue
+        with it:
+            for e in it:
+                try:
+                    if e.is_symlink() and not follow_symlinks:
+                        continue
+                    if e.is_dir(follow_symlinks=follow_symlinks):
+                        if e.name == ".git" and owners[-1] in agg:
+                            # A git working tree: its build outputs may be TRACKED
+                            # (tenant repos `git add -f` their dist/), so the policy
+                            # must not auto-delete inside one. Recorded, not decided.
+                            agg[owners[-1]]["git"] = True
+                        if e.name.lower() in skip:
+                            continue
+                        child_depth = depth + 1
+                        child_key = _norm(Path(e.path))
+                        for k in owners:
+                            account(k, 0, 0.0, is_file=False)
+                        if child_depth <= max_depth:
+                            ensure(child_key, child_depth)
+                            stack.append((e.path, child_depth, owners + (child_key,)))
+                        else:
+                            stack.append((e.path, child_depth, owners))
+                    else:
+                        st = e.stat(follow_symlinks=follow_symlinks)
+                        size = int(st.st_size)
+                        mtime = float(st.st_mtime)
+                        for k in owners:
+                            account(k, size, mtime, is_file=True)
+                        if top_files > 0:
+                            biggest.append((size, mtime, _norm(Path(e.path))))
+                            if len(biggest) > top_files * 4:
+                                biggest.sort(reverse=True)
+                                del biggest[top_files:]
+                except OSError as exc:
+                    errors.append(f"{_norm(Path(e.path))}: {type(exc).__name__}")
+
+    biggest.sort(reverse=True)
+    trees = []
+    for a in sorted(agg.values(), key=lambda x: (x["depth"], x["path"])):
+        a = dict(a)
+        a["fingerprint"] = fingerprint(a["bytes"], a["files"], a["newest_mtime"])
+        trees.append(a)
+
+    return {
+        "schema": SCHEMA_VERSION,
+        "node": node or socket.gethostname(),
+        "root": root_key,
+        "taken_at": _now_iso(),
+        "max_depth": int(max_depth),
+        "time_budget_s": float(time_budget_s),
+        "truncated": truncated,
+        "trees": trees,
+        "top_files": [
+            {"path": p, "bytes": b, "mtime": m} for b, m, p in biggest[:top_files]
+        ],
+        "errors": errors[:200],
+        "error_count": len(errors),
+        "elapsed_s": round(time.monotonic() - t0, 2),
+    }
+
+
+def exclusive_bytes(snapshot: dict) -> dict[str, int]:
+    """Bytes owned by each tree EXCLUDING its scanned children (depth-aware)."""
+    by_depth: dict[int, list[dict]] = {}
+    for t in snapshot["trees"]:
+        by_depth.setdefault(t["depth"], []).append(t)
+    children: dict[str, int] = {}
+    for depth, trees in by_depth.items():
+        if depth == 0:
+            continue
+        parents = by_depth.get(depth - 1, [])
+        for t in trees:
+            # The parent is the depth-1 tree whose path is the longest prefix of ours.
+            cand = [p["path"] for p in parents
+                    if t["path"].startswith(p["path"].rstrip("/") + "/")]
+            if cand:
+                parent = max(cand, key=len)
+                children[parent] = children.get(parent, 0) + t["bytes"]
+    return {t["path"]: max(0, t["bytes"] - children.get(t["path"], 0))
+            for t in snapshot["trees"]}

awstorage/catalog.py

@@ -0,0 +1,279 @@
+"""SQLite catalog: snapshots, trees, proposals, ledger.
+
+One file, stdlib sqlite3, WAL mode. The catalog is the thing a fleet posts
+scans INTO and a GUI reads OUT of; the package only needs it to answer "what
+did this root look like last time" and "what did we decide to do about it".
+
+Every apply writes a LEDGER row whether it acted, dry-ran, or refused. A
+storage tool that deletes without a ledger is indistinguishable from a bug.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+_DDL = """
+CREATE TABLE IF NOT EXISTS snapshots (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  node TEXT NOT NULL,
+  root TEXT NOT NULL,
+  taken_at TEXT NOT NULL,
+  schema INTEGER NOT NULL,
+  truncated INTEGER NOT NULL DEFAULT 0,
+  total_bytes INTEGER NOT NULL DEFAULT 0,
+  tree_count INTEGER NOT NULL DEFAULT 0,
+  meta TEXT NOT NULL DEFAULT '{}'
+);
+CREATE INDEX IF NOT EXISTS snapshots_node_root ON snapshots(node, root, taken_at);
+CREATE TABLE IF NOT EXISTS trees (
+  snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE,
+  path TEXT NOT NULL,
+  depth INTEGER NOT NULL,
+  bytes INTEGER NOT NULL,
+  files INTEGER NOT NULL,
+  dirs INTEGER NOT NULL,
+  newest_mtime REAL NOT NULL,
+  fingerprint TEXT NOT NULL,
+  cls TEXT,
+  refetchable INTEGER,
+  confidence REAL,
+  reason TEXT,
+  source TEXT,
+  PRIMARY KEY (snapshot_id, path)
+);
+CREATE TABLE IF NOT EXISTS top_files (
+  snapshot_id INTEGER NOT NULL REFERENCES snapshots(id) ON DELETE CASCADE,
+  path TEXT NOT NULL,
+  bytes INTEGER NOT NULL,
+  mtime REAL NOT NULL
+);
+CREATE TABLE IF NOT EXISTS proposals (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  created_at TEXT NOT NULL,
+  snapshot_id INTEGER REFERENCES snapshots(id) ON DELETE SET NULL,
+  node TEXT NOT NULL,
+  path TEXT NOT NULL,
+  action TEXT NOT NULL,
+  bytes INTEGER NOT NULL,
+  cls TEXT NOT NULL,
+  policy_rule TEXT,
+  auto INTEGER NOT NULL DEFAULT 0,
+  status TEXT NOT NULL DEFAULT 'proposed',
+  fingerprint TEXT,
+  note TEXT
+);
+CREATE TABLE IF NOT EXISTS ledger (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  at TEXT NOT NULL,
+  proposal_id INTEGER,
+  node TEXT NOT NULL,
+  path TEXT NOT NULL,
+  action TEXT NOT NULL,
+  outcome TEXT NOT NULL,
+  bytes INTEGER NOT NULL DEFAULT 0,
+  detail TEXT
+);
+"""
+
+_STATUSES = {"proposed", "approved", "rejected", "applied", "expired", "snoozed"}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+class Catalog:
+    def __init__(self, path: Path | str) -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._db = sqlite3.connect(str(self.path))
+        self._db.row_factory = sqlite3.Row
+        self._db.execute("PRAGMA journal_mode=WAL")
+        self._db.execute("PRAGMA foreign_keys=ON")
+        self._db.executescript(_DDL)
+        # v1.1: trees.git (scanner saw a .git). Additive, so an older catalog opens.
+        cols = {r[1] for r in self._db.execute("PRAGMA table_info(trees)")}
+        if "git" not in cols:
+            with self._db:
+                self._db.execute("ALTER TABLE trees ADD COLUMN git INTEGER NOT NULL DEFAULT 0")
+
+    def close(self) -> None:
+        self._db.close()
+
+    # -- snapshots -----------------------------------------------------------------
+
+    def put_snapshot(self, snap: dict) -> int:
+        trees = snap.get("trees", [])
+        root_row = next((t for t in trees if t["depth"] == 0), None)
+        total = int(root_row["bytes"]) if root_row else sum(t["bytes"] for t in trees)
+        meta = {k: snap.get(k) for k in ("max_depth", "time_budget_s", "elapsed_s",
+                                          "error_count", "classifier")}
+        with self._db:
+            cur = self._db.execute(
+                "INSERT INTO snapshots(node, root, taken_at, schema, truncated, total_bytes,"
+                " tree_count, meta) VALUES (?,?,?,?,?,?,?,?)",
+                (snap["node"], snap["root"], snap["taken_at"], int(snap["schema"]),
+                 1 if snap.get("truncated") else 0, total, len(trees), json.dumps(meta)),
+            )
+            sid = int(cur.lastrowid)
+            self._db.executemany(
+                "INSERT INTO trees(snapshot_id, path, depth, bytes, files, dirs, newest_mtime,"
+                " fingerprint, cls, refetchable, confidence, reason, source, git)"
+                " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+                [(sid, t["path"], t["depth"], t["bytes"], t["files"], t["dirs"],
+                  t["newest_mtime"], t["fingerprint"], t.get("cls"),
+                  None if t.get("refetchable") is None else int(bool(t["refetchable"])),
+                  t.get("confidence"), t.get("reason"), t.get("source"),
+                  1 if t.get("git") else 0) for t in trees],
+            )
+            self._db.executemany(
+                "INSERT INTO top_files VALUES (?,?,?,?)",
+                [(sid, f["path"], f["bytes"], f["mtime"]) for f in snap.get("top_files", [])],
+            )
+        return sid
+
+    def list_snapshots(self, node: str | None = None, root: str | None = None,
+                       limit: int = 50) -> list[dict]:
+        q = "SELECT * FROM snapshots"
+        cond, args = [], []
+        if node:
+            cond.append("node = ?")
+            args.append(node)
+        if root:
+            cond.append("root = ?")
+            args.append(root)
+        if cond:
+            q += " WHERE " + " AND ".join(cond)
+        q += " ORDER BY taken_at DESC, id DESC LIMIT ?"
+        args.append(int(limit))
+        return [dict(r) for r in self._db.execute(q, args)]
+
+    def get_snapshot(self, sid: int) -> dict | None:
+        row = self._db.execute("SELECT * FROM snapshots WHERE id = ?", (sid,)).fetchone()
+        if not row:
+            return None
+        snap = dict(row)
+        snap["meta"] = json.loads(snap.get("meta") or "{}")
+        snap["truncated"] = bool(snap["truncated"])
+        snap["trees"] = []
+        for t in self._db.execute(
+            "SELECT * FROM trees WHERE snapshot_id = ? ORDER BY depth, path", (sid,)
+        ):
+            d = dict(t)
+            d.pop("snapshot_id", None)
+            if d.get("refetchable") is not None:
+                d["refetchable"] = bool(d["refetchable"])
+            d["git"] = bool(d.get("git"))
+            snap["trees"].append(d)
+        snap["top_files"] = [
+            {"path": r["path"], "bytes": r["bytes"], "mtime": r["mtime"]}
+            for r in self._db.execute(
+                "SELECT path, bytes, mtime FROM top_files WHERE snapshot_id = ?"
+                " ORDER BY bytes DESC", (sid,))
+        ]
+        return snap
+
+    def latest_pair(self, node: str, root: str) -> tuple[dict | None, dict | None]:
+        """(newest, previous) snapshots for a node+root -- the diff's usual inputs."""
+        rows = self.list_snapshots(node=node, root=root, limit=2)
+        newest = self.get_snapshot(rows[0]["id"]) if rows else None
+        prev = self.get_snapshot(rows[1]["id"]) if len(rows) > 1 else None
+        return newest, prev
+
+    def drop_snapshot(self, sid: int) -> None:
+        with self._db:
+            self._db.execute("DELETE FROM snapshots WHERE id = ?", (sid,))
+
+    # -- proposals / ledger --------------------------------------------------------
+
+    def put_proposals(self, proposals: list) -> list[int]:
+        ids = []
+        with self._db:
+            for p in proposals:
+                d = p if isinstance(p, dict) else p.__dict__
+                cur = self._db.execute(
+                    "INSERT INTO proposals(created_at, snapshot_id, node, path, action, bytes,"
+                    " cls, policy_rule, auto, status, fingerprint, note)"
+                    " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+                    (_now(), d.get("snapshot_id"), d["node"], d["path"], d["action"],
+                     int(d["bytes"]), d["cls"], d.get("policy_rule"),
+                     1 if d.get("auto") else 0, d.get("status", "proposed"),
+                     d.get("fingerprint"), d.get("note")),
+                )
+                ids.append(int(cur.lastrowid))
+        return ids
+
+    def list_proposals(self, status: str | None = None, node: str | None = None,
+                       limit: int = 200) -> list[dict]:
+        q = "SELECT * FROM proposals"
+        cond, args = [], []
+        if status:
+            cond.append("status = ?")
+            args.append(status)
+        if node:
+            cond.append("node = ?")
+            args.append(node)
+        if cond:
+            q += " WHERE " + " AND ".join(cond)
+        q += " ORDER BY bytes DESC, id DESC LIMIT ?"
+        args.append(int(limit))
+        out = []
+        for r in self._db.execute(q, args):
+            d = dict(r)
+            d["auto"] = bool(d["auto"])
+            out.append(d)
+        return out
+
+    def get_proposal(self, pid: int) -> dict | None:
+        r = self._db.execute("SELECT * FROM proposals WHERE id = ?", (pid,)).fetchone()
+        if not r:
+            return None
+        d = dict(r)
+        d["auto"] = bool(d["auto"])
+        return d
+
+    def set_status(self, pid: int, status: str, note: str | None = None) -> None:
+        if status not in _STATUSES:
+            raise ValueError(f"unknown proposal status {status!r}")
+        with self._db:
+            if note is None:
+                self._db.execute("UPDATE proposals SET status = ? WHERE id = ?", (status, pid))
+            else:
+                self._db.execute("UPDATE proposals SET status = ?, note = ? WHERE id = ?",
+                                 (status, note, pid))
+
+    def ledger(self, *, proposal_id: int | None, node: str, path: str, action: str,
+               outcome: str, bytes_: int = 0, detail: str | None = None) -> int:
+        with self._db:
+            cur = self._db.execute(
+                "INSERT INTO ledger(at, proposal_id, node, path, action, outcome, bytes, detail)"
+                " VALUES (?,?,?,?,?,?,?,?)",
+                (_now(), proposal_id, node, path, action, outcome, int(bytes_), detail),
+            )
+        return int(cur.lastrowid)
+
+    def list_ledger(self, limit: int = 200) -> list[dict]:
+        return [dict(r) for r in self._db.execute(
+            "SELECT * FROM ledger ORDER BY id DESC LIMIT ?", (int(limit),))]
+
+    def totals(self) -> dict:
+        """Cross-node roll-up for a dashboard: newest snapshot per (node, root)."""
+        rows = self._db.execute(
+            "SELECT s.* FROM snapshots s JOIN ("
+            "  SELECT node, root, MAX(taken_at) AS t FROM snapshots GROUP BY node, root"
+            ") m ON s.node = m.node AND s.root = m.root AND s.taken_at = m.t"
+            " ORDER BY s.node, s.root"
+        ).fetchall()
+        out = {"roots": [], "total_bytes": 0, "nodes": set()}
+        for r in rows:
+            d = dict(r)
+            d["truncated"] = bool(d["truncated"])
+            out["roots"].append({k: d[k] for k in ("id", "node", "root", "taken_at",
+                                                     "total_bytes", "truncated")})
+            out["total_bytes"] += int(d["total_bytes"])
+            out["nodes"].add(d["node"])
+        out["nodes"] = sorted(out["nodes"])
+        return out