alissa-tools-github-revloop 0.31.24__tar.gz → 0.31.26__tar.gz

{alissa_tools_github_revloop-0.31.24/src/main/alissa_tools_github_revloop.egg-info → alissa_tools_github_revloop-0.31.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alissa-tools-github-revloop
-Version: 0.31.24
+Version: 0.31.26
 Summary: ALISSA-TOOLS-GITHUB-REVLOOP
 Home-page: https://alissa.app
 Author: Fahera

{alissa_tools_github_revloop-0.31.24 → alissa_tools_github_revloop-0.31.26}/src/main/alissa/tools/github/revloop/janitor.py

@@ -117,6 +117,34 @@ HOLD_SKEW_SECONDS = 5 * 60
 # round window, and one WARNING names what holds the most bytes.
 BACKSTOP_PCT = 95.0
 
+# How long a held folder's cached size is trusted (issue #187). The cache is
+# keyed on the folder's newest mtime, which only sees the folder and its
+# direct children: a write deeper down (an `npm ci` filling `node_modules`)
+# leaves it unchanged. So an entry is re-walked once it is this old, mtime
+# or not.
+SIZE_CACHE_SECONDS = 60 * 60
+
+
+@dataclass
+class HeldWalk:
+    """A held folder's size walk that ran out of budget (issue #189), kept
+    so the next sweep resumes it instead of starting over: the folder's
+    newest mtime when the walk began, the sweep `now` it began at, the bytes
+    counted so far, and the directories still to read. A resumed walk's
+    total may count a directory removed since it was read; it never counts a
+    directory twice."""
+
+    mtime: float
+    started: float
+    total: int
+    stack: "list[str]"
+
+
+# One held folder's cached size, by path: either a finished walk, as
+# (newest mtime, bytes, the sweep `now` the walk began at), or a `HeldWalk`
+# that the time budget cut short (issue #189).
+SizeCache = dict[Path, "tuple[float, int, float] | HeldWalk"]
+
 # The two reasons a hub-wide hold gives, so the sweep summary groups them.
 HUB_HOLD_BOUNDED = "a live round in this hub names no review task and spawned before this checkout"
 HUB_HOLD_UNBOUNDED = "a live round in this hub names no review task or spawn time"
@@ -259,6 +287,13 @@ class SweepResult:
     # checkouts only it released from a hub-wide hold.
     held_bytes: "dict[str, int]" = field(default_factory=dict)
     held_unsized: int = 0
+    # Issue #187: held folders whose size came from the cache, not a walk
+    # this sweep, so the summary can say its figure may be up to
+    # SIZE_CACHE_SECONDS old.
+    held_cached: int = 0
+    # Issue #189: unsized held folders whose walk is kept to resume next
+    # sweep (a subset of `held_unsized`).
+    held_resuming: int = 0
     backstop: bool = False
     backstop_released: int = 0
 
@@ -294,13 +329,20 @@ HELD_REASONS_LOGGED = 10
 def held_summary(result: SweepResult) -> str:
     """The sweep summary's held-bytes tail (issue #185): every hub/reason
     holding bytes, largest first, at most HELD_REASONS_LOGGED of them, and
-    how many held folders went unsized."""
+    how many held folders were sized from the cache or went unsized."""
     tail = held_reasons(result.top_held(HELD_REASONS_LOGGED))
     rest = len(result.held_bytes) - HELD_REASONS_LOGGED
     if rest > 0:
         tail += f", +{rest} more reason(s)"
+    if result.held_cached:
+        tail += (
+            f"; {result.held_cached} held size(s) cached, up to "
+            f"{SIZE_CACHE_SECONDS // 60} min old"
+        )
     if result.held_unsized:
         tail += f"; {result.held_unsized} held folder(s) not sized this sweep"
+        if result.held_resuming:
+            tail += f" ({result.held_resuming} walk(s) resume next sweep)"
     return tail
 
 
@@ -415,9 +457,35 @@ def dir_bytes(path: "Path | str") -> int:
     """Bytes allocated under `path` (st_blocks, the figure `du` reports and a
     removal frees), following no symlinks. Iterative, and every per-entry
     error is skipped -- devloop's `dir_size` walk, counting blocks."""
-    total = 0
-    stack = [os.fspath(path)]
+    return _walk_bytes(path, None) or 0
+
+
+def held_bytes_of(
+    path: "Path | str",
+    stop: "Callable[[], bool]",
+    progress: "HeldWalk | None" = None,
+) -> "int | None":
+    """`dir_bytes` for a held folder (issue #187): `stop` is asked before
+    each directory is read, and the walk gives up -- None, never a partial
+    figure -- the moment it answers True, so one large tree cannot run the
+    sweep past its time budget. With `progress` (issue #189) the walk starts
+    from its `total` and `stack` rather than from `path`, and an abandoned
+    walk leaves its place there, so a later call carries on from it."""
+    return _walk_bytes(path, stop, progress)
+
+
+def _walk_bytes(
+    path: "Path | str",
+    stop: "Callable[[], bool] | None",
+    progress: "HeldWalk | None" = None,
+) -> "int | None":
+    total = progress.total if progress is not None else 0
+    stack = progress.stack if progress is not None else [os.fspath(path)]
     while stack:
+        if stop is not None and stop():
+            if progress is not None:
+                progress.total = total  # `stack` is progress.stack, in place
+            return None
         current = stack.pop()
         try:
             with os.scandir(current) as entries:
@@ -528,7 +596,7 @@ def sweep(
     pressure_grace_seconds: "float | None" = None,
     under_pressure: "Callable[[], bool] | None" = None,
     stale_round_seconds: "float | None" = None,
-    sizes: "dict[Path, tuple[float, int]] | None" = None,
+    sizes: "SizeCache | None" = None,
 ) -> SweepResult:
     """Remove every candidate under `hubs` older than `grace_seconds` that
     `guard` does not hold, oldest first, then prune each hub's worktree
@@ -559,10 +627,18 @@ def sweep(
     checkout older than that window: no live round can own a checkout older
     than the stale window. `backstop_released` counts what only it freed.
 
-    Every held folder is sized into `held_bytes` per hub and reason. `sizes`
-    (path -> (mtime, bytes)), when given, caches those walks across sweeps
-    and is re-keyed to this sweep's held folders; sizing stops at the time
-    budget too, and what was left unsized is counted in `held_unsized`."""
+    Every held folder is sized into `held_bytes` per hub and reason, AFTER
+    the removal loop and on whatever is left of the time budget (issue
+    #187): removal never pays for observability. A walk that reaches the
+    budget mid-tree is abandoned, and every folder left unsized is counted in
+    `held_unsized`. `sizes` (a `SizeCache`), when given, caches those walks
+    across sweeps and is re-keyed to this sweep's held folders; an entry is
+    reused while the folder's mtime is unchanged AND it is younger than
+    SIZE_CACHE_SECONDS by `now`, and each reuse is counted in `held_cached`.
+    An abandoned walk is kept there too (issue #189): the next sweep sizes
+    every other held folder first and then resumes it where it stopped, so
+    one large tree neither starves the folders behind it nor restarts from
+    scratch every sweep."""
     result = SweepResult()
     started = clock()
     bounded = False
@@ -576,7 +652,7 @@ def sweep(
     if stale_round_seconds is not None:
         result.backstop = True
         void_before = now - stale_round_seconds
-    seen_sizes: "dict[Path, tuple[float, int]]" = {}
+    held_queue: "list[tuple[float, Candidate, str]]" = []
     aged: "list[tuple[float, Candidate]]" = []
     for hub in hub_list:
         result.hubs += 1
@@ -603,17 +679,7 @@ def sweep(
         if why is not None:
             log.debug("janitor: keeping %s — %s", candidate.path, why)
             result.held.append((candidate.path, why))
-            key = f"{candidate.hub_slug}: {why}"
-            cached = (sizes or {}).get(candidate.path)
-            if cached is not None and cached[0] == mtime:
-                size = cached[1]
-            elif clock() - started >= max_seconds:
-                result.held_unsized += 1
-                continue
-            else:
-                size = dir_bytes(candidate.path)
-            seen_sizes[candidate.path] = (mtime, size)
-            result.held_bytes[key] = result.held_bytes.get(key, 0) + size
+            held_queue.append((mtime, candidate, f"{candidate.hub_slug}: {why}"))
             continue
         early = age < grace_seconds
         if early and not dry_run and not (under_pressure and under_pressure()):
@@ -650,9 +716,8 @@ def sweep(
         result.removed.append((candidate.path, size))
         if early:
             result.pressure_removed += 1
-    if sizes is not None:
-        sizes.clear()
-        sizes.update(seen_sizes)
+    _size_held(held_queue, result, sizes, now=now,
+               out_of_time=lambda: clock() - started >= max_seconds)
     if dry_run:
         return result
     for hub in hub_list:
@@ -662,3 +727,65 @@ def sweep(
         except (CommandError, OSError) as exc:
             log.warning("janitor: git worktree prune failed in %s (%s)", hub, exc)
     return result
+
+
+def _size_held(
+    queue: "list[tuple[float, Candidate, str]]",
+    result: SweepResult,
+    sizes: "SizeCache | None",
+    *,
+    now: float,
+    out_of_time: "Callable[[], bool]",
+) -> None:
+    """Size the sweep's held folders into `result.held_bytes`, after the
+    removals (issue #187). A fresh cache entry costs nothing and is always
+    used; anything else is walked only while `out_of_time()` answers False,
+    and a walk is abandoned the moment it does.
+
+    An abandoned walk is kept in `sizes` as a `HeldWalk` (issue #189). Next
+    sweep its folder goes to the back of the queue, behind every folder
+    whose walk was not cut short, and its walk resumes from where it stopped
+    -- from scratch if the folder's mtime moved. So a tree too big for one
+    sweep's leftover budget no longer leaves the folders behind it unsized
+    on every sweep, and still gets its figure after a few. A finished
+    resumed walk is cached as of the sweep it began in, so the hourly
+    re-walk counts from the oldest directory it read.
+
+    `sizes` is then re-keyed to this sweep's held folders: entries for gone
+    folders drop out, and so do unsized folders whose walk never began."""
+    previous = sizes or {}
+
+    def cut_short(item: "tuple[float, Candidate, str]") -> bool:
+        return isinstance(previous.get(item[1].path), HeldWalk)
+
+    seen: SizeCache = {}
+    # Stable: oldest first within each group, as the sweep queued them.
+    for mtime, candidate, key in sorted(queue, key=cut_short):
+        cached = previous.get(candidate.path)
+        if (isinstance(cached, tuple) and cached[0] == mtime
+                and now - cached[2] < SIZE_CACHE_SECONDS):
+            size = cached[1]
+            seen[candidate.path] = cached
+            result.held_cached += 1
+            result.held_bytes[key] = result.held_bytes.get(key, 0) + size
+            continue
+        resumable = isinstance(cached, HeldWalk)
+        if isinstance(cached, HeldWalk) and cached.mtime == mtime:
+            progress = cached
+        else:
+            progress = HeldWalk(mtime, now, 0, [os.fspath(candidate.path)])
+        walked = None
+        if not out_of_time():
+            walked = held_bytes_of(candidate.path, out_of_time, progress)
+            resumable = resumable or walked is None
+        if walked is None:
+            result.held_unsized += 1
+            if resumable:
+                seen[candidate.path] = progress
+                result.held_resuming += 1
+            continue
+        seen[candidate.path] = (mtime, walked, progress.started)
+        result.held_bytes[key] = result.held_bytes.get(key, 0) + walked
+    if sizes is not None:
+        sizes.clear()
+        sizes.update(seen)

{alissa_tools_github_revloop-0.31.24 → alissa_tools_github_revloop-0.31.26}/src/main/alissa/tools/github/revloop/loop.py

@@ -2847,9 +2847,10 @@ class ReviewWatcher:
         # under the line closes it. In memory, so a restart under pressure
         # opens (and reports) a new episode.
         self._janitor_pressure_since: "float | None" = None
-        # Held folders' sizes from the last sweep, path -> (mtime, bytes)
-        # (issue #185): the held-bytes summary re-walks only what changed.
-        self._janitor_sizes: "dict[Path, tuple[float, int]]" = {}
+        # Held folders' sizes from the last sweep, path -> (mtime, bytes,
+        # sized at) (issues #185, #187): the held-bytes summary re-walks only
+        # what changed or is older than janitor_mod.SIZE_CACHE_SECONDS.
+        self._janitor_sizes: "janitor_mod.SizeCache" = {}
         # The (repo, number, round) whose alive session the stale-round probe
         # just killed as `wedged:first-run-dialog` (issue #136), so the
         # respawn site logs its re-enqueue at INFO instead of a second

alissa_tools_github_revloop-0.31.26/src/main/alissa/tools/github/revloop/version

@@ -0,0 +1 @@
+0.31.26

{alissa_tools_github_revloop-0.31.24 → alissa_tools_github_revloop-0.31.26/src/main/alissa_tools_github_revloop.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alissa-tools-github-revloop
-Version: 0.31.24
+Version: 0.31.26
 Summary: ALISSA-TOOLS-GITHUB-REVLOOP
 Home-page: https://alissa.app
 Author: Fahera

alissa_tools_github_revloop-0.31.24/src/main/alissa/tools/github/revloop/version

@@ -1 +0,0 @@
-0.31.24