nbdevAuto 0.3.64__tar.gz → 0.3.66__tar.gz

{nbdevauto-0.3.64/nbdevAuto.egg-info → nbdevauto-0.3.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.64
+Version: 0.3.66
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.66/nbdevAuto/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.66"

{nbdevauto-0.3.64 → nbdevauto-0.3.66}/nbdevAuto/_modidx.py

@@ -5,11 +5,23 @@ d = { 'settings': { 'branch': 'main',
                 'doc_host': 'https://bthek1.github.io',
                 'git_url': 'https://github.com/bthek1/nbdevAuto',
                 'lib_path': 'nbdevAuto'},
-  'syms': { 'nbdevAuto.automate': { 'nbdevAuto.automate._banner': ('automate.html#_banner', 'nbdevAuto/automate.py'),
+  'syms': { 'nbdevAuto.automate': { 'nbdevAuto.automate.StepFailed': ('automate.html#stepfailed', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate.StepFailed.__init__': ( 'automate.html#stepfailed.__init__',
+                                                                                'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._auto_msg': ('automate.html#_auto_msg', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._banner': ('automate.html#_banner', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._commit_summary': ('automate.html#_commit_summary', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate._done': ('automate.html#_done', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate._dur': ('automate.html#_dur', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._push_summary': ('automate.html#_push_summary', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._quiet': ('automate.html#_quiet', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._rule': ('automate.html#_rule', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._run': ('automate.html#_run', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._skip': ('automate.html#_skip', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate._status_table': ('automate.html#_status_table', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate._step': ('automate.html#_step', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._test_summary': ('automate.html#_test_summary', 'nbdevAuto/automate.py'),
+                                    'nbdevAuto.automate._version_summary': ('automate.html#_version_summary', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.gacp': ('automate.html#gacp', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.help_output': ('automate.html#help_output', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.prep': ('automate.html#prep', 'nbdevAuto/automate.py'),
@@ -18,6 +30,25 @@ d = { 'settings': { 'branch': 'main',
                                     'nbdevAuto.automate.release_pypi': ('automate.html#release_pypi', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.status': ('automate.html#status', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.upload': ('automate.html#upload', 'nbdevAuto/automate.py')},
+            'nbdevAuto.fleet': { 'nbdevAuto.fleet.Outcome': ('fleet.html#outcome', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report': ('fleet.html#report', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.code': ('fleet.html#report.code', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.failed': ('fleet.html#report.failed', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.ran': ('fleet.html#report.ran', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.skip_counts': ('fleet.html#report.skip_counts', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._classify': ('fleet.html#_classify', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._fail_reason': ('fleet.html#_fail_reason', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._render': ('fleet.html#_render', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.finish': ('fleet.html#finish', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet': ('fleet.html#fleet', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet_upload': ('fleet.html#fleet_upload', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.format_summary': ('fleet.html#format_summary', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.parse_verb': ('fleet.html#parse_verb', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.run_verb': ('fleet.html#run_verb', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.script_path': ('fleet.html#script_path', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.upload_one': ('fleet.html#upload_one', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.usage': ('fleet.html#usage', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.walk': ('fleet.html#walk', 'nbdevAuto/fleet.py')},
             'nbdevAuto.functions': { 'nbdevAuto.functions.classify_images': ('functions.html#classify_images', 'nbdevAuto/functions.py'),
                                      'nbdevAuto.functions.create_data_folder': ( 'functions.html#create_data_folder',
                                                                                  'nbdevAuto/functions.py'),

nbdevauto-0.3.66/nbdevAuto/automate.py

@@ -0,0 +1,341 @@
+"""Automate
+
+Docs: https://bthek1.github.io/nbdevAuto/automate.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_Automate.ipynb.
+
+# %% auto #0
+__all__ = ['prep', 'gacp', 'status', 'upload', 'release_git', 'release_pypi', 'release', 'help_output']
+
+# %% ../nbs/01_Automate.ipynb #6d1499d1-aa55-47d8-92fd-fa056b1fb135
+from fastcore.script import *
+from rich import box
+from rich.console import Console
+from rich.markup import escape
+from rich.rule import Rule
+from rich.table import Table
+
+# %% ../nbs/01_Automate.ipynb #7c1e5a90-3f52-4d21-9b6a-2c0f4e8d1a37
+import re
+import subprocess
+from contextlib import contextmanager, redirect_stderr, redirect_stdout
+from io import StringIO
+from time import perf_counter
+
+from nbdevAuto import __version__
+
+console = Console()
+
+# A rule drawn across a 200-column terminal is a wall of dashes. 88 reads as a header.
+_RULE_W = 88
+
+# `git status -s` codes: colour, the word for the third column, and the verb a commit
+# subject gets when every staged path carries that same code.
+_CODE_STYLE = {"M":"yellow", "A":"green", "D":"red", "R":"cyan", "C":"cyan", "U":"magenta", "?":"dim"}
+_CODE_LABEL = {"M":"modified", "A":"added", "D":"deleted", "R":"renamed", "C":"copied",
+               "U":"conflict", "?":"untracked"}
+_CODE_VERB  = {"M":"update", "A":"add", "D":"remove", "R":"rename", "C":"copy",
+               "U":"merge", "?":"add"}
+
+_RE_VERSION = re.compile(r"^(Old|New) version:\s*(\S+)", re.M)
+_RE_TIMING  = re.compile(r"^(?P<nb>.+?):\s+(?P<secs>[\d.]+)\s*secs?$")
+_RE_COMMIT  = re.compile(r"^\[(?P<branch>\S+)\s+(?P<sha>[0-9a-f]+)\]")
+_RE_COUNTS  = re.compile(r"(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?")
+_RE_WROTE   = re.compile(r"Writing objects:.*?,\s*(?P<size>[\d.]+\s*[KMG]?i?B)")
+
+class StepFailed(Exception):
+    "A stage exited non-zero. Carries the output so `_step` can show what went wrong"
+    def __init__(self, cmd, code, output=""):
+        self.cmd, self.code, self.output = cmd, code, output
+        super().__init__(f"{cmd} exited {code}")
+
+def _dur(t0):
+    "Elapsed time since `t0`, at a readable scale"
+    s = perf_counter() - t0
+    if s < 1:  return f"{s*1000:.0f}ms"
+    if s < 60: return f"{s:.1f}s"
+    return f"{int(s)//60}m{int(s)%60:02d}s"
+
+def _rule(text, style):
+    "A rule capped at `_RULE_W`; `console.rule` otherwise spans the whole terminal"
+    console.print(Rule(text, style=style), width=min(console.width, _RULE_W))
+
+def _banner(cmd):
+    "Command header, with the package version alongside"
+    _rule(f"[bold cyan]{cmd}[/bold cyan] [dim]nbdevAuto {__version__}[/dim]", "cyan")
+
+def _done(cmd, t0):
+    "Closing summary for a multi-stage command"
+    _rule(f"[green]{cmd} finished[/green] [dim]in {_dur(t0)}[/dim]", "green")
+
+@contextmanager
+def _step(n, total, label):
+    """Run one stage, reporting its outcome and duration.
+
+    Yields a list: append short strings and they print under the step, which is how a
+    stage says what it did without letting the underlying tool print it all. Re-raises
+    so nothing is swallowed, and a `StepFailed` shows the output it carries."""
+    console.print(f"[dim]{n}/{total}[/dim] [bold]{label}[/bold]")
+    t0, detail = perf_counter(), []
+    try:
+        yield detail
+    except StepFailed as e:
+        console.print(f"    [bold red]FAIL[/bold red] [dim]{_dur(t0)}, exit {e.code}[/dim]")
+        for line in e.output.splitlines():
+            if line.strip(): console.print(f"    [red]{escape(line.rstrip())}[/red]")
+        console.print()
+        raise
+    except BaseException:
+        console.print(f"    [bold red]FAIL[/bold red] [dim]{_dur(t0)}[/dim]\n")
+        raise
+    for line in detail: console.print(f"    [dim]{escape(line)}[/dim]")
+    console.print(f"    [green]OK[/green]   [dim]{_dur(t0)}[/dim]\n")
+
+def _skip(n, total, label, why):
+    "Report a stage that was deliberately not run"
+    console.print(f"[dim]{n}/{total}[/dim] [bold]{label}[/bold]")
+    console.print(f"    [yellow]SKIP[/yellow] [dim]{why}[/dim]\n")
+
+def _run(*cmd):
+    """Run `cmd` with output captured, returning stdout+stderr.
+
+    A bare `subprocess.run` returns an exit code nobody reads, so a failed `git push`
+    still printed a green OK. Raising is what makes that OK mean something."""
+    r = subprocess.run(cmd, capture_output=True, text=True)
+    out = f"{r.stdout or ''}{r.stderr or ''}"
+    if r.returncode: raise StepFailed(" ".join(cmd), r.returncode, out)
+    return out
+
+@contextmanager
+def _quiet():
+    "Collect what a wrapped nbdev call prints, so the step can summarise it instead"
+    buf = StringIO()
+    try:
+        with redirect_stdout(buf), redirect_stderr(buf): yield buf
+    except BaseException:
+        # A failing stage is the one time every raw line is worth having.
+        text = buf.getvalue().strip()
+        if text: console.print(f"[dim]{escape(text)}[/dim]")
+        raise
+
+def _version_summary(out):
+    "The two lines `nbdev_bump_version` prints, as one arrow"
+    v = dict(_RE_VERSION.findall(out))
+    if "Old" in v and "New" in v: return [f"{v['Old']} -> {v['New']}"]
+    return [l.strip() for l in out.splitlines() if l.strip()]
+
+def _test_summary(out, slow_secs=1.0, keep=3):
+    "`nbdev_test` prints a line per notebook; keep the count and only the genuinely slow ones"
+    times = []
+    for line in out.splitlines():
+        m = _RE_TIMING.match(line.strip())
+        if m:
+            try: times.append((m["nb"], float(m["secs"])))
+            except ValueError: pass
+    if not times: return [l.strip() for l in out.splitlines() if l.strip()][-3:]
+    head = f"{len(times)} notebook{'' if len(times) == 1 else 's'}"
+    slow = [t for t in sorted(times, key=lambda t: -t[1]) if t[1] >= slow_secs][:keep]
+    if not slow: return [f"{head}, none slower than {slow_secs:g}s"]
+    return [f"{head}, slowest:"] + [f"  {nb} {s:.1f}s" for nb, s in slow]
+
+def _commit_summary(out):
+    "Where the commit landed and how big it was, without echoing the subject back"
+    lines = [l.strip() for l in out.splitlines() if l.strip()]
+    if not lines: return []
+    m, c = _RE_COMMIT.match(lines[0]), _RE_COUNTS.search(out)
+    head = f"{m['branch']} {m['sha']}" if m else lines[0]
+    if not c: return [head]
+    files, ins, dels = c.group(1), c.group(2) or "0", c.group(3) or "0"
+    return [f"{head}  {files} file{'' if files == '1' else 's'}, +{ins} -{dels}"]
+
+def _push_summary(out):
+    "What moved and how much went over the wire; the progress meters are noise"
+    lines = [l.rstrip() for l in out.splitlines() if l.strip()]
+    if not lines: return ["nothing to push"]
+    if any("Everything up-to-date" in l for l in lines): return ["everything up-to-date"]
+    detail = []
+    for i, l in enumerate(lines):
+        if l.startswith("To ") and i + 1 < len(lines):
+            detail += [lines[i + 1].strip(), f"to {l[3:].strip()}"]
+            break
+    m = _RE_WROTE.search(out)
+    if m: detail.append(f"{m['size'].strip()} written")
+    return detail or lines[-1:]
+
+def _auto_msg(porcelain, limit=3):
+    "A commit subject built from the staged paths. The raw status output made a poor one"
+    rows = [(l[:2].strip()[:1], l[3:].strip()) for l in porcelain.splitlines() if l.strip()]
+    if not rows: return "update"
+    verb  = _CODE_VERB.get(rows[0][0], "update") if len({c for c, _ in rows}) == 1 else "update"
+    names = [p.split(" -> ")[-1] for _, p in rows]
+    head  = ", ".join(names[:limit])
+    if len(names) > limit: head += f" and {len(names) - limit} more"
+    return f"{verb} {head}"
+
+def _status_table(porcelain):
+    "Render `git status -s` output as a table, one row per path"
+    t = Table(box=box.SIMPLE, show_header=False, pad_edge=False, expand=False)
+    t.add_column("code", no_wrap=True, justify="right")
+    t.add_column("path", overflow="fold")
+    t.add_column("what", style="dim", no_wrap=True)
+    n = 0
+    for line in porcelain.splitlines():
+        if not line.strip(): continue
+        code, path = line[:2].strip(), line[3:]
+        style = _CODE_STYLE.get(code[:1], "white")
+        t.add_row(f"[{style}]{code}[/{style}]", escape(path), _CODE_LABEL.get(code[:1], ""))
+        n += 1
+    return t, n
+
+# %% ../nbs/01_Automate.ipynb #3a6f81d2-5c47-4e19-8b03-9d2a7f6c0e15
+@call_parse
+def prep(
+    p:int = 2, # Increment Part
+):
+    "Bump version part `p`, then export, test and clean the notebooks, refreshing _quarto.yml and README"
+
+    import nbdev.test, nbdev.clean, nbdev.quarto, nbdev.release
+    _banner("prep")
+    t0 = perf_counter()
+    with _step(1, 6, f"bump version (part {p})") as d, _quiet() as out:
+        nbdev.release.nbdev_bump_version(p)
+        d += _version_summary(out.getvalue())
+    with _step(2, 6, "nbdev_export"), _quiet():
+        nbdev.quarto.nbdev_export.__wrapped__()
+    with _step(3, 6, "nbdev_test") as d, _quiet() as out:
+        nbdev.test.nbdev_test.__wrapped__(
+            n_workers = 8,  # Number of workers
+            timing = True,  # Time each notebook to see which are slow
+        )
+        d += _test_summary(out.getvalue())
+    with _step(4, 6, "nbdev_clean"), _quiet():
+        nbdev.clean.nbdev_clean.__wrapped__()
+    with _step(5, 6, "refresh_quarto_yml"), _quiet():
+        nbdev.quarto.refresh_quarto_yml()
+    with _step(6, 6, "nbdev_readme"), _quiet():
+        nbdev.quarto.nbdev_readme.__wrapped__(chk_time=True)
+    _done("prep", t0)
+
+# %% ../nbs/01_Automate.ipynb #f66ef35f-b7fa-4039-aa07-e9b782d204bb
+@call_parse
+def gacp(
+    m:str = '', # Commit message
+):
+    "git add, commit and push. Without `-m` the message is built from the staged paths"
+
+    _banner("gacp")
+    t0 = perf_counter()
+    with _step(1, 3, "git add"):
+        _run("git", "add", ".")
+    # NOT via `_run`: `git status -s` pads the code to two columns, and the capture has
+    # to keep that leading space or the first path shifts by one.
+    status = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
+    staged = bool(status.strip())
+    if staged:
+        table, n = _status_table(status)
+        console.print(table)
+        console.print(f"[dim]{n} path{'' if n == 1 else 's'} staged[/dim]\n")
+    else:
+        console.print("[dim]nothing staged, the tree is clean[/dim]\n")
+
+    if staged:
+        with _step(2, 3, "git commit") as d:
+            d += _commit_summary(_run("git", "commit", "-m", m if m != '' else _auto_msg(status)))
+    else:
+        # `git commit` exits 1 with nothing staged, which is now a hard failure rather
+        # than a silent one. There may still be unpushed commits, so go on to the push.
+        _skip(2, 3, "git commit", "nothing staged")
+    with _step(3, 3, "git push") as d:
+        d += _push_summary(_run("git", "push"))
+    _done("gacp", t0)
+
+# %% ../nbs/01_Automate.ipynb #88017a1b-55c1-4833-aeb6-277a7e5b6314
+def status():
+    "Show the working tree state"
+    import subprocess
+
+    def _git(*a, **kw): return subprocess.check_output(["git", *a], **kw).decode('utf-8').strip()
+
+    _banner("status")
+    branch = _git("branch", "--show-current") or "DETACHED"
+    line = f"on [bold]{branch}[/bold]"
+    try:
+        behind, ahead = _git("rev-list", "--left-right", "--count", "@{upstream}...HEAD",
+                             stderr=subprocess.DEVNULL).split()
+        if behind != "0": line += f"  [yellow]behind {behind}[/yellow]"
+        if ahead  != "0": line += f"  [cyan]ahead {ahead}[/cyan]"
+        if behind == ahead == "0": line += "  [dim]in step with upstream[/dim]"
+    except subprocess.CalledProcessError:
+        line += "  [dim]no upstream[/dim]"
+    console.print(line)
+
+    # NOT via _git: `git status -s` pads the code to two columns, so stripping the
+    # output would eat the leading space of the first line and shift its path by one.
+    porcelain = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
+    if not porcelain:
+        console.print("[green]clean[/green]")
+        return
+    table, n = _status_table(porcelain)
+    console.print(table)
+    console.print(f"[dim]{n} path{'' if n == 1 else 's'} changed[/dim]")
+
+# %% ../nbs/01_Automate.ipynb #8d777a41-0160-47f8-b51c-b14360b62da4
+@call_parse
+def upload(
+    m:str = '', # Commit message
+    p:int = 2, # Increment part
+):
+    "prep then gacp: the everyday command for shipping a notebook change"
+    prep(p)
+    gacp(m)
+
+# %% ../nbs/01_Automate.ipynb #dee37b06-c946-4b40-be0f-056e133751ea
+def release_git():
+    "Bump the MINOR version, then tag and create a GitHub release"
+    import nbdev.release
+    _banner("gitrelease")
+    t0 = perf_counter()
+    with _step(1, 2, "bump minor version") as d, _quiet() as out:
+        nbdev.release.nbdev_bump_version(1)
+        d += _version_summary(out.getvalue())
+    with _step(2, 2, "tag and create the GitHub release"):
+        nbdev.release.release_git()
+    _done("gitrelease", t0)
+
+# %% ../nbs/01_Automate.ipynb #288708c6-194a-4059-9ff0-ae330079c199
+def release_pypi():
+    "Build the sdist and wheel, then upload to PyPI with twine. CI publishes on push, so this is the manual path"
+    import nbdev.release
+    _banner("piprelease")
+    t0 = perf_counter()
+    # Deliberately not quieted: a twine upload is worth watching line by line.
+    with _step(1, 1, "build and upload to PyPI"):
+        nbdev.release.release_pypi()
+    _done("piprelease", t0)
+
+# %% ../nbs/01_Automate.ipynb #dd798178-9d58-457e-969a-d86a5e8bf875
+def release():
+    "release_git then release_pypi"
+    release_git()
+    release_pypi()
+
+# %% ../nbs/01_Automate.ipynb #c4d2bd52-04c4-43fe-890b-b4cb5da8d206
+def help_output():
+    "Print every console script this package installs, with its help"
+    from importlib.metadata import distribution
+    try:
+        eps = [e for e in distribution("nbdevAuto").entry_points if e.group == "console_scripts"]
+    except Exception:
+        # Running from a source tree with nothing installed: fall back to fastcore.
+        from fastcore.xtras import console_help
+        return console_help('nbdevAuto')
+    _banner("commands")
+    t = Table(box=box.SIMPLE, show_header=False, pad_edge=False, expand=False)
+    t.add_column("command", style="bold cyan", no_wrap=True)
+    t.add_column("does", overflow="fold")
+    for e in sorted(eps, key=lambda e: e.name):
+        try: doc = escape((e.load().__doc__ or "").strip().splitlines()[0])
+        except Exception: doc = "[red]could not load[/red]"
+        t.add_row(e.name, doc)
+    console.print(t)
+    console.print(escape("ghstatus needs the gh extra: pip install 'nbdevAuto[gh]'"), style="dim")

nbdevauto-0.3.66/nbdevAuto/fleet.py

@@ -0,0 +1,201 @@
+"""Run one command across every submodule of a superproject
+
+Docs: https://bthek1.github.io/nbdevAuto/fleet.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_Fleet.ipynb.
+
+# %% auto #0
+__all__ = ['VERBS', 'Outcome', 'Report', 'format_summary', 'walk', 'finish', 'script_path', 'upload_one', 'fleet_upload',
+           'parse_verb', 'usage', 'run_verb', 'fleet']
+
+# %% ../nbs/04_Fleet.ipynb #d361ee5f-9f09-464f-aec6-0e27cec90fe9
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from fastcore.script import *
+
+# The rendering layer lives in `automate`, the git/discovery layer in `github`. This
+# module is the only one that knows about MANY repos; those two stay single-repo.
+from .automate import StepFailed, console, escape
+from .github import find_root, git, parse_gitmodules
+
+# %% ../nbs/04_Fleet.ipynb #4e6b24c5-cd71-434c-84e6-e6994d5959c7
+@dataclass
+class Outcome:
+    "What happened in one submodule"
+    path:   str
+    state:  str                                  # ran | skipped | failed
+    reason: str  = ""                            # why it was skipped, or how it failed
+    detail: list = field(default_factory=list)   # short lines to show under the path
+
+@dataclass
+class Report:
+    "Everything a `walk` did, and the exit code it implies"
+    outcomes: list = field(default_factory=list)
+
+    @property
+    def ran(self):    return [o for o in self.outcomes if o.state == "ran"]
+    @property
+    def failed(self): return [o for o in self.outcomes if o.state == "failed"]
+    @property
+    def code(self):   return 1 if self.failed else 0
+
+    def skip_counts(self):
+        "Skip reason -> count, in the order the reasons were first seen"
+        counts = {}
+        for o in self.outcomes:
+            if o.state == "skipped": counts[o.reason] = counts.get(o.reason, 0) + 1
+        return counts
+
+def format_summary(rep, verb="ran"):
+    "The one line that replaces a message per submodule. Quiet submodules are counted, not narrated"
+    n = len(rep.outcomes)
+    parts = [f"{len(rep.ran)} {verb}"]
+    parts += [f"{c} {reason}" for reason, c in rep.skip_counts().items()]
+    if rep.failed: parts.append(f"{len(rep.failed)} FAILED")
+    return f"{n} submodule{'' if n == 1 else 's'}: " + ", ".join(parts)
+
+# %% ../nbs/04_Fleet.ipynb #099242e8-c9ad-4b48-8ebc-7a818e5f8f81
+def _classify(base, sub, need_nbs, only_dirty):
+    "The reason to skip this submodule, or None to run it"
+    p = base/sub.path
+    # An uninitialised submodule is an empty directory: present, but with no .git
+    if not (p/".git").exists():          return "not initialised"
+    if need_nbs and not (p/"nbs").is_dir(): return "without nbs"
+    if only_dirty and not git(p, "status", "--porcelain"): return "clean"
+    return None
+
+def _fail_reason(e):
+    "One line describing why a submodule's callable did not finish"
+    if isinstance(e, StepFailed): return f"{e.cmd} exited {e.code}"
+    return f"{type(e).__name__}: {e}"
+
+def walk(fn, *, root=None, only_dirty=False, need_nbs=False, workers=1, header=True):
+    """Run `fn(path, sub)` in each submodule, returning a `Report`.
+
+    `fn` may return a list of short strings to show under the submodule's name. Only
+    submodules that actually do something get printed; the rest are counted by the caller
+    through `format_summary`. Sequential runs print the header BEFORE the callable so its
+    own output streams under it; parallel runs print after, so lines cannot interleave."""
+    base = Path(root) if root else find_root()
+    rep, todo = Report(), []
+    for s in parse_gitmodules(base):
+        why = _classify(base, s, need_nbs, only_dirty)
+        if why: rep.outcomes.append(Outcome(s.path, "skipped", why))
+        else:   todo.append(s)
+
+    def _one(s):
+        try: return Outcome(s.path, "ran", detail=list(fn(base/s.path, s) or []))
+        except BaseException as e: return Outcome(s.path, "failed", _fail_reason(e))
+
+    def _head(path): console.print(f"\n[bold cyan]==> {escape(path)}[/bold cyan]")
+
+    if workers > 1 and len(todo) > 1:
+        # Results are rendered in submodule order, not completion order, so two runs of
+        # the same fleet read the same way.
+        with ThreadPoolExecutor(max_workers=workers) as ex: done = list(ex.map(_one, todo))
+        for o in done:
+            if header: _head(o.path)
+            _render(o)
+            rep.outcomes.append(o)
+    else:
+        for s in todo:
+            if header: _head(s.path)
+            o = _one(s)
+            _render(o)
+            rep.outcomes.append(o)
+    rep.outcomes.sort(key=lambda o: o.path)
+    return rep
+
+def _render(o):
+    "A submodule's own lines, or why it failed"
+    if o.state == "failed":
+        console.print(f"    [bold red]FAIL[/bold red] [dim]{escape(o.reason)}[/dim]")
+    for line in o.detail: console.print(f"    [dim]{escape(line)}[/dim]")
+
+def finish(rep, verb):
+    "Print the counted summary, name any failures, and return the exit code"
+    console.print(f"\n[bold]{escape(format_summary(rep, verb))}[/bold]")
+    for o in rep.failed:
+        console.print(f"[bold red]FAILED[/bold red] {escape(o.path)}: [dim]{escape(o.reason)}[/dim]")
+    return rep.code
+
+# %% ../nbs/04_Fleet.ipynb #99e2c4fe-c885-4880-b9df-31da51e61f3a
+def script_path(name):
+    """The console script `name` from the environment this is running in.
+
+    Taking it from `sys.executable`'s directory is what makes this work under a
+    non-interactive shell that never put the venv on PATH."""
+    cand = Path(sys.executable).parent/name
+    if cand.is_file(): return str(cand)
+    found = shutil.which(name)
+    if found: return found
+    raise RuntimeError(f"Cannot find the '{name}' console script. Run `uv sync` first")
+
+def upload_one(path, sub=None, args=()):
+    """Run the single-repo `upload` in `path`, letting its own rich output through.
+
+    A subprocess rather than an in-process call: `prep` reads nbdev's config from the cwd
+    and caches it, and `nbdev_test` spawns its own process pool, so nineteen repos in one
+    interpreter is a trap. The subprocess keeps this tty, so colour survives."""
+    r = subprocess.run([script_path("upload"), *args], cwd=str(path))
+    if r.returncode: raise StepFailed("upload", r.returncode)
+    return []
+
+# %% ../nbs/04_Fleet.ipynb #2ef9f1e5-239a-4790-8cec-43fffb8bc422
+def fleet_upload(
+    all:bool = False,    # Upload every submodule with an nbs/ folder, not just the changed ones
+    root:str = None,     # Superproject root (default: nearest ancestor with .gitmodules)
+):
+    "Run `upload` in each submodule that has local changes"
+    # Sequential on purpose: `upload` runs nbdev_test with 8 workers, and knowledge-lab
+    # has 4 vCPU. Parallel uploads would oversubscribe the box and interleave the output.
+    rep = walk(upload_one, root=root, need_nbs=True, only_dirty=not all, workers=1)
+    sys.exit(finish(rep, "uploaded"))
+
+# %% ../nbs/04_Fleet.ipynb #0281bd2b-91ba-4e49-90dc-d1a95d54d5a6
+VERBS = {"upload": fleet_upload}
+
+def parse_verb(argv):
+    "(verb, remaining args) for `fleet ...`, or None when `argv` does not name a verb"
+    if not argv or argv[0].startswith("-"): return None
+    return (argv[0], argv[1:]) if argv[0] in VERBS else None
+
+def usage():
+    "The verb list, for --help and for an unknown verb"
+    lines = ["Usage: fleet <verb> [options]", "", "Verbs:"]
+    lines += [f"  {name:<10} {(fn.__doc__ or '').strip()}" for name, fn in sorted(VERBS.items())]
+    lines += ["", "Any verb takes --help for its own options."]
+    return "\n".join(lines)
+
+def run_verb(name, argv):
+    """Parse `argv` against the verb's own annotations, then call it.
+
+    Deliberately NOT `@call_parse`: fastcore runs the first decorated function in a module
+    the moment that module is `__main__`, which is exactly what makes `python -m
+    nbdevAuto.github` work with its single verb. With several verbs it would run whichever
+    happened to be defined first, so the verbs stay plain functions and the parser is built
+    here. `anno_parser` is the same one `call_parse` uses, so `--help` is unchanged."""
+    fn = VERBS[name]
+    args = vars(anno_parser(fn, prog=f"fleet {name}").parse_args(argv))
+    for k in ("pdb", "xtra"): args.pop(k, None)
+    return fn(**args)
+
+def fleet():
+    """Dispatch `fleet <verb>` to the matching command.
+
+    `upload` and `status` are already console scripts bound to the single-repo versions in
+    `automate`, so the fleet-wide ones cannot claim those names. One script with verbs
+    sidesteps the collision and keeps `--help` working per verb."""
+    picked = parse_verb(sys.argv[1:])
+    if not picked:
+        print(usage())
+        sys.exit(0 if sys.argv[1:2] in ([], ["-h"], ["--help"]) else 2)
+    return run_verb(*picked)
+
+# %% ../nbs/04_Fleet.ipynb #7adb89c6-6788-4241-acff-53cb4566b5ce
+if __name__ == "__main__": fleet()

{nbdevauto-0.3.64 → nbdevauto-0.3.66/nbdevAuto.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.64
+Version: 0.3.66
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

{nbdevauto-0.3.64 → nbdevauto-0.3.66}/nbdevAuto.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ nbdevAuto/__init__.py
 nbdevAuto/_modidx.py
 nbdevAuto/automate.py
 nbdevAuto/core.py
+nbdevAuto/fleet.py
 nbdevAuto/functions.py
 nbdevAuto/github.py
 nbdevAuto/pdf.py

{nbdevauto-0.3.64 → nbdevauto-0.3.66}/nbdevAuto.egg-info/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+fleet = nbdevAuto.fleet:fleet
 gacp = nbdevAuto.automate:gacp
 ghstatus = nbdevAuto.github:ghstatus
 gitrelease = nbdevAuto.automate:release_git

{nbdevauto-0.3.64 → nbdevauto-0.3.66}/pyproject.toml

@@ -51,6 +51,7 @@ piprelease = "nbdevAuto.automate:release_pypi"
 release = "nbdevAuto.automate:release"
 h = "nbdevAuto.automate:help_output"
 ghstatus = "nbdevAuto.github:ghstatus"
+fleet = "nbdevAuto.fleet:fleet"
 
 [tool.setuptools.dynamic]
 version = {attr = "nbdevAuto.__version__"}

nbdevauto-0.3.64/nbdevAuto/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.64"

nbdevauto-0.3.64/nbdevAuto/automate.py

@@ -1,209 +0,0 @@
-"""Automate
-
-Docs: https://bthek1.github.io/nbdevAuto/automate.html.md"""
-
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_Automate.ipynb.
-
-# %% auto #0
-__all__ = ['prep', 'gacp', 'status', 'upload', 'release_git', 'release_pypi', 'release', 'help_output']
-
-# %% ../nbs/01_Automate.ipynb #6d1499d1-aa55-47d8-92fd-fa056b1fb135
-from fastcore.script import *
-from rich import box
-from rich.console import Console
-from rich.markup import escape
-from rich.table import Table
-
-# %% ../nbs/01_Automate.ipynb #7c1e5a90-3f52-4d21-9b6a-2c0f4e8d1a37
-from contextlib import contextmanager
-from time import perf_counter
-
-from nbdevAuto import __version__
-
-console = Console()
-
-# `git status -s` codes: colour, and the word for the third column.
-_CODE_STYLE = {"M":"yellow", "A":"green", "D":"red", "R":"cyan", "C":"cyan", "U":"magenta", "?":"dim"}
-_CODE_LABEL = {"M":"modified", "A":"added", "D":"deleted", "R":"renamed", "C":"copied",
-               "U":"conflict", "?":"untracked"}
-
-def _dur(t0):
-    "Elapsed time since `t0`, at a readable scale"
-    s = perf_counter() - t0
-    if s < 1:  return f"{s*1000:.0f}ms"
-    if s < 60: return f"{s:.1f}s"
-    return f"{int(s)//60}m{int(s)%60:02d}s"
-
-def _banner(cmd):
-    "Command header, with the package version alongside"
-    console.rule(f"[bold cyan]{cmd}[/bold cyan] [dim]nbdevAuto {__version__}[/dim]", style="cyan")
-
-def _done(cmd, t0):
-    "Closing summary for a multi-stage command"
-    console.rule(f"[green]{cmd} finished[/green] [dim]in {_dur(t0)}[/dim]", style="green")
-
-@contextmanager
-def _step(n, total, label):
-    "Run one stage, reporting its outcome and duration. Re-raises so nothing is swallowed"
-    console.print(f"[dim]{n}/{total}[/dim] [bold]{label}[/bold]")
-    t0 = perf_counter()
-    try:
-        yield
-    except BaseException:
-        console.print(f"    [bold red]FAIL[/bold red] [dim]{_dur(t0)}[/dim]\n")
-        raise
-    console.print(f"    [green]OK[/green]   [dim]{_dur(t0)}[/dim]\n")
-
-def _status_table(porcelain):
-    "Render `git status -s` output as a table, one row per path"
-    t = Table(box=box.SIMPLE, show_header=False, pad_edge=False, expand=False)
-    t.add_column("code", no_wrap=True, justify="right")
-    t.add_column("path", overflow="fold")
-    t.add_column("what", style="dim", no_wrap=True)
-    n = 0
-    for line in porcelain.splitlines():
-        if not line.strip(): continue
-        code, path = line[:2].strip(), line[3:]
-        style = _CODE_STYLE.get(code[:1], "white")
-        t.add_row(f"[{style}]{code}[/{style}]", escape(path), _CODE_LABEL.get(code[:1], ""))
-        n += 1
-    return t, n
-
-# %% ../nbs/01_Automate.ipynb #3a6f81d2-5c47-4e19-8b03-9d2a7f6c0e15
-@call_parse
-def prep(
-    p:int = 2, # Increment Part
-):
-    "Bump version part `p`, then export, test and clean the notebooks, refreshing _quarto.yml and README"
-
-    import nbdev.test, nbdev.clean, nbdev.quarto, nbdev.release
-    _banner("prep")
-    t0 = perf_counter()
-    with _step(1, 6, f"bump version (part {p})"):
-        nbdev.release.nbdev_bump_version(p)
-    with _step(2, 6, "nbdev_export"):
-        nbdev.quarto.nbdev_export.__wrapped__()
-    with _step(3, 6, "nbdev_test"):
-        nbdev.test.nbdev_test.__wrapped__(
-            n_workers = 8,  # Number of workers
-            timing = True,  # Time each notebook to see which are slow
-        )
-    with _step(4, 6, "nbdev_clean"):
-        nbdev.clean.nbdev_clean.__wrapped__()
-    with _step(5, 6, "refresh_quarto_yml"):
-        nbdev.quarto.refresh_quarto_yml()
-    with _step(6, 6, "nbdev_readme"):
-        nbdev.quarto.nbdev_readme.__wrapped__(chk_time=True)
-    _done("prep", t0)
-
-# %% ../nbs/01_Automate.ipynb #f66ef35f-b7fa-4039-aa07-e9b782d204bb
-@call_parse
-def gacp(
-    m:str = '', # Commit message
-):
-    "git add, commit and push. Without `-m` the commit message is the status output"
-
-    import subprocess
-    _banner("gacp")
-    t0 = perf_counter()
-    with _step(1, 3, "git add"):
-        subprocess.run(["git", "add", "."])
-    status = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
-    if status.strip():
-        table, n = _status_table(status)
-        console.print(table)
-        console.print(f"[dim]{n} path{'' if n == 1 else 's'} staged[/dim]\n")
-    else:
-        console.print("[dim]nothing staged, the tree is clean[/dim]\n")
-
-    msg = m if m != '' else status
-    with _step(2, 3, "git commit"):
-        subprocess.run(["git", "commit", "-m", msg])
-    with _step(3, 3, "git push"):
-        subprocess.run(["git", "push"])
-    _done("gacp", t0)
-
-# %% ../nbs/01_Automate.ipynb #88017a1b-55c1-4833-aeb6-277a7e5b6314
-def status():
-    "Show the working tree state"
-    import subprocess
-
-    def _git(*a, **kw): return subprocess.check_output(["git", *a], **kw).decode('utf-8').strip()
-
-    _banner("status")
-    branch = _git("branch", "--show-current") or "DETACHED"
-    line = f"on [bold]{branch}[/bold]"
-    try:
-        behind, ahead = _git("rev-list", "--left-right", "--count", "@{upstream}...HEAD",
-                             stderr=subprocess.DEVNULL).split()
-        if behind != "0": line += f"  [yellow]behind {behind}[/yellow]"
-        if ahead  != "0": line += f"  [cyan]ahead {ahead}[/cyan]"
-        if behind == ahead == "0": line += "  [dim]in step with upstream[/dim]"
-    except subprocess.CalledProcessError:
-        line += "  [dim]no upstream[/dim]"
-    console.print(line)
-
-    # NOT via _git: `git status -s` pads the code to two columns, so stripping the
-    # output would eat the leading space of the first line and shift its path by one.
-    porcelain = subprocess.check_output(["git", "status", "-s"]).decode('utf-8')
-    if not porcelain:
-        console.print("[green]clean[/green]")
-        return
-    table, n = _status_table(porcelain)
-    console.print(table)
-    console.print(f"[dim]{n} path{'' if n == 1 else 's'} changed[/dim]")
-
-# %% ../nbs/01_Automate.ipynb #8d777a41-0160-47f8-b51c-b14360b62da4
-@call_parse
-def upload(
-    m:str = '', # Commit message
-    p:int = 2, # Increment part
-):
-    "prep then gacp: the everyday command for shipping a notebook change"
-    prep(p)
-    gacp(m)
-
-# %% ../nbs/01_Automate.ipynb #dee37b06-c946-4b40-be0f-056e133751ea
-def release_git():
-    "Bump the MINOR version, then tag and create a GitHub release"
-    import nbdev.release
-    console.rule("[bold]gitrelease")
-    with _step("bump minor version"):
-        nbdev.release.nbdev_bump_version(1)
-    with _step("tag and create the GitHub release"):
-        nbdev.release.release_git()
-
-# %% ../nbs/01_Automate.ipynb #288708c6-194a-4059-9ff0-ae330079c199
-def release_pypi():
-    "Build the sdist and wheel, then upload to PyPI with twine. CI publishes on push, so this is the manual path"
-    import nbdev.release
-    console.rule("[bold]piprelease")
-    with _step("build and upload to PyPI"):
-        nbdev.release.release_pypi()
-
-# %% ../nbs/01_Automate.ipynb #dd798178-9d58-457e-969a-d86a5e8bf875
-def release():
-    "release_git then release_pypi"
-    release_git()
-    release_pypi()
-
-# %% ../nbs/01_Automate.ipynb #c4d2bd52-04c4-43fe-890b-b4cb5da8d206
-def help_output():
-    "Print every console script this package installs, with its help"
-    from importlib.metadata import distribution
-    try:
-        eps = [e for e in distribution("nbdevAuto").entry_points if e.group == "console_scripts"]
-    except Exception:
-        # Running from a source tree with nothing installed: fall back to fastcore.
-        from fastcore.xtras import console_help
-        return console_help('nbdevAuto')
-    _banner("commands")
-    t = Table(box=box.SIMPLE, show_header=False, pad_edge=False, expand=False)
-    t.add_column("command", style="bold cyan", no_wrap=True)
-    t.add_column("does", overflow="fold")
-    for e in sorted(eps, key=lambda e: e.name):
-        try: doc = escape((e.load().__doc__ or "").strip().splitlines()[0])
-        except Exception: doc = "[red]could not load[/red]"
-        t.add_row(e.name, doc)
-    console.print(t)
-    console.print(escape("ghstatus needs the gh extra: pip install 'nbdevAuto[gh]'"), style="dim")