nbdevAuto 0.3.65__tar.gz → 0.3.66__tar.gz

{nbdevauto-0.3.65/nbdevAuto.egg-info → nbdevauto-0.3.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.65
+Version: 0.3.66
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.66/nbdevAuto/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.66"

{nbdevauto-0.3.65 → nbdevauto-0.3.66}/nbdevAuto/_modidx.py

@@ -30,6 +30,25 @@ d = { 'settings': { 'branch': 'main',
                                     'nbdevAuto.automate.release_pypi': ('automate.html#release_pypi', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.status': ('automate.html#status', 'nbdevAuto/automate.py'),
                                     'nbdevAuto.automate.upload': ('automate.html#upload', 'nbdevAuto/automate.py')},
+            'nbdevAuto.fleet': { 'nbdevAuto.fleet.Outcome': ('fleet.html#outcome', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report': ('fleet.html#report', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.code': ('fleet.html#report.code', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.failed': ('fleet.html#report.failed', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.ran': ('fleet.html#report.ran', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.Report.skip_counts': ('fleet.html#report.skip_counts', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._classify': ('fleet.html#_classify', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._fail_reason': ('fleet.html#_fail_reason', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._render': ('fleet.html#_render', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.finish': ('fleet.html#finish', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet': ('fleet.html#fleet', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet_upload': ('fleet.html#fleet_upload', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.format_summary': ('fleet.html#format_summary', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.parse_verb': ('fleet.html#parse_verb', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.run_verb': ('fleet.html#run_verb', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.script_path': ('fleet.html#script_path', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.upload_one': ('fleet.html#upload_one', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.usage': ('fleet.html#usage', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.walk': ('fleet.html#walk', 'nbdevAuto/fleet.py')},
             'nbdevAuto.functions': { 'nbdevAuto.functions.classify_images': ('functions.html#classify_images', 'nbdevAuto/functions.py'),
                                      'nbdevAuto.functions.create_data_folder': ( 'functions.html#create_data_folder',
                                                                                  'nbdevAuto/functions.py'),

nbdevauto-0.3.66/nbdevAuto/fleet.py

@@ -0,0 +1,201 @@
+"""Run one command across every submodule of a superproject
+
+Docs: https://bthek1.github.io/nbdevAuto/fleet.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_Fleet.ipynb.
+
+# %% auto #0
+__all__ = ['VERBS', 'Outcome', 'Report', 'format_summary', 'walk', 'finish', 'script_path', 'upload_one', 'fleet_upload',
+           'parse_verb', 'usage', 'run_verb', 'fleet']
+
+# %% ../nbs/04_Fleet.ipynb #d361ee5f-9f09-464f-aec6-0e27cec90fe9
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from fastcore.script import *
+
+# The rendering layer lives in `automate`, the git/discovery layer in `github`. This
+# module is the only one that knows about MANY repos; those two stay single-repo.
+from .automate import StepFailed, console, escape
+from .github import find_root, git, parse_gitmodules
+
+# %% ../nbs/04_Fleet.ipynb #4e6b24c5-cd71-434c-84e6-e6994d5959c7
+@dataclass
+class Outcome:
+    "What happened in one submodule"
+    path:   str
+    state:  str                                  # ran | skipped | failed
+    reason: str  = ""                            # why it was skipped, or how it failed
+    detail: list = field(default_factory=list)   # short lines to show under the path
+
+@dataclass
+class Report:
+    "Everything a `walk` did, and the exit code it implies"
+    outcomes: list = field(default_factory=list)
+
+    @property
+    def ran(self):    return [o for o in self.outcomes if o.state == "ran"]
+    @property
+    def failed(self): return [o for o in self.outcomes if o.state == "failed"]
+    @property
+    def code(self):   return 1 if self.failed else 0
+
+    def skip_counts(self):
+        "Skip reason -> count, in the order the reasons were first seen"
+        counts = {}
+        for o in self.outcomes:
+            if o.state == "skipped": counts[o.reason] = counts.get(o.reason, 0) + 1
+        return counts
+
+def format_summary(rep, verb="ran"):
+    "The one line that replaces a message per submodule. Quiet submodules are counted, not narrated"
+    n = len(rep.outcomes)
+    parts = [f"{len(rep.ran)} {verb}"]
+    parts += [f"{c} {reason}" for reason, c in rep.skip_counts().items()]
+    if rep.failed: parts.append(f"{len(rep.failed)} FAILED")
+    return f"{n} submodule{'' if n == 1 else 's'}: " + ", ".join(parts)
+
+# %% ../nbs/04_Fleet.ipynb #099242e8-c9ad-4b48-8ebc-7a818e5f8f81
+def _classify(base, sub, need_nbs, only_dirty):
+    "The reason to skip this submodule, or None to run it"
+    p = base/sub.path
+    # An uninitialised submodule is an empty directory: present, but with no .git
+    if not (p/".git").exists():          return "not initialised"
+    if need_nbs and not (p/"nbs").is_dir(): return "without nbs"
+    if only_dirty and not git(p, "status", "--porcelain"): return "clean"
+    return None
+
+def _fail_reason(e):
+    "One line describing why a submodule's callable did not finish"
+    if isinstance(e, StepFailed): return f"{e.cmd} exited {e.code}"
+    return f"{type(e).__name__}: {e}"
+
+def walk(fn, *, root=None, only_dirty=False, need_nbs=False, workers=1, header=True):
+    """Run `fn(path, sub)` in each submodule, returning a `Report`.
+
+    `fn` may return a list of short strings to show under the submodule's name. Only
+    submodules that actually do something get printed; the rest are counted by the caller
+    through `format_summary`. Sequential runs print the header BEFORE the callable so its
+    own output streams under it; parallel runs print after, so lines cannot interleave."""
+    base = Path(root) if root else find_root()
+    rep, todo = Report(), []
+    for s in parse_gitmodules(base):
+        why = _classify(base, s, need_nbs, only_dirty)
+        if why: rep.outcomes.append(Outcome(s.path, "skipped", why))
+        else:   todo.append(s)
+
+    def _one(s):
+        try: return Outcome(s.path, "ran", detail=list(fn(base/s.path, s) or []))
+        except BaseException as e: return Outcome(s.path, "failed", _fail_reason(e))
+
+    def _head(path): console.print(f"\n[bold cyan]==> {escape(path)}[/bold cyan]")
+
+    if workers > 1 and len(todo) > 1:
+        # Results are rendered in submodule order, not completion order, so two runs of
+        # the same fleet read the same way.
+        with ThreadPoolExecutor(max_workers=workers) as ex: done = list(ex.map(_one, todo))
+        for o in done:
+            if header: _head(o.path)
+            _render(o)
+            rep.outcomes.append(o)
+    else:
+        for s in todo:
+            if header: _head(s.path)
+            o = _one(s)
+            _render(o)
+            rep.outcomes.append(o)
+    rep.outcomes.sort(key=lambda o: o.path)
+    return rep
+
+def _render(o):
+    "A submodule's own lines, or why it failed"
+    if o.state == "failed":
+        console.print(f"    [bold red]FAIL[/bold red] [dim]{escape(o.reason)}[/dim]")
+    for line in o.detail: console.print(f"    [dim]{escape(line)}[/dim]")
+
+def finish(rep, verb):
+    "Print the counted summary, name any failures, and return the exit code"
+    console.print(f"\n[bold]{escape(format_summary(rep, verb))}[/bold]")
+    for o in rep.failed:
+        console.print(f"[bold red]FAILED[/bold red] {escape(o.path)}: [dim]{escape(o.reason)}[/dim]")
+    return rep.code
+
+# %% ../nbs/04_Fleet.ipynb #99e2c4fe-c885-4880-b9df-31da51e61f3a
+def script_path(name):
+    """The console script `name` from the environment this is running in.
+
+    Taking it from `sys.executable`'s directory is what makes this work under a
+    non-interactive shell that never put the venv on PATH."""
+    cand = Path(sys.executable).parent/name
+    if cand.is_file(): return str(cand)
+    found = shutil.which(name)
+    if found: return found
+    raise RuntimeError(f"Cannot find the '{name}' console script. Run `uv sync` first")
+
+def upload_one(path, sub=None, args=()):
+    """Run the single-repo `upload` in `path`, letting its own rich output through.
+
+    A subprocess rather than an in-process call: `prep` reads nbdev's config from the cwd
+    and caches it, and `nbdev_test` spawns its own process pool, so nineteen repos in one
+    interpreter is a trap. The subprocess keeps this tty, so colour survives."""
+    r = subprocess.run([script_path("upload"), *args], cwd=str(path))
+    if r.returncode: raise StepFailed("upload", r.returncode)
+    return []
+
+# %% ../nbs/04_Fleet.ipynb #2ef9f1e5-239a-4790-8cec-43fffb8bc422
+def fleet_upload(
+    all:bool = False,    # Upload every submodule with an nbs/ folder, not just the changed ones
+    root:str = None,     # Superproject root (default: nearest ancestor with .gitmodules)
+):
+    "Run `upload` in each submodule that has local changes"
+    # Sequential on purpose: `upload` runs nbdev_test with 8 workers, and knowledge-lab
+    # has 4 vCPU. Parallel uploads would oversubscribe the box and interleave the output.
+    rep = walk(upload_one, root=root, need_nbs=True, only_dirty=not all, workers=1)
+    sys.exit(finish(rep, "uploaded"))
+
+# %% ../nbs/04_Fleet.ipynb #0281bd2b-91ba-4e49-90dc-d1a95d54d5a6
+VERBS = {"upload": fleet_upload}
+
+def parse_verb(argv):
+    "(verb, remaining args) for `fleet ...`, or None when `argv` does not name a verb"
+    if not argv or argv[0].startswith("-"): return None
+    return (argv[0], argv[1:]) if argv[0] in VERBS else None
+
+def usage():
+    "The verb list, for --help and for an unknown verb"
+    lines = ["Usage: fleet <verb> [options]", "", "Verbs:"]
+    lines += [f"  {name:<10} {(fn.__doc__ or '').strip()}" for name, fn in sorted(VERBS.items())]
+    lines += ["", "Any verb takes --help for its own options."]
+    return "\n".join(lines)
+
+def run_verb(name, argv):
+    """Parse `argv` against the verb's own annotations, then call it.
+
+    Deliberately NOT `@call_parse`: fastcore runs the first decorated function in a module
+    the moment that module is `__main__`, which is exactly what makes `python -m
+    nbdevAuto.github` work with its single verb. With several verbs it would run whichever
+    happened to be defined first, so the verbs stay plain functions and the parser is built
+    here. `anno_parser` is the same one `call_parse` uses, so `--help` is unchanged."""
+    fn = VERBS[name]
+    args = vars(anno_parser(fn, prog=f"fleet {name}").parse_args(argv))
+    for k in ("pdb", "xtra"): args.pop(k, None)
+    return fn(**args)
+
+def fleet():
+    """Dispatch `fleet <verb>` to the matching command.
+
+    `upload` and `status` are already console scripts bound to the single-repo versions in
+    `automate`, so the fleet-wide ones cannot claim those names. One script with verbs
+    sidesteps the collision and keeps `--help` working per verb."""
+    picked = parse_verb(sys.argv[1:])
+    if not picked:
+        print(usage())
+        sys.exit(0 if sys.argv[1:2] in ([], ["-h"], ["--help"]) else 2)
+    return run_verb(*picked)
+
+# %% ../nbs/04_Fleet.ipynb #7adb89c6-6788-4241-acff-53cb4566b5ce
+if __name__ == "__main__": fleet()

{nbdevauto-0.3.65 → nbdevauto-0.3.66/nbdevAuto.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.65
+Version: 0.3.66
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

{nbdevauto-0.3.65 → nbdevauto-0.3.66}/nbdevAuto.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ nbdevAuto/__init__.py
 nbdevAuto/_modidx.py
 nbdevAuto/automate.py
 nbdevAuto/core.py
+nbdevAuto/fleet.py
 nbdevAuto/functions.py
 nbdevAuto/github.py
 nbdevAuto/pdf.py

{nbdevauto-0.3.65 → nbdevauto-0.3.66}/nbdevAuto.egg-info/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+fleet = nbdevAuto.fleet:fleet
 gacp = nbdevAuto.automate:gacp
 ghstatus = nbdevAuto.github:ghstatus
 gitrelease = nbdevAuto.automate:release_git

{nbdevauto-0.3.65 → nbdevauto-0.3.66}/pyproject.toml

@@ -51,6 +51,7 @@ piprelease = "nbdevAuto.automate:release_pypi"
 release = "nbdevAuto.automate:release"
 h = "nbdevAuto.automate:help_output"
 ghstatus = "nbdevAuto.github:ghstatus"
+fleet = "nbdevAuto.fleet:fleet"
 
 [tool.setuptools.dynamic]
 version = {attr = "nbdevAuto.__version__"}

nbdevauto-0.3.65/nbdevAuto/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.65"