nbdevAuto 0.3.75__tar.gz → 0.3.77__tar.gz

{nbdevauto-0.3.75/nbdevAuto.egg-info → nbdevauto-0.3.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.75
+Version: 0.3.77
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.77/nbdevAuto/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.77"

{nbdevauto-0.3.75 → nbdevauto-0.3.77}/nbdevAuto/_modidx.py

@@ -47,7 +47,9 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet._hits': ('fleet.html#_hits', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._line_of': ('fleet.html#_line_of', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._porcelain': ('fleet.html#_porcelain', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._public_notebooks': ('fleet.html#_public_notebooks', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._render': ('fleet.html#_render', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._src': ('fleet.html#_src', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.ahead_of_remote': ('fleet.html#ahead_of_remote', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.copy_one': ('fleet.html#copy_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.dirty_submodules': ('fleet.html#dirty_submodules', 'nbdevAuto/fleet.py'),
@@ -55,6 +57,7 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet.finish': ('fleet.html#finish', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet': ('fleet.html#fleet', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_copy': ('fleet.html#fleet_copy', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet_nblint': ('fleet.html#fleet_nblint', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_push': ('fleet.html#fleet_push', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_scan': ('fleet.html#fleet_scan', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_status': ('fleet.html#fleet_status', 'nbdevAuto/fleet.py'),
@@ -64,6 +67,9 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet.format_summary': ('fleet.html#format_summary', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.init_new': ('fleet.html#init_new', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.is_published': ('fleet.html#is_published', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.lint_notebook': ('fleet.html#lint_notebook', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.lint_one': ('fleet.html#lint_one', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.lint_totals': ('fleet.html#lint_totals', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.looks_secret': ('fleet.html#looks_secret', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.notebook_text': ('fleet.html#notebook_text', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.parse_verb': ('fleet.html#parse_verb', 'nbdevAuto/fleet.py'),
@@ -74,6 +80,7 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet.scan_one': ('fleet.html#scan_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.script_path': ('fleet.html#script_path', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.status_one': ('fleet.html#status_one', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.unlinked_from_index': ('fleet.html#unlinked_from_index', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.update_one': ('fleet.html#update_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.upload_one': ('fleet.html#upload_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.usage': ('fleet.html#usage', 'nbdevAuto/fleet.py'),

{nbdevauto-0.3.75 → nbdevauto-0.3.77}/nbdevAuto/fleet.py

@@ -6,11 +6,13 @@ Docs: https://bthek1.github.io/nbdevAuto/fleet.html.md"""
 
 # %% auto #0
 __all__ = ['RAW_DOC_SUFFIXES', 'KNOWN_TOKEN', 'SECRET_PATTERNS', 'ALLOW', 'RENDERED_SUFFIXES', 'TEXT_SUFFIXES', 'EMAIL',
-           'RESERVED_DOMAINS', 'BULK_EMAIL_MIN', 'VERBS', 'Skip', 'Outcome', 'Report', 'format_summary', 'walk',
-           'finish', 'script_path', 'upload_one', 'fleet_upload', 'ahead_of_remote', 'push_one', 'copy_one',
-           'fleet_push', 'fleet_copy', 'status_one', 'fleet_status', 'fetch_all', 'update_one', 'dirty_submodules',
-           'init_new', 'pull_root', 'fleet_update', 'fleet_sync', 'looks_secret', 'notebook_text', 'is_published',
-           'scan_file', 'scan_one', 'fleet_scan', 'parse_verb', 'usage', 'run_verb', 'fleet']
+           'RESERVED_DOMAINS', 'BULK_EMAIL_MIN', 'NB_RULES', 'DLTASK_SECTIONS', 'DLTASK_NOT_TASKS', 'LARGE_NB_BYTES',
+           'NOT_PUBLISHED', 'VERBS', 'Skip', 'Outcome', 'Report', 'format_summary', 'walk', 'finish', 'script_path',
+           'upload_one', 'fleet_upload', 'ahead_of_remote', 'push_one', 'copy_one', 'fleet_push', 'fleet_copy',
+           'status_one', 'fleet_status', 'fetch_all', 'update_one', 'dirty_submodules', 'init_new', 'pull_root',
+           'fleet_update', 'fleet_sync', 'looks_secret', 'notebook_text', 'is_published', 'scan_file', 'scan_one',
+           'fleet_scan', 'lint_notebook', 'unlinked_from_index', 'lint_one', 'lint_totals', 'fleet_nblint',
+           'parse_verb', 'usage', 'run_verb', 'fleet']
 
 # %% ../nbs/04_Fleet.ipynb #d361ee5f-9f09-464f-aec6-0e27cec90fe9
 import json
@@ -21,6 +23,7 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from pathlib import Path
+from urllib.parse import unquote
 
 from fastcore.script import *
 
@@ -354,13 +357,20 @@ def init_new(root=None, dry_run=False):
     return new
 
 def pull_root(base):
-    "Fast-forward the superproject itself, with the advice its usual failure needs"
+    "Fast-forward the superproject itself, with the advice its failure actually needs"
     try: return _run("git", "-C", str(base), "pull", "--ff-only").strip().splitlines()[-1:]
     except StepFailed as e:
-        raise StepFailed(e.cmd, e.code, f"{e.output}\n"
-            "If it complains about submodule paths, the recorded gitlinks drifted.\n"
-            "Discard them (the update step re-advances them anyway):\n"
-            "  git checkout -- $(git diff --name-only)") from None
+        # Two failures, two fixes. Discarding gitlinks does nothing for a diverged branch,
+        # and a rebase does nothing for drifted gitlinks.
+        if "Not possible to fast-forward" in e.output:
+            hint = ("Local main and origin/main both have new commits.\n"
+                    "Replay yours on top of the remote, then re-run:\n"
+                    "  git pull --rebase && just sync")
+        else:
+            hint = ("If it complains about submodule paths, the recorded gitlinks drifted.\n"
+                    "Discard them (the update step re-advances them anyway):\n"
+                    "  git checkout -- $(git diff --name-only)")
+        raise StepFailed(e.cmd, e.code, f"{e.output}\n{hint}") from None
 
 # %% ../nbs/04_Fleet.ipynb #8fb7bc91-d79a-4c1a-bb81-e28b3a77b118
 def fleet_update(
@@ -568,10 +578,187 @@ def fleet_scan(
     sys.exit(1 if published or (strict and rep.ran) else 0)
 
 
+# %% ../nbs/04_Fleet.ipynb #7a605ea9
+# (key, label, issue #24 phase). The phase is what a count is FOR: each phase of the
+# cleanup drives its own rows to 0, so the table doubles as the plan's progress bar.
+NB_RULES = [
+    ("mermaid_plain",     "Plain ```mermaid fence (raw source on the site)", 1),
+    ("title_leading_rule","Title cell starts with --- (breaks nbdev-test)",   1),
+    ("title_no_h1",       "Title cell has no # H1",                          1),
+    ("index_unlinked",    "Public notebook missing from index.ipynb",        1),
+    ("mermaid_braced",    "Braced {mermaid} not yet an .svg file",           2),
+    ("svg_in_index",      "Inline <svg> in index.ipynb (README strips it)",  2),
+    ("title_code",        "Title cell is a code cell",                       3),
+    ("title_no_desc",     "Title cell has no > description",                 3),
+    ("no_skip_showdoc",   "Concept notebook without skip_showdoc",           3),
+    ("non_ascii",         "Non-ASCII characters in code cells",              4),
+    ("naive_datetime",    "Naive now() or utcnow(), no timezone",            4),
+    ("bad_filename",      "Filename with a space or &",                      4),
+    ("dl_matplotlib",     "DL_methods chart in matplotlib",                  5),
+    ("dltask_sections",   "DL_tasks notebook missing a required section",    5),
+    ("dltask_no_require", "DL_tasks live-capture demo without require(...)", 5),
+    ("stale_outputs",     "skip_exec notebook carrying outputs",             6),
+    ("large_file",        "Notebook over 3 MB",                              6),
+    ("multi_h2",          "Markdown cell with several ## sections",          7),
+    ("no_trailing_rule",  "## cell without a trailing ---",                  7),
+]
+# Matched case-insensitively against the `##` headings, as substrings, so "Datasets"
+# also accepts "Datasets and splits". From dl-task-notebook-format.instructions.md.
+DLTASK_SECTIONS = ["What is", "Real-world use cases", "Evaluation", "Datasets",
+                   "Setup", "benchmark", "Common frameworks", "Going further"]
+DLTASK_NOT_TASKS = ("Theory", "Frameworks")
+LARGE_NB_BYTES = 3_000_000
+# Never a site, so never held to the publishing standard (see special-submodules).
+NOT_PUBLISHED = ("Personal/bthek1_career",)
+
+def _src(c):
+    s = c.get("source", "")
+    return "".join(s) if isinstance(s, list) else s
+
+def lint_notebook(path, sub=""):
+    """{rule key: count} for one notebook. A count is the number of offending items
+    (diagrams, characters, cells); 1 for a yes/no rule. Empty means it meets the standard."""
+    out = {}
+    name = path.name
+    if " " in name or "&" in name: out["bad_filename"] = 1
+    try: nb = json.loads(path.read_text(encoding="utf-8"))
+    except Exception: return out
+    cells = nb.get("cells", [])
+    if not cells: return out
+    code = [_src(c) for c in cells if c.get("cell_type") == "code"]
+    md   = [_src(c) for c in cells if c.get("cell_type") == "markdown"]
+    allcode, allmd = "\n".join(code), "\n".join(md)
+    exports = re.search(r"^#\|\s*(export|exporti|exports|default_exp)\b", allcode, re.M)
+    first = _src(cells[0]).strip()
+
+    if cells[0].get("cell_type") != "markdown": out["title_code"] = 1
+    else:
+        if first.startswith("---"): out["title_leading_rule"] = 1
+        if not re.match(r"^(---[\s\S]*?---\s*)?#\s+\S", first): out["title_no_h1"] = 1
+        if not re.search(r"^>\s*\S", first, re.M): out["title_no_desc"] = 1
+        if not exports and "skip_showdoc: true" not in first: out["no_skip_showdoc"] = 1
+
+    # The title cell is checked above; a `##` packed into it is a title problem first.
+    multi = sum(len(re.findall(r"^##\s", m, re.M)) > 1 for m in md[1:])
+    trail = sum(bool(re.search(r"^##\s", m, re.M)) and not m.rstrip().endswith("---")
+                for m in md[1:])
+    if multi: out["multi_h2"] = multi
+    if trail: out["no_trailing_rule"] = trail
+
+    plain = len(re.findall(r"^```mermaid\s*$", allmd, re.M))
+    brace = len(re.findall(r"^```\{mermaid\}", allmd, re.M))
+    if plain: out["mermaid_plain"] = plain
+    if brace: out["mermaid_braced"] = brace
+    if name == "index.ipynb" and re.search(r"<svg[\s>]", allmd): out["svg_in_index"] = 1
+
+    nonascii = sum(ord(ch) > 127 for ch in allcode)
+    if nonascii: out["non_ascii"] = nonascii
+    if re.search(r"datetime\.utcnow\(|\b(?:dt|datetime)\.now\(\s*\)", allcode):
+        out["naive_datetime"] = 1
+
+    if sub == "DL/DL_methods" and re.search(
+            r"^\s*(import|from)\s+(matplotlib|seaborn|plotly)", allcode, re.M):
+        out["dl_matplotlib"] = 1
+    # Only a real task notebook: not the index, a library notebook, a stub, or the
+    # Theory/ and Frameworks/ reference pages, which have no task to benchmark.
+    if (sub == "DL/DL_tasks" and name != "index.ipynb" and not exports and len(cells) > 3
+            and path.parent.name not in DLTASK_NOT_TASKS):
+        heads = "\n".join(re.findall(r"^##\s+.*$", allmd, re.M)).lower()
+        missing = [s for s in DLTASK_SECTIONS if s.lower() not in heads]
+        if missing: out["dltask_sections"] = len(missing)
+        if re.search(r"cv2\.VideoCapture|sounddevice|pyaudio", allcode) and "require(" not in allcode:
+            out["dltask_no_require"] = 1
+
+    # DL notebooks keep outputs on purpose (benchmarks, demos), so only elsewhere is a
+    # skip_exec notebook with outputs a sign of stale results.
+    outs = sum(len(c.get("outputs", [])) for c in cells if c.get("cell_type") == "code")
+    if outs and "skip_exec: true" in first and not sub.startswith("DL/"):
+        out["stale_outputs"] = outs
+    if path.stat().st_size > LARGE_NB_BYTES: out["large_file"] = 1
+    return out
+
+def _public_notebooks(nbs):
+    "Every published notebook under `nbs`, relative paths. `_`/`.` dirs are build output"
+    found = []
+    for p in sorted(nbs.rglob("*.ipynb")):
+        rel = p.relative_to(nbs)
+        if any(part.startswith((".", "_")) for part in rel.parts[:-1]): continue
+        if p.name.lower().startswith("p_"): continue
+        found.append(rel)
+    return found
+
+def unlinked_from_index(nbs, public=None):
+    "Public notebooks (as posix paths) that `nbs/index.ipynb` does not link to"
+    idx = nbs/"index.ipynb"
+    if not idx.exists(): return []
+    public = public if public is not None else _public_notebooks(nbs)
+    body = "".join(_src(c) for c in json.loads(idx.read_text(encoding="utf-8")).get("cells", []))
+    links = {unquote(l).split("#")[0] for l in re.findall(r"\]\(([^)\s]+)\)", body)
+             if not l.startswith(("http", "#"))}
+    return [r.as_posix() for r in public
+            if r.name != "index.ipynb" and r.as_posix() not in links]
+
+def lint_one(path, sub=None, nbs="nbs"):
+    "{notebook path relative to the superproject: {rule key: count}} for one submodule"
+    name = sub.path if sub else path.name
+    root = path/nbs
+    if name in NOT_PUBLISHED: raise Skip("not published")
+    if not root.is_dir(): raise Skip("without nbs")
+    public = _public_notebooks(root)
+    found = {}
+    for rel in public:
+        hits = lint_notebook(root/rel, name)
+        if hits: found[f"{name}/{nbs}/{rel.as_posix()}"] = hits
+    for rel in unlinked_from_index(root, public):
+        found.setdefault(f"{name}/{nbs}/{rel}", {})["index_unlinked"] = 1
+    return found
+
+def lint_totals(found):
+    "{rule key: (notebooks, items)} over the output of `lint_one`, every rule present"
+    tot = {k: [0, 0] for k, _, _ in NB_RULES}
+    for hits in found.values():
+        for k, n in hits.items(): tot[k][0] += 1; tot[k][1] += n
+    return {k: tuple(v) for k, v in tot.items()}
+
+def fleet_nblint(
+    detail:bool = False,  # List the offending notebooks under each rule
+    phase:int   = None,   # Only the rules for this phase of issue #24
+    strict:bool = False,  # Exit 1 when any rule is above 0
+    root:str    = None,   # Superproject root (default: nearest ancestor with .gitmodules)
+):
+    "Count public notebooks that break the notebook standard, one row per rule"
+    from rich.table import Table
+    found = {}
+    def _collect(p, s):
+        found.update(lint_one(p, s))
+        raise Skip("linted")   # counted below in one table, not narrated per submodule
+    rep = walk(_collect, root=root, workers=8, style="line")
+    if rep.failed: sys.exit(finish(rep, "linted"))
+    rules = [r for r in NB_RULES if phase is None or r[2] == phase]
+    tot = lint_totals(found)
+    t = Table(title="Notebook standard (issue #24)", title_justify="left")
+    for col, kw in (("Phase", {"justify": "right"}), ("Rule", {}),
+                    ("Notebooks", {"justify": "right"}), ("Items", {"justify": "right"})):
+        t.add_column(col, **kw)
+    for key, label, ph in rules:
+        nbks, items = tot[key]
+        style = "green" if nbks == 0 else None
+        t.add_row(str(ph), escape(label), str(nbks), str(items), style=style)
+    console.print(t)
+    left = sum(tot[k][0] for k, _, _ in rules)
+    console.print(f"[bold]{len(found)} notebooks flagged, {left} rule hits in view[/bold]")
+    if detail:
+        for key, label, _ in rules:
+            hits = sorted((p, h[key]) for p, h in found.items() if key in h)
+            if not hits: continue
+            console.print(f"\n[bold cyan]{escape(label)}[/bold cyan]")
+            for p, n in hits: console.print(f"  {escape(p)} [dim]({n})[/dim]")
+    sys.exit(1 if strict and left else 0)
+
 # %% ../nbs/04_Fleet.ipynb #0281bd2b-91ba-4e49-90dc-d1a95d54d5a6
 VERBS = {"upload": fleet_upload, "push": fleet_push, "copy": fleet_copy,
          "status": fleet_status, "update": fleet_update, "sync": fleet_sync,
-         "scan": fleet_scan}
+         "scan": fleet_scan, "nblint": fleet_nblint}
 
 def parse_verb(argv):
     "(verb, remaining args) for `fleet ...`, or None when `argv` does not name a verb"
@@ -608,7 +795,12 @@ def fleet():
     if not picked:
         print(usage())
         sys.exit(0 if sys.argv[1:2] in ([], ["-h"], ["--help"]) else 2)
-    return run_verb(*picked)
+    # `_step` has already printed the failing command's output, so a traceback on top of
+    # it only buries the hint. One line and the exit code are all that is left to say.
+    try: return run_verb(*picked)
+    except StepFailed as e:
+        console.print(f"[bold red]FAILED[/bold red] [dim]{escape(str(e))}[/dim]")
+        sys.exit(1)
 
 # %% ../nbs/04_Fleet.ipynb #7adb89c6-6788-4241-acff-53cb4566b5ce
 if __name__ == "__main__": fleet()

{nbdevauto-0.3.75 → nbdevauto-0.3.77/nbdevAuto.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.75
+Version: 0.3.77
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.75/nbdevAuto/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.75"