nbdevAuto 0.3.70__tar.gz → 0.3.73__tar.gz

{nbdevauto-0.3.70/nbdevAuto.egg-info → nbdevauto-0.3.73}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.70
+Version: 0.3.73
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.73/nbdevAuto/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.73"

{nbdevauto-0.3.70 → nbdevauto-0.3.73}/nbdevAuto/_modidx.py

@@ -44,6 +44,8 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet._count': ('fleet.html#_count', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._fail_parts': ('fleet.html#_fail_parts', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._git_ok': ('fleet.html#_git_ok', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._hits': ('fleet.html#_hits', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet._line_of': ('fleet.html#_line_of', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._porcelain': ('fleet.html#_porcelain', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet._render': ('fleet.html#_render', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.ahead_of_remote': ('fleet.html#ahead_of_remote', 'nbdevAuto/fleet.py'),
@@ -54,16 +56,22 @@ d = { 'settings': { 'branch': 'main',
                                  'nbdevAuto.fleet.fleet': ('fleet.html#fleet', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_copy': ('fleet.html#fleet_copy', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_push': ('fleet.html#fleet_push', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.fleet_scan': ('fleet.html#fleet_scan', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_status': ('fleet.html#fleet_status', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_sync': ('fleet.html#fleet_sync', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_update': ('fleet.html#fleet_update', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.fleet_upload': ('fleet.html#fleet_upload', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.format_summary': ('fleet.html#format_summary', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.init_new': ('fleet.html#init_new', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.is_published': ('fleet.html#is_published', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.looks_secret': ('fleet.html#looks_secret', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.notebook_text': ('fleet.html#notebook_text', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.parse_verb': ('fleet.html#parse_verb', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.pull_root': ('fleet.html#pull_root', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.push_one': ('fleet.html#push_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.run_verb': ('fleet.html#run_verb', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.scan_file': ('fleet.html#scan_file', 'nbdevAuto/fleet.py'),
+                                 'nbdevAuto.fleet.scan_one': ('fleet.html#scan_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.script_path': ('fleet.html#script_path', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.status_one': ('fleet.html#status_one', 'nbdevAuto/fleet.py'),
                                  'nbdevAuto.fleet.update_one': ('fleet.html#update_one', 'nbdevAuto/fleet.py'),

{nbdevauto-0.3.70 → nbdevauto-0.3.73}/nbdevAuto/fleet.py

@@ -5,12 +5,15 @@ Docs: https://bthek1.github.io/nbdevAuto/fleet.html.md"""
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_Fleet.ipynb.
 
 # %% auto #0
-__all__ = ['VERBS', 'Skip', 'Outcome', 'Report', 'format_summary', 'walk', 'finish', 'script_path', 'upload_one', 'fleet_upload',
-           'ahead_of_remote', 'push_one', 'copy_one', 'fleet_push', 'fleet_copy', 'status_one', 'fleet_status',
-           'fetch_all', 'update_one', 'dirty_submodules', 'init_new', 'pull_root', 'fleet_update', 'fleet_sync',
-           'parse_verb', 'usage', 'run_verb', 'fleet']
+__all__ = ['RAW_DOC_SUFFIXES', 'KNOWN_TOKEN', 'SECRET_PATTERNS', 'ALLOW', 'RENDERED_SUFFIXES', 'TEXT_SUFFIXES', 'EMAIL',
+           'RESERVED_DOMAINS', 'BULK_EMAIL_MIN', 'VERBS', 'Skip', 'Outcome', 'Report', 'format_summary', 'walk',
+           'finish', 'script_path', 'upload_one', 'fleet_upload', 'ahead_of_remote', 'push_one', 'copy_one',
+           'fleet_push', 'fleet_copy', 'status_one', 'fleet_status', 'fetch_all', 'update_one', 'dirty_submodules',
+           'init_new', 'pull_root', 'fleet_update', 'fleet_sync', 'looks_secret', 'notebook_text', 'is_published',
+           'scan_file', 'scan_one', 'fleet_scan', 'parse_verb', 'usage', 'run_verb', 'fleet']
 
 # %% ../nbs/04_Fleet.ipynb #d361ee5f-9f09-464f-aec6-0e27cec90fe9
+import json
 import re
 import shutil
 import subprocess
@@ -397,9 +400,178 @@ def fleet_sync(
     rep = walk(lambda p, s: update_one(p, s, dry_run=dry_run), root=base, workers=1, style="line")
     sys.exit(finish(rep, "would update" if dry_run else "updated"))
 
+# %% ../nbs/04_Fleet.ipynb #a2907233
+# A finding's severity is which of these two lists a file falls in. `_quarto.yml`
+# renders every notebook EXCEPT `p_*`, so "published" is a filename test, not a
+# guess: a hit in a published file is live on the web, a hit in a `p_` one is
+# only in git.
+RAW_DOC_SUFFIXES = {".pdf", ".csv", ".ods", ".xlsx", ".xls", ".db", ".sqlite3"}
+
+# Vendor prefixes that are a secret by construction, whatever they are assigned to.
+KNOWN_TOKEN = re.compile(
+    r"\b(?:AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{20,}"
+    r"|sk-[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,})\b")
+
+SECRET_PATTERNS = [
+    # The header alone is not a key. Every VPN and SSH page in the fleet shows the
+    # armour wrapped around a "(Insert Client Key)" placeholder, so real base64
+    # key material has to follow before this counts.
+    ("private key",   re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----\s*[\\n\s]*"
+                                 r"[A-Za-z0-9+/]{40,}")),
+    ("known token",   KNOWN_TOKEN),
+    ("password hash", re.compile(r"\b(?:pbkdf2_sha256|argon2[a-z]*)\$"
+                                 r"[^\s\"']*\$[A-Za-z0-9+/]{16,}")),
+    ("tax file no",   re.compile(r"(?i)\btax file number\b[^\n]{0,20}?\b\d{8,9}\b")),
+    ("passport no",   re.compile(r"(?i)\bpassport\b[^\n]{0,30}?\b[A-Z]{1,2}\d{7}\b")),
+    ("hex secret",    re.compile(r"(?i)\b(?:secret|token|api[_ -]?key|client[_ -]?id)\b"
+                                 r"\W{0,4}\b([0-9a-f]{32,})\b")),
+    ("credential",    re.compile(r"(?i)\b(?:password|passwd|api[_ -]?key|secret|token)\b"
+                                 r"\s*[:=]\s*[\"']?([^\s\"',}{]{8,})")),
+]
+
+def looks_secret(v):
+    """Is this captured value a literal secret, or just code that mentions one?
+
+    The credential regex cannot tell `token = 'b4bTbQ...'` from
+    `password = forms.CharField(...)`, and the second is what every Django,
+    Stripe and Grafana page in the fleet is FULL of. Ruling those out is the
+    whole difference between a scan worth running and one everybody ignores.
+
+    So the value has to survive four filters: be a plain literal rather than an
+    expression, not be an identifier or env-var name, not already be masked, and
+    mix character classes the way a generated secret does and a word does not."""
+    if len(v) < 16:                                     return False
+    if not re.fullmatch(r"[A-Za-z0-9_\-+/=.~]+", v):    return False  # an expression
+    if re.fullmatch(r"[A-Z0-9_]+", v):                  return False  # ENV_VAR_NAME
+    if re.fullmatch(r"[A-Za-z_][\w.]*", v) and "." in v:return False  # settings.API_KEY
+    if re.search(r"[*x]{4,}|\.{3,}", v):                return False  # already masked
+    classes = sum(bool(re.search(c, v)) for c in (r"[a-z]", r"[A-Z]", r"\d"))
+    return classes >= 3
+
+# An escape hatch for the deliberately secret-shaped: this module's own test
+# vectors, and a doc that has to show what a real token looks like. Put the
+# marker on the same line. It is a line comment, so it survives `nbdev-export`.
+ALLOW = "scan:allow"
+
+def _line_of(text, pos):
+    "The whole line containing offset `pos`, so an inline marker can be seen"
+    return text[text.rfind("\n", 0, pos) + 1 : (text.find("\n", pos) + 1 or None)]
+
+def _hits(text):
+    "Every secret-pattern name matching `text`, in pattern order, without duplicates"
+    out = []
+    for name, rx in SECRET_PATTERNS:
+        for m in rx.finditer(text):
+            if ALLOW in _line_of(text, m.start()): continue
+            # A pattern with a group captures the VALUE, so it can be judged.
+            # A pattern without one matched something already unambiguous.
+            if m.groups() and name == "credential" and not looks_secret(m.group(1)): continue
+            out.append(name); break
+    return out
+
+def notebook_text(path):
+    """Cell sources plus TEXT outputs, with image payloads dropped.
+
+    Scanning outputs is the point, not an extra: a code cell rewritten to drop its
+    figures still carries the OLD numbers in its stored output, and Quarto
+    publishes that output verbatim. That is the exact leak issue #18 found.
+
+    Skipping `image/*` is not just speed. A notebook of plots is mostly base64,
+    and feeding megabytes of it to these regexes manufactures false positives out
+    of pure entropy."""
+    try: nb = json.loads(path.read_text(encoding="utf-8", errors="replace"))
+    except Exception: return ""
+    parts = []
+    for c in nb.get("cells", []):
+        parts.append("".join(c.get("source", [])))
+        for o in c.get("outputs", []):
+            parts.append("".join(o.get("text", [])))
+            for mime, val in (o.get("data") or {}).items():
+                if not mime.startswith("image/"):
+                    parts.append("".join(val) if isinstance(val, list) else str(val))
+    return "\n".join(parts)
+
+# Rendered by `_quarto.yml`, so a hit here is on the public web. Everything else
+# tracked is only in git, which is a different and smaller problem.
+RENDERED_SUFFIXES = {".ipynb", ".md"}
+# Read for content. A binary document cannot be scanned without dragging in a
+# parser, so it gets counted instead of judged.
+TEXT_SUFFIXES = RENDERED_SUFFIXES | {".py", ".yml", ".yaml", ".txt", ".json", ".csv",
+                                     ".ini", ".cfg", ".toml", ".sh", ".env"}
+
+EMAIL = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
+# RFC 2606 reserves these for documentation, so a page full of them is a tutorial,
+# not a contact list. Without this every mail and auth page in the fleet reports.
+RESERVED_DOMAINS = re.compile(r"(?i)@(?:[\w.-]+\.)?(?:example\.(?:com|org|net)|"
+                              r"test|localhost|invalid|example)$")
+# One address is a contact detail; a column of them is somebody's user table.
+BULK_EMAIL_MIN = 5
+
+def is_published(path):
+    "Does the site render this file? `p_*` is excluded by `project.render` in `_quarto.yml`"
+    return path.suffix in RENDERED_SUFFIXES and not path.name.startswith("p_")
+
+def scan_file(path):
+    "Pattern names found in `path`, or [] for nothing. Non-text files are never scanned"
+    if path.suffix.lower() not in TEXT_SUFFIXES: return []
+    text = notebook_text(path) if path.suffix == ".ipynb" else \
+           path.read_text(encoding="utf-8", errors="replace")
+    names = _hits(text)
+    addrs = {m.group(0).lower() for m in EMAIL.finditer(text)
+             if not RESERVED_DOMAINS.search(m.group(0))}
+    if len(addrs) >= BULK_EMAIL_MIN: names.append(f"{len(addrs)} email addresses")
+    return names
+
+
+# %% ../nbs/04_Fleet.ipynb #55d1232c
+def scan_one(path, sub=None, detail=False, nbs="nbs"):
+    """Secret and personal-data findings for one submodule.
+
+    Only TRACKED files are scanned. An untracked file is on this disk and nowhere
+    else, which is exactly where `.gitignore` is supposed to put a payslip, so
+    flagging it would punish the fix.
+
+    Binary documents are counted rather than read. The count is informational and
+    never fails the scan: a tracked PDF may be a payslip or may be a datasheet,
+    and this cannot tell without a parser it has no business importing."""
+    root = path/nbs
+    if not root.is_dir(): raise Skip("without nbs")
+    tracked = [l for l in git(path, "ls-files", "-z", nbs).split("\0") if l]
+    pub, tracked_hits, docs = [], [], 0
+    for rel in sorted(tracked):
+        f = path/rel
+        if not f.is_file(): continue
+        if f.suffix.lower() in RAW_DOC_SUFFIXES and f.suffix.lower() != ".csv":
+            docs += 1; continue
+        names = scan_file(f)
+        if not names: continue
+        (pub if is_published(f) else tracked_hits).append(f"{rel}: {', '.join(names)}")
+    if not (pub or tracked_hits): raise Skip("clean")
+    line = f"{len(pub):>3} published {len(tracked_hits):>3} tracked {docs:>4} binary docs"
+    if not detail: return [line]
+    return [line] + [f"PUBLISHED {h}" for h in pub] + [f"tracked   {h}" for h in tracked_hits]
+
+def fleet_scan(
+    detail:bool = False,  # List every finding, not just the per-submodule counts
+    strict:bool = False,  # Exit 1 on any finding, not only on a published one
+    root:str    = None,   # Superproject root (default: nearest ancestor with .gitmodules)
+):
+    "Find personal data and credentials in tracked files, published ones first"
+    style = "block" if detail else "line"
+    rep = walk(lambda p, s: scan_one(p, s, detail=detail), root=root,
+               workers=8, style=style)
+    code = finish(rep, "with findings")
+    if code: sys.exit(code)
+    # A published hit is live on the web and always fails. A tracked-but-unpublished
+    # one is a cleanup job, so it only fails under --strict.
+    published = sum(int(o.detail[0].split()[0]) for o in rep.ran if o.detail)
+    sys.exit(1 if published or (strict and rep.ran) else 0)
+
+
 # %% ../nbs/04_Fleet.ipynb #0281bd2b-91ba-4e49-90dc-d1a95d54d5a6
 VERBS = {"upload": fleet_upload, "push": fleet_push, "copy": fleet_copy,
-         "status": fleet_status, "update": fleet_update, "sync": fleet_sync}
+         "status": fleet_status, "update": fleet_update, "sync": fleet_sync,
+         "scan": fleet_scan}
 
 def parse_verb(argv):
     "(verb, remaining args) for `fleet ...`, or None when `argv` does not name a verb"

{nbdevauto-0.3.70 → nbdevauto-0.3.73}/nbdevAuto/github.py

@@ -179,7 +179,8 @@ async def gather_status(subs, token=None, root=None):
 
 # %% ../nbs/03_GitHub.ipynb #d4d8fd92-b4a2-4479-bd76-df3cc7fc9f77
 FAILED  = {"failure", "timed_out", "cancelled", "action_required", "startup_failure"}
-RUNNING = {"in_progress", "queued", "requested", "waiting", "pending"}
+RUNNING = {"in_progress", "queued", "requested", "waiting", "pending",
+           "building"}  # "building" is the Pages build in flight
 BOLD, RED, YELLOW, CYAN, DIM, OFF = "\033[1m", "\033[31m", "\033[33m", "\033[36m", "\033[2m", "\033[0m"
 
 def _state(v):
@@ -214,7 +215,12 @@ def failures(rows):
 
 def has_failure(rows):
     "True when any workflow failed or a Pages build errored"
-    return bool(failures(rows)) or any(r["built"] not in (None, "built") for r in rows)
+    # A Pages build in flight is not a verdict: "building" is the state every
+    # healthy deploy passes through, and counting it red made `just ci` exit 1
+    # on a fleet that was merely mid-build, with no red row in the table to
+    # explain it. Same shape as the `_latest_run` bug in issue #8: a transient
+    # state read as a result. Unknown states still count as failures.
+    return bool(failures(rows)) or any(r["built"] not in (None, "built", *RUNNING) for r in rows)
 
 # %% ../nbs/03_GitHub.ipynb #25434173-a101-4ecd-802a-f349f6233fa3
 def git(path, *args):

{nbdevauto-0.3.70 → nbdevauto-0.3.73/nbdevAuto.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nbdevAuto
-Version: 0.3.70
+Version: 0.3.73
 Summary: automating nbdev
 Author-email: Benedict Thekkel <bthekkel1@gmail.com>
 License-Expression: Apache-2.0

nbdevauto-0.3.70/nbdevAuto/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.70"