sllmwiki 1.2.0__py3-none-any.whl

sllmwiki/__init__.py

@@ -0,0 +1,8 @@
+"""sllmwiki: the Structural LLM Wiki's code. The plugin's instructions run it as
+`sllmwiki <command>`; a wiki's GitHub Actions run it from this repository at the tag."""
+
+import os
+
+__version__ = '1.2.0'  # the plugin's version too (.claude-plugin/plugin.json); a test keeps them equal
+
+TEMPLATES = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')  # what init installs, the narrative templates, the book's

sllmwiki/__main__.py

@@ -0,0 +1,7 @@
+"""python3 -m sllmwiki <command> ...: the same as the sllmwiki command."""
+
+import sys
+
+from sllmwiki.cli import main
+
+sys.exit(main())

sllmwiki/cli.py

@@ -0,0 +1,31 @@
+"""sllmwiki: the Structural LLM Wiki's commands.
+
+  sllmwiki <command> [<args>]   run one; sllmwiki <command> --help explains it
+  sllmwiki --version            this package's version, the plugin's too
+"""
+
+import importlib
+import sys
+
+from sllmwiki import __version__
+from sllmwiki.lib.util import setup_stdio
+
+COMMANDS = ('book', 'check-upstream', 'cited-sections', 'contributors', 'doctor', 'fetch-source', 'finish-narrative', 'init', 'manifest',
+            'narrative-format', 'publish', 'reading-plan', 'release', 'review-checklist', 'search', 'tag-tree', 'wiki-check')
+
+
+def main(argv=None):
+    """Run `sllmwiki <command> <args>`; its exit code. 2 for an unknown command."""
+    argv = sys.argv[1:] if argv is None else list(argv)
+    setup_stdio()
+    if argv[:1] == ['--version']:
+        print(f'sllmwiki {__version__}')
+        return 0
+    if not argv or argv[0] in ('-h', '--help'):
+        print(__doc__.strip() + '\n\ncommands: ' + ', '.join(COMMANDS))
+        return 0 if argv else 2
+    if argv[0] not in COMMANDS:
+        print(f"sllmwiki: no command {argv[0]!r}; the commands: {', '.join(COMMANDS)}", file=sys.stderr)
+        return 2
+    module = importlib.import_module('sllmwiki.commands.' + argv[0].replace('-', '_'))
+    return module.main(argv[1:])

sllmwiki/commands/__init__.py

@@ -0,0 +1 @@
+"""One module per `sllmwiki <command>`, each with main(argv)."""

sllmwiki/commands/book.py

@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""book: the wiki as a book (docs/book_design.md).
+
+  sllmwiki book stage OUT [--name N] [--version V] [--date YYYY-MM-DD] [--commit C] [--source URL]
+      Writes the book as plain Markdown for Sphinx into the folder OUT, which
+      must be empty or not exist yet: index.md (the cover, the front page and
+      the contents), parts/<tag>.md, chapters/<slug>.md with their figures,
+      and the appendices (sources, glossary, open questions). Also writes
+      OUT/book.json: the parts, the chapter order, every anchor, every diagram
+      and formula to render, and the problems found. The wiki is only read.
+      --name (the files' name) defaults to the wiki's folder name, --version
+      to "preview", --date to today, --commit to the current commit and
+      --source (the wiki's web page, which the cover links to) to its origin
+      remote's. The cover's title and the author are Config.toml's title and
+      author ([structural-llm-wiki-release]), by default the name and the title.
+      Problems: BOOK_LINK (a link or anchor the book can't resolve),
+      BOOK_CYCLE (narratives built on each other in a cycle), BOOK_FIGURE (a
+      figure that isn't there). Exit code 1 when there are any.
+  sllmwiki book render STAGED --vendor DIR [--chrome PATH] [--cache DIR]
+      Draws every diagram and formula book.json lists as SVG, in headless
+      Chrome with the mermaid.js and MathJax in DIR (mermaid.min.js,
+      tex-svg-full.js), and puts the SVG in the staged pages in their place.
+      --chrome defaults to $CHROME, else the Chrome on PATH; --cache keeps
+      each SVG by its source and the libraries, so a rerun draws only what
+      changed. One that fails to draw stays as written and is a warning,
+      BOOK_RENDER, kept in book.json's warnings.
+  sllmwiki book build OUT --vendor DIR [--formats pdf,epub,html] [--chrome PATH] [--cache DIR]
+                [--sphinx CMD] [--name N] [--version V] [--date D] [--commit C] [--source URL]
+      stage into OUT/staged, render, then Sphinx (--sphinx, default
+      sphinx-build, with MyST-Parser) with the package's templates/book/
+      settings, and writes OUT/<name>-<version>.html: one file with every
+      stylesheet, image and diagram in it; for pdf, Chrome prints that file
+      to OUT/<name>-<version>.pdf (A4, page numbers, the title as the running
+      head); for epub, Sphinx's EPUB 3 to OUT/<name>-<version>.epub, from the
+      same rendered pages. --formats (pdf, epub, html) defaults to the ones
+      Config.toml's [structural-llm-wiki-release] turns on; with none, the
+      book is staged and no file is written. Problems and warnings are listed
+      (Sphinx's are BOOK_SPHINX) and don't stop the book; exit code 1 when a
+      format fails.
+  sllmwiki book parse --vendor DIR [--base REF] [--chrome PATH]
+      Checks that each diagram and formula parses, as GitHub draws them, with
+      the same mermaid.js (mermaid.parse) and MathJax the book uses: with
+      --base, only those a change brings, in the narratives changed since REF
+      (a pull request's base), and not in REF's version of them; without it,
+      every one. Reports DIAGRAM_SYNTAX and MATH_SYNTAX with the file, the
+      line and the parser's message (exit code 1). Nothing to check needs no
+      Chrome.
+  sllmwiki book issues STAGED
+      The release notes' "Known issues" section for a built book: its
+      problems, warnings and failed formats from book.json, in the wiki's
+      language; nothing when there are none.
+
+Options: --json, --root <wiki>.
+Exit code: as above; 2 on a usage error.
+"""
+
+import argparse
+import datetime
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+
+sys.dont_write_bytecode = True
+
+from sllmwiki import TEMPLATES  # noqa: E402
+from sllmwiki.lib import book, bookbuild, compat, config, language, layout  # noqa: E402
+from sllmwiki.lib.layout import Cli, entry  # noqa: E402
+from sllmwiki.lib.util import dumps, write_text  # noqa: E402
+
+cli = Cli('sllmwiki book', 'usage: sllmwiki book stage|render|build ... (sllmwiki book --help)')
+FORMATS = ('pdf', 'epub', 'html')  # what build makes, as Config.toml's <format>_book names it
+TEMPLATE = os.path.join(TEMPLATES, 'book')
+
+
+def _git(root, *args):
+    """git's output in root, or '' when it fails."""
+    try:
+        r = subprocess.run(['git', '-C', root, *args], capture_output=True, text=True, check=False)
+    except OSError:  # pragma: no cover (no git)
+        return ''
+    return r.stdout.strip() if r.returncode == 0 else ''
+
+
+def _empty(cli_out):
+    out = os.path.abspath(cli_out)
+    if os.path.exists(out) and (not os.path.isdir(out) or os.listdir(out)):
+        cli.usage(f'{cli_out} must be an empty folder, or not exist yet')
+    return out
+
+
+def stage(root, out, args):
+    """Write the staged book into out; its book.json."""
+    release = config.get(root).release
+    staged = book.stage(root, args.name or os.path.basename(root), args.version, args.date,
+                        _git(root, 'rev-parse', '--short', 'HEAD') if args.commit is None else args.commit,
+                        book.web_url(_git(root, 'remote', 'get-url', 'origin')) if args.source is None else args.source or None,
+                        release['title'] or None, release['author'] or None)
+    for rel, text in staged.files.items():
+        write_text(os.path.join(out, rel), text)
+    for fig in staged.figures:
+        dest = os.path.join(out, 'chapters', fig)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        shutil.copyfile(os.path.join(root, layout.NARRATIVES, fig), dest)
+    _save(out, staged.manifest)
+    return staged.manifest
+
+
+def _save(staged, data):
+    write_text(os.path.join(staged, 'book.json'), json.dumps(data, indent=1, ensure_ascii=False) + '\n')
+
+
+def render(staged, vendor, chrome, cache):
+    """Draw book.json's diagrams and formulas and put them in the staged pages; book.json, with its warnings."""
+    with open(os.path.join(staged, 'book.json'), encoding='utf-8') as f:
+        data = json.load(f)
+    items = {r['id']: r for r in data['renders']}
+    libs = bookbuild.libraries(vendor)
+    svgs = {}
+    for rid in items:
+        path = os.path.join(cache, f'{rid}-{libs}.svg') if cache else None
+        if path and os.path.isfile(path):
+            with open(path, encoding='utf-8') as f:
+                svgs[rid] = f.read()
+    todo = [{'id': r['id'], 'kind': r['kind'], 'source': r['source']} for rid, r in items.items() if rid not in svgs]
+    if todo:
+        found = bookbuild.find_chrome(chrome)
+        if not found:
+            cli.fail('no Chrome found: pass --chrome or set $CHROME')
+        try:
+            drawn = bookbuild.draw(todo, vendor, found, staged)
+        except ValueError as e:
+            cli.fail(str(e))
+        for rid, svg in drawn.items():
+            if isinstance(svg, str):
+                svgs[rid] = bookbuild.clean(svg, items[rid]['source'])
+                if cache:
+                    write_text(os.path.join(cache, f'{rid}-{libs}.svg'), svgs[rid])
+            else:
+                svgs[rid] = svg
+    warnings = [w for w in data.get('warnings', []) if w['code'] != 'BOOK_RENDER']
+    for rid, svg in svgs.items():
+        if not isinstance(svg, str):
+            r = items[rid]
+            warnings.append({'code': 'BOOK_RENDER', 'where': r['files'][0],
+                             'message': f'a {r["kind"]} that doesn\'t draw stays as written: {svg["error"]}: {r["source"][:80]}'})
+    counts = {}
+    for folder, _, files in sorted(os.walk(staged)):
+        for name in sorted(files):
+            if name.endswith('.md'):
+                path = os.path.join(folder, name)
+                with open(path, encoding='utf-8') as f:
+                    text = f.read()
+                write_text(path, bookbuild.put_in(text, svgs, book.render_id, counts))
+    data['warnings'] = warnings
+    _save(staged, data)
+    return data
+
+
+def _sphinx(root, out, staged, data, sphinx, builder):
+    """Run Sphinx's builder on the rendered staged book into out/_build/<builder>: (its warnings, None) or (None, why it failed)."""
+    lang = bookbuild.sphinx_language(config.get(root).settings['language'])
+    build = os.path.join(out, '_build')
+    name = f'{data["name"]}-{data["version"]}'
+    cmd = [*shlex.split(sphinx), '-q', '-b', builder, '-c', TEMPLATE, '-d', os.path.join(build, 'doctrees'),
+           '-D', f'project={data["title"]}', '-D', f'version={data["version"]}', '-D', f'release={data["version"]}',
+           '-D', f'html_title={data["title"]} {data["version"]}',  # not Sphinx's "<project> <release> documentation"
+           '-D', f'copyright={data["author"]}', '-D', f'language={lang}', '-D', f'epub_basename={name}',
+           '-D', f'epub_author={data["author"]}', '-D', 'epub_publisher=sllmwiki', staged, os.path.join(build, builder)]
+    try:
+        run = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    except OSError as e:
+        return None, f'{sphinx} didn\'t run: {e}'
+    if run.returncode != 0:
+        return None, f'Sphinx failed: {(run.stderr or run.stdout).strip()[-2000:]}'
+    return [line for line in run.stderr.splitlines() if 'WARNING' in line or 'ERROR' in line], None
+
+
+def html_book(root, out, staged, data, sphinx):
+    """Sphinx's single-file HTML of the rendered staged book: (path, Sphinx's warnings), or (None, why it failed)."""
+    warnings, why = _sphinx(root, out, staged, data, sphinx, 'singlehtml')
+    folder = os.path.join(out, '_build', 'singlehtml')
+    if why or not os.path.isfile(os.path.join(folder, 'index.html')):
+        return None, why or 'Sphinx wrote no index.html'
+    path = os.path.join(out, f'{data["name"]}-{data["version"]}.html')
+    write_text(path, bookbuild.self_contained(folder))
+    return path, warnings
+
+
+def epub_book(root, out, staged, data, sphinx):
+    """Sphinx's EPUB 3 of the rendered staged book, finished (bookbuild.finish_epub): (path, Sphinx's warnings),
+    or (None, why it failed)."""
+    warnings, why = _sphinx(root, out, staged, data, sphinx, 'epub')
+    name = f'{data["name"]}-{data["version"]}'
+    built = os.path.join(out, '_build', 'epub', f'{name}.epub')
+    if why or not os.path.isfile(built):
+        return None, why or 'Sphinx wrote no EPUB'
+    path = os.path.join(out, f'{name}.epub')
+    bookbuild.finish_epub(built, path)
+    return path, warnings
+
+
+def to_parse(root, base=None):
+    """[(file, line, kind, source)] of the diagrams and formulas to check: with base, those in narratives
+    changed since base (the working tree against it, new files git doesn't track yet included) whose kind
+    and source aren't in base's version."""
+    folder = layout.NARRATIVES
+    if base:
+        names = (_git(root, 'diff', '--name-only', base, '--', folder).split('\n')
+                 + _git(root, 'ls-files', '--others', '--exclude-standard', '--', folder).split('\n'))
+    else:
+        names = [f'{folder}/{n}' for n in sorted(os.listdir(os.path.join(root, folder)))]
+    out = []
+    for rel in names:
+        path = os.path.join(root, rel)
+        if not (rel.endswith('.md') and os.path.dirname(rel) == folder and os.path.isfile(path)):
+            continue
+        before = {(r.kind, r.source) for r in compat.renderables(_git(root, 'show', f'{base}:{rel}'))} if base else set()
+        with open(path, encoding='utf-8') as f:
+            text = f.read()
+        out += [(rel, r.first + 1, r.kind, r.source) for r in compat.renderables(text) if (r.kind, r.source) not in before]
+    return out
+
+
+def parse(root, base, vendor, chrome, as_json):
+    """Check what to_parse finds in Chrome; print the findings; the exit code."""
+    found = to_parse(root, base)
+    items, where = {}, {}
+    for rel, line, kind, source in found:
+        rid = book.render_id(kind, source)
+        items[rid] = {'id': rid, 'kind': kind, 'source': source}
+        where.setdefault(rid, []).append(f'{rel}:{line}')
+    findings = []
+    if items:
+        browser = bookbuild.find_chrome(chrome)
+        if not browser:
+            cli.fail('no Chrome found to parse the diagrams and formulas: pass --chrome or set $CHROME')
+        with tempfile.TemporaryDirectory() as tmp:
+            try:
+                results = bookbuild.draw(list(items.values()), vendor, browser, tmp, parse=True)
+            except ValueError as e:
+                cli.fail(str(e))
+        for rid, result in results.items():
+            if isinstance(result, dict):
+                code = 'DIAGRAM_SYNTAX' if items[rid]['kind'] == 'mermaid' else 'MATH_SYNTAX'
+                message = ' '.join(result['error'].split())[:300]
+                findings += [{'code': code, 'where': w, 'message': message} for w in where[rid]]
+    findings.sort(key=lambda f: f['where'])
+    if as_json:
+        print(dumps({'checked': len(found), 'findings': findings}))
+    else:
+        for f in findings:
+            print(f'error  {f["code"]:<15} {f["where"]}: {f["message"]}')
+        print(f'parse: {len(found)} diagram(s) and formula(s) checked, {len(findings)} error(s)')
+    return 1 if findings else 0
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog='sllmwiki book', description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('command', choices=['stage', 'render', 'build', 'issues', 'parse'])
+    p.add_argument('out', nargs='?')
+    p.add_argument('--base')
+    p.add_argument('--name')
+    p.add_argument('--version', default='preview')
+    p.add_argument('--date', default=datetime.date.today().isoformat())
+    p.add_argument('--commit')
+    p.add_argument('--source')
+    p.add_argument('--formats', type=lambda v: [f.strip() for f in v.split(',') if f.strip()])
+    p.add_argument('--vendor')
+    p.add_argument('--chrome')
+    p.add_argument('--cache')
+    p.add_argument('--sphinx', default='sphinx-build')
+    p.add_argument('--json', action='store_true')
+    p.add_argument('--root')
+    args = p.parse_args(argv)
+    if args.command == 'parse':
+        args.vendor = bookbuild.unpacked(args.vendor)
+        if not args.vendor:
+            cli.usage(f'parse needs --vendor DIR holding {bookbuild.MERMAID} and {bookbuild.MATHJAX} (either may be kept in numbered parts)')
+        return parse(cli.wiki(args.root), args.base, args.vendor, args.chrome, args.json)
+    if not args.out:
+        cli.usage(f'{args.command} needs a folder')
+    if args.command in ('render', 'build'):
+        args.vendor = bookbuild.unpacked(args.vendor)
+        if not args.vendor:
+            cli.usage(f'{args.command} needs --vendor DIR holding {bookbuild.MERMAID} and {bookbuild.MATHJAX} (either may be kept in numbered parts)')
+        unknown = [f for f in args.formats or [] if f not in FORMATS]
+        if unknown or args.formats == []:
+            cli.usage(f'--formats can be {", ".join(FORMATS)}, not {", ".join(unknown) or "nothing"}')
+    if args.command == 'issues':
+        path = os.path.join(args.out, 'book.json')
+        if not os.path.isfile(path):
+            cli.usage(f'{args.out} isn\'t a staged book (no book.json)')
+        with open(path, encoding='utf-8') as f:
+            data = json.load(f)
+        root = cli.wiki(args.root) if args.root else None
+        text = book.issues(data, language.pack(config.get(root).settings['language']) if root else None)
+        if text:
+            print(text, end='')
+        return 0
+    if args.command == 'render':
+        staged = os.path.abspath(args.out)
+        if not os.path.isfile(os.path.join(staged, 'book.json')):
+            cli.usage(f'{args.out} isn\'t a staged book (no book.json): run sllmwiki book stage first')
+        data = render(staged, args.vendor, args.chrome, args.cache)
+        return _report(args, data, staged)
+    root = cli.wiki(args.root)
+    out = _empty(args.out)
+    if args.command == 'stage':
+        data = stage(root, out, args)
+        _report(args, data, out)
+        return 1 if data['problems'] else 0
+    staged = os.path.join(out, 'staged')
+    formats = args.formats if args.formats is not None else config.get(root).book_formats()
+    data = stage(root, staged, args)
+    data['files'], data['failed'] = {}, {}
+    if formats:
+        data = render(staged, args.vendor, args.chrome, args.cache)
+        data['files'], data['failed'] = {}, {}
+        path, extra = html_book(root, out, staged, data, args.sphinx)  # the PDF is printed from it
+        if path:
+            data['warnings'] += [{'code': 'BOOK_SPHINX', 'where': 'sphinx', 'message': w} for w in extra]
+            if 'html' in formats:
+                data['files']['html'] = path
+        for fmt in formats:
+            if fmt == 'epub':
+                epub, more = epub_book(root, out, staged, data, args.sphinx)
+                if epub:
+                    data['files']['epub'] = epub
+                    data['warnings'] += [{'code': 'BOOK_SPHINX', 'where': 'sphinx (epub)', 'message': w} for w in more
+                                         if w not in extra]
+                else:
+                    data['failed']['epub'] = more
+            elif not path:
+                data['failed'][fmt] = extra
+            elif fmt == 'pdf':
+                pdf = os.path.splitext(path)[0] + '.pdf'
+                chrome = bookbuild.find_chrome(args.chrome)
+                why = bookbuild.print_pdf(path, pdf, chrome, f'{{{data["title"]} 📖}}') if chrome else 'no Chrome found to print it'
+                if why:
+                    data['failed']['pdf'] = why
+                else:
+                    data['files']['pdf'] = pdf
+    _save(staged, data)
+    _report(args, data, out)
+    return 1 if data['failed'] else 0
+
+
+def _report(args, data, out):
+    problems, warnings = data['problems'], data.get('warnings', [])
+    if args.json:
+        print(dumps({'out': out, 'chapters': len(data['order']), 'renders': len(data['renders']), 'problems': problems,
+                     'warnings': warnings, 'files': data.get('files', {}), 'failed': data.get('failed', {})}))
+        return 0
+    for f in problems + warnings:
+        print(f'{f["code"]} {f["where"]}: {f["message"]}')
+    for fmt, why in data.get('failed', {}).items():
+        print(f'{fmt} failed: {why}')
+    for fmt, path in data.get('files', {}).items():
+        print(f'{fmt}: {path} ({os.path.getsize(path):,} bytes)')
+    print(f'{len(data["order"])} chapters, {len(data["renders"])} diagrams and formulas, '
+          f'{len(problems)} problem{"s" if len(problems) != 1 else ""}, '
+          f'{len(warnings)} warning{"s" if len(warnings) != 1 else ""}: {out}')
+    return 0
+
+
+if __name__ == '__main__':  # pragma: no cover
+    entry(main)

sllmwiki/commands/check_upstream.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""check-upstream: has a saved source changed where it came from? (design doc §7.2)
+
+  sllmwiki check-upstream [<slug> ...] [--tags a,b] [--json] [--root <wiki>]
+
+Fetches each saved source again (all of them, the given slugs, or those under
+the given tags), converts it the way fetch-source saved it, and compares it
+with the saved copy. A changed one is marked upstream_changed: <today> in the
+sources manifest, which wiki-check then reports (SOURCE_UPSTREAM_CHANGED: re-ingest
+it with sllmwiki fetch-source --refresh, and narratives pinned to it get rechecked).
+An unchanged one loses any earlier mark. A source that can't be fetched or
+converted is reported and left as it is. A long PDF saved in slices is fetched
+once and each slice compared with the new slice at the same place: the same
+page of a PDF, the same top-level entry of an EPUB's contents.
+
+A saved copy also differs when the skill's converter has improved since it was
+saved (a PDF's placeholder title dropped, say): re-ingesting takes the better copy.
+
+It uses the network, so it runs on request, never as part of a check.
+Exit code: 0, 1 on a refusal (no such slug or tag), 2 on a usage error.
+"""
+
+import argparse
+import os
+import re
+import sys
+
+sys.dont_write_bytecode = True
+
+from sllmwiki.lib import convert, fetch, layout, manifest, wikiindex  # noqa: E402
+from sllmwiki.lib.layout import Cli, entry  # noqa: E402
+from sllmwiki.lib.util import dumps, read_bytes, split_footer, today  # noqa: E402
+
+cli = Cli('sllmwiki check-upstream', 'usage: sllmwiki check-upstream [<slug> ...] [--tags a,b] [--json]')
+
+
+def fetch_converted(url):
+    """(markdown bytes, slices) of the source at url, or raises FetchError or ConvertError."""
+    r = fetch.fetch(url)
+    return convert.convert(fetch.kind_of(r.content_type, r.url), r.data, r.url, r.content_type, None, r.url)
+
+
+def place_of(url):
+    """A slice's place from its URL's fragment (page=N, chapter=N), or None."""
+    _, _, frag = url.partition('#')
+    return frag if re.match(r'(page|chapter)=\d+\Z', frag) else None
+
+
+def compare(root, entry, fetched):
+    """(status, detail) for one saved source against what its URL serves now."""
+    base = entry['url'].split('#')[0]
+    if base not in fetched:
+        try:
+            fetched[base] = fetch_converted(base)
+        except (fetch.FetchError, convert.ConvertError) as e:
+            fetched[base] = e
+    got = fetched[base]
+    if isinstance(got, Exception):
+        return 'unreachable', str(got)
+    body, slices = got
+    place = place_of(entry['url'])
+    if place is not None:
+        match = [s for s in slices if s[1] == place]
+        if not match:
+            kind, n = place.split('=')
+            return 'changed', f'no slice starts on page {n} any more' if kind == 'page' else f'no top-level entry {n} in its contents any more'
+        body = match[0][2].encode()
+    saved = split_footer(read_bytes(os.path.join(root, layout.in_content(entry['file']))))[0]
+    if split_footer(body)[0].rstrip(b'\n') == saved.rstrip(b'\n'):
+        return 'unchanged', ''
+    return 'changed', f'{len(saved):,} bytes saved, {len(body):,} now'
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog='sllmwiki check-upstream', description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('slugs', nargs='*')
+    p.add_argument('--tags', type=lambda v: [t.strip() for t in v.split(',') if t.strip()], default=[])
+    p.add_argument('--json', action='store_true')
+    p.add_argument('--root')
+    args = p.parse_args(argv)
+    root = cli.wiki(args.root)
+    sources = manifest.load(root, 'sources')
+    unknown = [s for s in args.slugs if s not in sources] + [t for t in args.tags if t not in manifest.load(root, 'tags')]
+    if unknown:
+        cli.fail(f'no such source or tag: {", ".join(unknown)}')
+    chosen = [s for s, e in sources.items() if e.get('file') and (not args.slugs and not args.tags or s in args.slugs
+                                                                    or set(args.tags) & set(e.get('tags') or []))]
+    results, fetched, marked = [], {}, False
+    for slug in chosen:
+        e = sources[slug]
+        status, detail = compare(root, e, fetched)
+        if status == 'changed' and not e.get('upstream_changed'):
+            manifest.save(root, 'sources', slug, dict(e, upstream_changed=today()))
+            marked = True
+        elif status == 'unchanged' and e.get('upstream_changed'):
+            manifest.save(root, 'sources', slug, {k: v for k, v in e.items() if k != 'upstream_changed'})
+            marked = True
+        results.append({'slug': slug, 'status': status, 'detail': detail})
+    if marked:
+        wikiindex.sync(root, write=True)
+    if args.json:
+        print(dumps({'sources': results}))
+        return 0
+    for r in results:
+        print(f'{r["status"]:<12} {r["slug"]}' + (f': {r["detail"]}' if r['detail'] else ''))
+    changed = [r['slug'] for r in results if r['status'] == 'changed']
+    print(f'{len(results)} checked, {len(changed)} changed' + (f': re-ingest with sllmwiki fetch-source --refresh ({", ".join(changed)})' if changed else ''))
+    return 0
+
+
+if __name__ == '__main__':  # pragma: no cover
+    entry(main)

sllmwiki/commands/cited_sections.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""cited-sections: each section a narrative cites, once, with the claims that cite it
+(design doc §7.3), so lint reads those sections and not whole sources.
+
+  sllmwiki cited-sections <narrative> [--claims-only] [--json] [--root <wiki>]
+
+<narrative> is its slug or its path. For each cited target, in order of first
+citation: the target, the claims citing it (the sentence before each citation,
+with its line), then the section's text (a code source's lines, numbered).
+--claims-only leaves the sections' text out. A target that doesn't resolve is
+listed with the reason; wiki-check reports it too.
+
+Exit code: 0, 1 when the narrative isn't found, 2 on a usage error.
+"""
+
+import argparse
+import os
+import re
+import sys
+
+sys.dont_write_bytecode = True
+
+from sllmwiki.lib import citations, layout, search  # noqa: E402
+from sllmwiki.lib.layout import Cli, entry  # noqa: E402
+from sllmwiki.lib.util import dumps, read_text  # noqa: E402
+
+cli = Cli('sllmwiki cited-sections', 'usage: sllmwiki cited-sections <narrative> [--claims-only] [--json]')
+_LINKS = re.compile(r'\[\\\[[0-9]+\\\]\]\([^)\s]*\)')
+_SENTENCE_END = re.compile(r'(?<=[.!?:;])[*_]*\s+(?=\S)|(?<=[。！？；：])[*_]*\s*(?=\S)')
+
+
+def claim(line, start):
+    """The sentence a citation at line[start] backs: the text before it since the last sentence end."""
+    before = _LINKS.sub('', line[:start]).strip()
+    before = re.sub(r'^\s*(?:[-*+]|\d+\.|\|)\s*', '', before)
+    parts = [p for p in _SENTENCE_END.split(before) if p.strip()]
+    return parts[-1].strip(' |*,') if parts else ''
+
+
+def collect(text):
+    """[(target, [(line, n, claim)])] in order of first citation."""
+    lines = text.split('\n')
+    found, last = {}, {}
+    for c in citations.cites(text):
+        line = lines[c.line]
+        start = line.find(f'[\\[{c.n}\\]]({c.target})')
+        said = claim(line, start) or last.get(c.line, '')  # a second citation right after the first backs the same claim
+        last[c.line] = said
+        found.setdefault(c.target, []).append((c.line + 1, c.n, said))
+    return list(found.items())
+
+
+def main(argv):
+    p = argparse.ArgumentParser(prog='sllmwiki cited-sections', description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('narrative')
+    p.add_argument('--claims-only', action='store_true')
+    p.add_argument('--json', action='store_true')
+    p.add_argument('--root')
+    args = p.parse_args(argv)
+    root = cli.wiki(args.root)
+    rel, _ = search.resolve(root, args.narrative)
+    if rel is None or not rel.startswith(layout.NARRATIVES + '/'):
+        cli.fail(f'no narrative {args.narrative}')
+    text = read_text(os.path.join(root, rel))
+    out = []
+    for target, claims in collect(text):
+        path, anchor = search.resolve(root, target, base=os.path.dirname(rel))
+        section, problem = None, None
+        if path is None:
+            problem = 'no such file'
+        elif not anchor:
+            problem = 'cites the whole file, not a section'
+        else:
+            section = search.section(search.read_file(root, path), anchor)
+            problem = None if section is not None else f'{path} has no section #{anchor}'
+        out.append({'target': target, 'file': path, 'anchor': anchor, 'claims': [{'line': ln, 'n': n, 'claim': c} for ln, n, c in claims],
+                    'problem': problem, 'text': None if args.claims_only else section})
+    if args.json:
+        print(dumps({'narrative': rel, 'cited': out}))
+        return 0
+    for item in out:
+        print(f'## {item["target"]}' + (f'  ({item["problem"]})' if item['problem'] else ''))
+        print()
+        for c in item['claims']:
+            print(f'- line {c["line"]} [{c["n"]}]: {c["claim"]}')
+        if item['text']:
+            print()
+            print(item['text'].rstrip('\n'))
+        print()
+    if not out:
+        print(f'{rel} cites nothing')
+    return 0
+
+
+if __name__ == '__main__':  # pragma: no cover
+    entry(main)