esbi-cli 0.2.1__py3-none-any.whl

esbi_cli/init.py

@@ -0,0 +1,176 @@
+"""`sb init`: a new vault and a config file, without overwriting anything that already exists."""
+
+import json
+import re
+import subprocess
+from pathlib import Path
+
+from esbi_cli import lang
+from esbi_cli.gitops import STATE_IGNORE, has_git
+
+TEMPLATES = Path(__file__).parent / "templates"
+EXAMPLE_CONFIG = TEMPLATES / "config.example.toml"  # inside the package: an installed copy has it
+FOLDERS = (
+    "inbox",
+    "raw",
+    "attachments",
+    *(
+        f"wiki/{kind}"
+        for kind in ("sources", "concepts", "entities", "syntheses", "review", "daily")
+    ),
+)
+
+
+def _files(language: str) -> dict[str, str]:
+    schema = (TEMPLATES / "SCHEMA.md").read_text(encoding="utf-8")
+    return {
+        "SCHEMA.md": schema.replace("{language}", lang.name(language)),
+        "index.md": f"# {lang.t(language, 'index_title')}\n",
+        "log.md": "# Log\n",
+        ".gitignore": ".obsidian/workspace*.json\n.obsidian/cache\n.trash/\n.DS_Store\n"
+        + "\n".join(STATE_IGNORE)
+        + "\n",
+    }
+
+
+def init_vault(root: Path, language: str = lang.DEFAULT) -> list[str]:
+    """Create what is missing; return what was created."""
+    created = []
+    for folder in FOLDERS:
+        if not (root / folder).is_dir():
+            (root / folder).mkdir(parents=True)
+            created.append(f"{folder}/")
+    for name, text in _files(language).items():
+        if not (root / name).exists():
+            (root / name).write_text(text, encoding="utf-8")
+            created.append(name)
+    if not (root / ".git").exists() and has_git():
+        subprocess.run(["git", "init", "-q", "-b", "main"], cwd=root, check=True)
+        created.append("git repository")
+    return created
+
+
+MODELS = {
+    "local": "Everything stays on this Mac: the notes are written by a local model.",
+    "subscription": (
+        "The text of each source goes to Anthropic through your Claude subscription "
+        "(the official `claude` tool, `claude auth login` once). Email is read only by the local model."
+    ),
+    "api": (
+        "The text of each source goes to Anthropic through its API, billed per token: set "
+        "ANTHROPIC_API_KEY in the environment. Email is read only by the local model."
+    ),
+}
+RUNTIMES = ("ollama", "lmstudio")
+
+
+def local_model(runtime: str, name: str | None) -> str:
+    """`<provider>/<name>` of the local model: Ollama's default, or the LM Studio identifier."""
+    return f"lmstudio/{name}" if runtime == "lmstudio" else f"ollama/{name or 'llama3.2:latest'}"
+
+
+def llm_sections(model: str, runtime: str, name: str | None, base_url: str | None = None) -> str:
+    """The [llm.*] sections for the chosen kind of model."""
+    local = local_model(runtime, name)
+    if model == "local":
+        return (
+            f'[llm.summarize]\nmodel = "{local}"\ntimeout_seconds = 300\n'
+            + (f"base_url = {json.dumps(base_url)}\n" if base_url else "")
+            + ("num_ctx = 8192\nmax_tokens = 1200\n" if runtime == "ollama" else "")
+        )
+    main = "claude-cli/default" if model == "subscription" else "anthropic/claude-sonnet-5-5"
+    sections = [
+        f'[llm.summarize]\nmodel = "{main}"\nfallback = "{local}"\ntimeout_seconds = 300\n',
+        f'[llm.ask]\nmodel = "{main}"\nfallback = "{local}"\ntimeout_seconds = 600\n',
+        f'[llm.private]\nmodel = "{local}"\n',  # email is read only by the local model
+    ]
+    if model == "subscription":
+        sections.insert(
+            1, f'[llm.synthesize]\nmodel = "{main}"\nfallback = "{local}"\ntimeout_seconds = 600\n'
+        )
+    return "\n".join(sections)
+
+
+OCR_MODEL = "qwen3-vl:2b-instruct"
+OCR_SECTION = f'[llm.ocr]\nmodel = "ollama/{OCR_MODEL}"\ntimeout_seconds = 600\n'
+
+
+def write_config(
+    path: Path,
+    vault: Path,
+    model: str = "local",
+    *,
+    viewer: str = "obsidian",
+    nightly: str | None = None,
+    runtime: str = "ollama",
+    local_name: str | None = None,
+    ocr: bool = False,
+    base_url: str | None = None,
+    language: str = lang.DEFAULT,
+) -> bool:
+    """Write the example config with this vault and choices; False if a config is already there."""
+    if path.exists():
+        return False
+    text = EXAMPLE_CONFIG.read_text(encoding="utf-8")
+    text = re.sub(r'^vault = ".*"', f'vault = "{vault}"', text, count=1, flags=re.M)
+    text = re.sub(
+        r'^language = ".*"$',
+        f'language = "{language}"\nviewer = "{viewer}"',
+        text,
+        count=1,
+        flags=re.M,
+    )
+    if nightly:
+        text = re.sub(
+            r'^nightly_time = "[^"]*"', f'nightly_time = "{nightly}"', text, count=1, flags=re.M
+        )
+    llm = llm_sections(model, runtime, local_name, base_url)
+    if ocr:  # a local vision model reads images and scanned PDFs: nothing leaves this machine
+        llm += "\n" + OCR_SECTION
+    text = re.sub(
+        r"^\[llm\.summarize\].*?(?=^# Optional|^\[bench\])",
+        llm + "\n",
+        text,
+        count=1,
+        flags=re.M | re.S,
+    )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+    return True
+
+
+def connect_remote(root: Path, url: str) -> bool:
+    """Add `origin` as the backup remote; False if the vault already has one (left alone)."""
+    if (
+        subprocess.run(
+            ["git", "remote", "get-url", "origin"], cwd=root, capture_output=True
+        ).returncode
+        == 0
+    ):
+        return False
+    subprocess.run(["git", "remote", "add", "origin", url], cwd=root, check=True)
+    return True
+
+
+def set_email_block(path: Path, user: str, label: str = "esbi-cli") -> None:
+    """Turn on the [email] section of a config file for this Gmail address and label, replacing the
+    block if there is one and touching nothing else. The password is never written here."""
+    block = (
+        "[email]\n"
+        "enabled = true\n"
+        'imap_host = "imap.gmail.com"\n'
+        f'mailbox = "{label}"\n'
+        f'user = "{user}"\n'
+        "# The app password lives in the macOS Keychain (service esbi-cli-imap), never in this file.\n"
+    )
+    text = path.read_text(encoding="utf-8")
+    match = re.search(r"^\[email\]\n.*?(?=^\[|\Z)", text, re.S | re.M)
+    if match:
+        kept = [  # choices made by hand survive a re-run of the setup
+            line for line in match.group().splitlines() if line.startswith("follow_links")
+        ]
+        block += "".join(line + "\n" for line in kept)
+        text = text[: match.start()] + block + "\n" + text[match.end() :]
+    else:
+        text = text.rstrip("\n") + "\n\n" + block
+    path.write_text(text, encoding="utf-8")

esbi_cli/interrupts.py

@@ -0,0 +1,90 @@
+"""SIGINT and SIGTERM as an exception in the main thread, so the code that owns a resource can put
+it back. Handlers exist only inside `handling()`. Within it a signal raises `Interrupted` only
+inside `interruptible()` and outside `deferred()`; anywhere else it waits for the next safe point,
+so a note is written whole or not at all.
+"""
+
+import signal
+import sys
+import threading
+from contextlib import contextmanager
+
+SIGNALS = (signal.SIGINT, signal.SIGTERM)
+
+# ponytail: module state, one run per process; a per-run object if two runs ever share a process
+_depth = 0  # a signal raises only at 0
+_pending: int | None = None  # the first signal received; later ones are ignored while it is handled
+
+
+class Interrupted(BaseException):
+    """Not an Exception, like KeyboardInterrupt: `except Exception` must not turn it into a failure."""
+
+    def __init__(self, signum: int):
+        super().__init__(signum)
+        self.signum = signum
+
+
+def _handle(signum, frame) -> None:
+    global _pending
+    if _pending is None:
+        _pending = signum
+        if _depth == 0:
+            raise Interrupted(signum)
+
+
+def _raise_if_pending() -> None:
+    if _depth == 0 and _pending is not None:
+        raise Interrupted(_pending)
+
+
+@contextmanager
+def handling():
+    """Install the handlers for the block, then restore the previous ones. The block is protected
+    until `interruptible()`. Outside the main thread nothing is installed."""
+    global _depth, _pending
+    if threading.current_thread() is not threading.main_thread():  # signal.signal needs it
+        yield
+        return
+    previous = {s: signal.signal(s, _handle) for s in SIGNALS}
+    _depth, _pending = 1, None
+    try:
+        yield
+    finally:
+        _depth, _pending = 0, None
+        for s, handler in previous.items():
+            signal.signal(s, handler)
+
+
+@contextmanager
+def interruptible():
+    """A signal raises `Interrupted` inside this block (one that came earlier raises on entry)."""
+    global _depth
+    _depth -= 1
+    try:
+        _raise_if_pending()
+        yield
+    finally:
+        _depth += 1
+
+
+@contextmanager
+def deferred():
+    """A signal inside this block waits until it ends. Costs nothing when no handler is installed."""
+    global _depth
+    _depth += 1
+    try:
+        yield
+    finally:
+        _depth -= 1
+    _raise_if_pending()  # normal exit only: a real error is not replaced
+
+
+@contextmanager
+def exit_when_interrupted():
+    """For commands with no queue (`sb ingest`, `sb reingest`): stop with the conventional code."""
+    try:
+        with handling(), interruptible():
+            yield
+    except Interrupted as exc:
+        print("Interrupted.", file=sys.stderr)
+        raise SystemExit(128 + exc.signum) from None

esbi_cli/lang.py

@@ -0,0 +1,341 @@
+"""The languages the wiki can be written in: one catalogue, nothing else knows a language.
+
+Code asks for text by language-independent key (`t("es", "summary")`) and recognises headings of
+every catalogued language (`key_of`), so a vault written in one language and later switched to
+another still reads. Adding a language is adding one entry to `LANGUAGES`: see
+https://rubenamaury.github.io/esbi-cli/docs/how-to/add-a-language/.
+
+Each entry has:
+- `name`: how the prompts call the language ("Write all text in Spanish").
+- `hint`: an optional extra line for the prompts, for a language a small model needs more help with.
+- `stopwords`: common words used to notice an answer in the wrong language; empty skips the check.
+- `generic_terms`: words too general to be a glossary entry ("data", "system"); they are dropped.
+- `disclaimers`: a regex for a definition that says it has none ("not defined in the text").
+- `relation_examples`: short labels for how two ideas relate, as the model should write them.
+- `placeholders`: how a model that copied the prompt's wording starts a "summary" ("Executive summary of ...").
+- `ask_example` / `rewrite_example`: worked outputs shown to the model, in this language.
+- `labels`: every piece of text the program writes into the wiki.
+"""
+
+import re
+import unicodedata
+from collections.abc import Iterable
+
+DEFAULT = "en"
+
+LANGUAGES: dict[str, dict] = {
+    "en": {
+        "name": "English",
+        "hint": "",
+        "relation_examples": '"extends", "complements", "improves", "uses", "is an example of"',
+        "stopwords": "the of and to in is that for with are this on as by from be an",
+        "generic_terms": "data information system process technology example method approach result problem",
+        "disclaimers": r"not (defined|mentioned|specified|provided|explained)|(does|do) not (define|mention|specify|explain|provide)|no definition",
+        "placeholders": ("executive summary", "summary of"),
+        "ask_example": (
+            '{"title": "What is a graph", "one_liner": "A graph is a set of nodes joined by edges.", '
+            '"answer": "A graph is a set of nodes connected by edges [[Graph]]. '
+            'It is used to model networks [[Networks]].", "cited_pages": ["Graph", "Networks"]}'
+        ),
+        "rewrite_example": (
+            'question "When should a judge accept and when escalate?" -> {"terms": ["judge", '
+            '"accept", "escalate", "confidence", "confident", "threshold", "review"]}'
+        ),
+        "labels": {
+            # sections of a source note
+            "summary": "Executive summary",
+            "abstract": "Detailed summary",
+            "insights": "Key ideas",
+            "key_points": "Key points",
+            "terms": "Key terms",
+            "quotes": "Key quotes",
+            "diagram": "Diagram",
+            "figures": "Figures",
+            "connections": "Connections to your wiki",
+            "open_questions": "Open questions",
+            "concepts": "Concepts",
+            "entities": "Entities",
+            "related": "Related",
+            "contradictions": "Possible contradictions",
+            # elsewhere in notes
+            "from_source": "From",
+            "original_source": "Original source",
+            "untitled": "Untitled",
+            "source_suffix": "(source)",
+            "figure": "Figure",
+            "image": "Image",
+            "no_subject": "No subject",
+            "page_abbr": "p.",
+            "contradiction_callout": "Possible contradiction ({date}) with {link}: {note}",
+            "contradiction_review_name": "{date} contradiction - {source}",
+            "contradiction_review_title": "Possible contradictions from {link}",
+            "contradiction_review_footer": "Review and resolve; then delete this note.",
+            # ask
+            "no_answer": "I find nothing about this in the wiki.",
+            "question_label": "Question",
+            "sources": "Sources",
+            "synthesis_suffix": "(synthesis)",
+            "answer_title": "Answer",
+            # index.md and log.md
+            "index_title": "Index",
+            "index_blurb": "Catalogue of the wiki ({total} pages). Kept by the worker.",
+            "syntheses": "Syntheses",
+            "log_created": "created: {names}",
+            "log_updated": "updated: {names}",
+            "log_review": "to review: {n}",
+            # the daily index
+            "daily_title": "Index of {date}",
+            "daily_read": "Read",
+            "daily_processed": "Processed today",
+            "daily_queue": "Tomorrow's queue",
+            "daily_review": "To review",
+            "daily_revisit": "Revisit",
+            "daily_runs": "Runs",
+            "daily_stats": "Statistics",
+            "nothing_read": "_Nothing new marked as read._",
+            "nothing_processed": "_Nothing was processed today._",
+            "queue_empty": "_The queue is empty._",
+            "queue_one": "1 source in the queue",
+            "queue_many": "{n} sources in the queue",
+            "queue_cap": " (showing the {n} oldest)",
+            "nothing_pending": "_Nothing pending._",
+            "could_not_process": "- Could not process {name} ({error})",
+            "retrying": "- Retrying {name} (attempt {n} of {max}): {error}",
+            "no_old_notes": "_No older notes to revisit yet._",
+            "stat_pages": "- Pages: {total} (sources: {sources}, concepts: {concepts}, entities: {entities})",
+            "stat_today": "- Today: {processed} sources processed, {touched} concepts/entities updated",
+            "stat_read": "- Read: {read} of {total} sources",
+            "stat_orphans": "- Orphans (no incoming links): {n}",
+            "no_runs": "_No runs yet today._",
+            "run_manual": " (manual)",
+            "run_line": "- {at}{manual} — processed: {ingested}, skipped: {skipped}, failed: {failed} · {tokens} tokens · {duration}",
+            "run_minutes": "{n} min",
+            "run_under_minute": "<1 min",
+            "run_stopped": " · stopped: {reason}",
+            "stop_max_sources": "source limit",
+            "stop_token_budget": "token budget",
+            "stop_llm_unavailable": "model unavailable",
+            "home_today": "- Today's index: {link}",
+            "home_unread": "- Unread: {n} sources",
+            "home_queue": "- Queued: {n} sources",
+            # the lint report
+            "lint_title": "Lint report ({date})",
+            "lint_orphan": "Orphan pages",
+            "lint_broken_link": "Broken links",
+            "lint_missing_field": "Missing fields",
+            "lint_unlinked_mention": "Unlinked mentions",
+            "lint_near_duplicate": "Possible duplicates",
+            "lint_missing_concept": "Concepts without a page",
+            "lint_missing_field_line": "- [[{page}]]: missing `{field}`",
+            "lint_unlinked_line": "- [[{page}]] mentions [[{target}]] without linking it",
+            "lint_missing_concept_line": "- «{name}» appears in {sources} and has no page of its own",
+            "lint_more": "… and {n} more",
+            "lint_footer": "The worker only reports; nothing is fixed by itself. This report is regenerated on every run.",
+            # the benchmark report
+            "bench_title": "Model benchmark ({when})",
+            "bench_header": "| model | success | no retry | median (s) | tokens/case | est. cost (USD) | {extra} |",
+            "bench_extra_ingest": "right language / concepts",
+            "bench_extra_ask": "cites the expected page",
+            "bench_routing": "Routing recommendation",
+            "bench_none": "no recommendation (no reliable model)",
+            "bench_footer": "Only a suggestion: change `[llm.*]` in config.toml by hand if you agree.",
+            "bench_question": "What is {title}?",
+        },
+    },
+    "es": {
+        "name": "Spanish",
+        "hint": "",
+        "relation_examples": '"amplía", "complementa", "mejora", "usa", "es un ejemplo de"',
+        "stopwords": "de la el que en los las y un una para con por del se es al como más pero sus",
+        "generic_terms": "datos información sistema proceso tecnología ejemplo método enfoque resultado problema",
+        "disclaimers": r"no (se )?(define|menciona|especifica|explica|proporciona|detalla)|no (est[aá]|aparece) (definid|especificad|en el texto)|sin definici[oó]n",
+        "placeholders": ("resumen ejecutivo", "resumen de"),
+        "ask_example": (
+            '{"title": "Qué es un grafo", "one_liner": "Un grafo es un conjunto de nodos unidos por aristas.", '
+            '"answer": "Un grafo es un conjunto de nodos conectados por aristas [[Grafo]]. '
+            'Se usa para modelar redes [[Redes]].", "cited_pages": ["Grafo", "Redes"]}'
+        ),
+        "rewrite_example": (
+            'question "¿Cuándo debe un juez aceptar y cuándo escalar?" -> {"terms": ["juez", "judge", '
+            '"aceptar", "accept", "escalar", "escalate", "confianza", "confident"]}'
+        ),
+        "labels": {
+            "summary": "Resumen ejecutivo",
+            "abstract": "Resumen detallado",
+            "insights": "Ideas clave",
+            "key_points": "Puntos clave",
+            "terms": "Términos clave",
+            "quotes": "Frases clave",
+            "diagram": "Diagrama",
+            "figures": "Figuras",
+            "connections": "Conexiones con tu wiki",
+            "open_questions": "Preguntas abiertas",
+            "concepts": "Conceptos",
+            "entities": "Entidades",
+            "related": "Relacionado",
+            "contradictions": "Posibles contradicciones",
+            "from_source": "Desde",
+            "original_source": "Fuente original",
+            "untitled": "Sin título",
+            "source_suffix": "(fuente)",
+            "figure": "Figura",
+            "image": "Imagen",
+            "no_subject": "Sin asunto",
+            "page_abbr": "p.",
+            "contradiction_callout": "Posible contradicción ({date}) con {link}: {note}",
+            "contradiction_review_name": "{date} contradicción - {source}",
+            "contradiction_review_title": "Posibles contradicciones desde {link}",
+            "contradiction_review_footer": "Revisa y resuelve; luego borra esta nota.",
+            "no_answer": "No encuentro nada sobre esto en la wiki.",
+            "question_label": "Pregunta",
+            "sources": "Fuentes",
+            "synthesis_suffix": "(síntesis)",
+            "answer_title": "Respuesta",
+            "index_title": "Índice",
+            "index_blurb": "Catálogo de la wiki ({total} páginas). Lo mantiene el worker.",
+            "syntheses": "Síntesis",
+            "log_created": "creadas: {names}",
+            "log_updated": "actualizadas: {names}",
+            "log_review": "por revisar: {n}",
+            "daily_title": "Índice del {date}",
+            "daily_read": "Leído",
+            "daily_processed": "Procesado hoy",
+            "daily_queue": "Cola de mañana",
+            "daily_review": "Por revisar",
+            "daily_revisit": "Repasar",
+            "daily_runs": "Ejecuciones",
+            "daily_stats": "Estadísticas",
+            "nothing_read": "_Nada nuevo marcado como leído._",
+            "nothing_processed": "_Hoy no se procesó nada._",
+            "queue_empty": "_La cola está vacía._",
+            "queue_one": "1 fuente en cola",
+            "queue_many": "{n} fuentes en cola",
+            "queue_cap": " (se muestran las {n} más antiguas)",
+            "nothing_pending": "_Nada pendiente._",
+            "could_not_process": "- No se pudo procesar {name} ({error})",
+            "retrying": "- Reintentando {name} (intento {n} de {max}): {error}",
+            "no_old_notes": "_Todavía no hay notas antiguas para repasar._",
+            "stat_pages": "- Páginas: {total} (fuentes: {sources}, conceptos: {concepts}, entidades: {entities})",
+            "stat_today": "- Hoy: {processed} fuentes procesadas, {touched} conceptos/entidades actualizados",
+            "stat_read": "- Leídas: {read} de {total} fuentes",
+            "stat_orphans": "- Huérfanas (sin enlaces entrantes): {n}",
+            "no_runs": "_Todavía no hubo ejecuciones hoy._",
+            "run_manual": " (manual)",
+            "run_line": "- {at}{manual} — procesadas: {ingested}, omitidas: {skipped}, fallidas: {failed} · {tokens} tokens · {duration}",
+            "run_minutes": "{n} min",
+            "run_under_minute": "<1 min",
+            "run_stopped": " · detenida: {reason}",
+            "stop_max_sources": "límite de fuentes",
+            "stop_token_budget": "presupuesto de tokens",
+            "stop_llm_unavailable": "LLM no disponible",
+            "home_today": "- Índice de hoy: {link}",
+            "home_unread": "- Sin leer: {n} fuentes",
+            "home_queue": "- En cola: {n} fuentes",
+            "lint_title": "Informe de lint ({date})",
+            "lint_orphan": "Páginas huérfanas",
+            "lint_broken_link": "Enlaces rotos",
+            "lint_missing_field": "Campos que faltan",
+            "lint_unlinked_mention": "Menciones sin enlazar",
+            "lint_near_duplicate": "Posibles duplicados",
+            "lint_missing_concept": "Conceptos sin página",
+            "lint_missing_field_line": "- [[{page}]]: falta `{field}`",
+            "lint_unlinked_line": "- [[{page}]] menciona [[{target}]] sin enlazarlo",
+            "lint_missing_concept_line": "- «{name}» aparece en {sources} y no tiene página propia",
+            "lint_more": "… y {n} más",
+            "lint_footer": "El worker solo informa; nada se corrige solo. Este informe se regenera en cada ejecución.",
+            "bench_title": "Benchmark de modelos ({when})",
+            "bench_header": "| modelo | éxito | sin reintento | mediana (s) | tokens/caso | coste est. (USD) | {extra} |",
+            "bench_extra_ingest": "idioma correcto / conceptos",
+            "bench_extra_ask": "cita la página esperada",
+            "bench_routing": "Recomendación de enrutado",
+            "bench_none": "sin recomendación (ningún modelo fiable)",
+            "bench_footer": "Solo es una sugerencia: cambia `[llm.*]` en config.toml a mano si te convence.",
+            "bench_question": "¿Qué es {title}?",
+        },
+    },
+}
+
+
+def supported() -> str:
+    return ", ".join(LANGUAGES)
+
+
+def get(code: str) -> dict:
+    """The catalogue entry for `code`; a clear error naming the supported languages otherwise."""
+    try:
+        return LANGUAGES[code]
+    except (KeyError, TypeError):
+        raise ValueError(f"[notes].language must be one of: {supported()}, got {code!r}") from None
+
+
+def name(code: str) -> str:
+    return get(code)["name"]
+
+
+def t(code: str, key: str, **values) -> str:
+    """The text for `key` in language `code`, with `{placeholders}` filled in."""
+    return get(code)["labels"][key].format(**values)
+
+
+def _fold(text: str) -> str:
+    plain = "".join(c for c in unicodedata.normalize("NFKD", text) if not unicodedata.combining(c))
+    return " ".join(plain.casefold().split())
+
+
+def every(key: str) -> list[str]:
+    """The label for `key` in every language: what a reader must accept."""
+    return list(dict.fromkeys(entry["labels"][key] for entry in LANGUAGES.values()))
+
+
+def key_of(heading: str) -> str | None:
+    """The key a heading belongs to, in whatever language it was written; None for any other."""
+    wanted = _fold(heading)
+    for entry in LANGUAGES.values():
+        for key, label in entry["labels"].items():
+            if _fold(label) == wanted:
+                return key
+    return None
+
+
+def instruction(code: str) -> str:
+    """The sentence every prompt carries to say which language to write in."""
+    entry = get(code)
+    return (
+        f"Write ALL text in {entry['name']}, whatever language the source or the SCHEMA is in; "
+        f"only verbatim quotes and term names keep the source's language. {entry['hint']}"
+    ).strip()
+
+
+def _hits(text: str, code: str) -> int:
+    words = set(LANGUAGES[code]["stopwords"].split())
+    return sum(w in words for w in re.findall(r"[^\W\d_]+", text.lower()))
+
+
+def wrong_language(text: str, code: str, min_hits: int = 4) -> bool:
+    """Is `text` clearly in another catalogued language than `code`? A cheap stopword count: small
+    models often answer in the source's language. A language with no stopwords is never judged.
+    `min_hits` is how many stopwords it takes; a one-sentence definition needs fewer."""
+    if not get(code)["stopwords"]:
+        return False
+    own = _hits(text, code)
+    return any(
+        _hits(text, other) >= min_hits and _hits(text, other) > 1.5 * own
+        for other, entry in LANGUAGES.items()
+        if other != code and entry["stopwords"]
+    )
+
+
+SHORT_FIELD_CHARS = 300  # under this, a field is a sentence or two: fewer stopwords are enough
+
+
+def leaking(fields: Iterable[tuple[str, str]], code: str) -> list[str]:
+    """The names of the (name, text) fields written in another catalogued language than `code`,
+    each field judged on its own: joined with a long field in the right language, a short one in
+    the wrong language goes unnoticed. A language without a word list is never judged."""
+    return list(
+        dict.fromkeys(
+            name
+            for name, text in fields
+            if wrong_language(text, code, 4 if len(text) >= SHORT_FIELD_CHARS else 2)
+        )
+    )

esbi_cli/links.py

@@ -0,0 +1,10 @@
+"""Wikilink parsing shared by the daily index and lint."""
+
+import re
+
+WIKILINK = re.compile(r"\[\[([^\]|#]+)")
+
+
+def link_targets(text: str) -> list[str]:
+    """Page names linked from text: `[[Title]]`, `[[Title|alias]]` and `[[Title#Heading]]` all give Title."""
+    return [t.strip() for t in WIKILINK.findall(text)]