esbi-cli 0.2.1__py3-none-any.whl

esbi_cli/doctor.py

@@ -0,0 +1,391 @@
+"""`sb doctor`: check the whole setup and say what to fix. Warnings are inconvenient, FAIL is broken."""
+
+import json
+import os
+import plistlib
+import re
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+import httpx
+
+from esbi_cli import __version__, lang, update
+from esbi_cli import schedule as launchd
+from esbi_cli.config import Config, find_config, load_config
+from esbi_cli.gitops import has_git
+from esbi_cli.llm.adapter import make_llm, make_ocr
+from esbi_cli.mail.credentials import CredentialError, get_password
+from esbi_cli.privacy import remote_host, remote_warning
+from esbi_cli.queue import Queue
+from esbi_cli.runlog import RunLog
+
+KEY_VARS = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY"}
+SERVER_PROBE_TIMEOUT_SECONDS = 3  # a model server that does not answer this fast is down
+CLAUDE_STATUS_TIMEOUT_SECONDS = 20
+
+
+@dataclass
+class Check:
+    level: str  # "ok" | "WARN" | "FAIL"
+    name: str
+    text: str
+    fix: str = ""
+
+
+def _model(task: str, cfg: Config, fallback: bool = False) -> Check:
+    llm = cfg.llm[task]
+    if fallback:  # the fallback inherits the section's settings, with its own model
+        llm = replace(llm, model=llm.fallback)
+        task = f"{task} fallback"
+    provider, _, name = llm.model.partition("/")
+    label = f"model {task}"
+    if provider == "ollama":
+        base = (llm.base_url or "http://localhost:11434").rstrip("/")
+        try:
+            tags = httpx.get(f"{base}/api/tags", timeout=SERVER_PROBE_TIMEOUT_SECONDS).json()[
+                "models"
+            ]
+        except (httpx.HTTPError, KeyError, ValueError):
+            return Check(
+                "FAIL",
+                label,
+                f"Ollama is not reachable at {base}",
+                "brew services start ollama (or open the Ollama app)",
+            )
+        have = {m["name"] for m in tags}
+        if name in have or f"{name}:latest" in have:
+            return Check("ok", label, f"{llm.model} is installed")
+        return Check("FAIL", label, f"{llm.model} is not installed", f"ollama pull {name}")
+    if provider == "lmstudio":
+        base = (llm.base_url or "http://localhost:1234/v1").rstrip("/")
+        try:
+            served = {
+                m["id"]
+                for m in httpx.get(f"{base}/models", timeout=SERVER_PROBE_TIMEOUT_SECONDS).json()[
+                    "data"
+                ]
+            }
+        except (httpx.HTTPError, KeyError, ValueError):
+            return Check(
+                "FAIL",
+                label,
+                f"LM Studio is not reachable at {base}",
+                "open LM Studio, load a model, and start the local server (Developer tab)",
+            )
+        if name in served:
+            return Check("ok", label, f"{llm.model} is served by LM Studio")
+        return Check(
+            "FAIL",
+            label,
+            f"LM Studio does not serve {name!r}",
+            f"use one of: {', '.join(sorted(served)) or '(none loaded)'}",
+        )
+    if provider == "claude-cli":
+        if not shutil.which("claude"):
+            return Check(
+                "FAIL", label, "the `claude` command is not installed", "https://claude.com/code"
+            )
+        try:
+            who = json.loads(
+                subprocess.run(
+                    ["claude", "auth", "status"],
+                    capture_output=True,
+                    text=True,
+                    timeout=CLAUDE_STATUS_TIMEOUT_SECONDS,
+                ).stdout
+            )
+        except (OSError, subprocess.TimeoutExpired, ValueError):
+            who = {}
+        if who.get("loggedIn"):
+            return Check(
+                "ok", label, f"{llm.model} (subscription of {who.get('email', 'your account')})"
+            )
+        return Check("FAIL", label, "`claude` is not logged in", "claude auth login")
+    if provider in KEY_VARS:
+        env = llm.api_key_env or KEY_VARS[provider]
+        if os.environ.get(env):
+            return Check("ok", label, f"{llm.model} (key in ${env})")
+        return Check("FAIL", label, f"{llm.model}: ${env} is not set", f"export {env}=...")
+    return Check(
+        "FAIL",
+        label,
+        f"unknown provider in {llm.model!r}",
+        "use ollama/, lmstudio/, openai/, anthropic/ or claude-cli/",
+    )
+
+
+def _ocr(cfg: Config) -> list[Check]:
+    """Reading images and scanned PDFs is optional; when set up it must be a model on this machine."""
+    if "ocr" not in cfg.llm:
+        return [
+            Check(
+                "ok",
+                "ocr",
+                "off (optional): add [llm.ocr] to read images and scanned PDFs "
+                "(`sb init --ocr`, https://rubenamaury.github.io/esbi-cli/docs/reference/configuration/)",
+            )
+        ]
+    try:
+        make_ocr(cfg.llm["ocr"])
+    except ValueError as exc:
+        return [Check("FAIL", "ocr", str(exc), 'model = "ollama/qwen3-vl:2b-instruct"')]
+    return [_model("ocr", cfg)]
+
+
+def _sends_text_out(llm_cfg) -> bool:
+    try:
+        return bool(make_llm(llm_cfg).sends_text_out)  # building a model makes no request
+    except ValueError:
+        return False  # an unknown provider is reported by the model check
+
+
+def _privacy(cfg: Config) -> Check:
+    """Email must be read only by a model on this machine, and cloud models must never get it."""
+    private = cfg.llm.get("private")
+    if private and _sends_text_out(private):
+        return Check(
+            "FAIL",
+            "email privacy",
+            "[llm.private] (or its fallback) sends text away, so email would leave this machine",
+            'use a local model: [llm.private] model = "ollama/llama3.2:latest", with no cloud fallback',
+        )
+    cloud = [t for t, m in cfg.llm.items() if t not in ("private", "embed") and _sends_text_out(m)]
+    if not cloud or private:
+        return Check(
+            "ok",
+            "email privacy",
+            "email is read only by a model on this machine" if private else "all models are local",
+        )
+    return Check(
+        "WARN",
+        "email privacy",
+        f"{', '.join(cloud)} send text away and no [llm.private] is set: emails will be refused",
+        'add [llm.private] model = "ollama/llama3.2:latest" to config.toml',
+    )
+
+
+def _remote_servers(cfg: Config) -> list[Check]:
+    """A "local" runtime pointed at another machine still sends the notes' text out. (A remote
+    [llm.private] is already a FAIL in _privacy: there email would go to that host.)"""
+    checks = []
+    for task, llm in cfg.llm.items():
+        for label, model in ((task, llm.model), (f"{task} fallback", llm.fallback)):
+            host = model and task != "private" and remote_host(model, llm.base_url)
+            if host:
+                checks.append(
+                    Check(
+                        "WARN",
+                        f"server {label}",
+                        remote_warning(model, host),
+                        "serve the model from this machine, or use it only for text you would share",
+                    )
+                )
+    return checks
+
+
+def _network(cfg: Config) -> list[Check]:
+    if not cfg.network.use_environment_proxy:
+        return []
+    return [
+        Check(
+            "WARN",
+            "network",
+            "[network].use_environment_proxy is on: address checks are done by the proxy, not by "
+            "esbi-cli, so a link in an email could make the proxy reach an internal host",
+            "set it to false unless this network only has a proxy",
+        )
+    ]
+
+
+def _email(cfg: Config) -> Check:
+    if not cfg.email.enabled:
+        return Check("ok", "email", "off (optional)")
+    try:
+        get_password(cfg.email.user or "")
+        return Check("ok", "email", f"password for {cfg.email.user} is in the Keychain")
+    except CredentialError as exc:
+        return Check("FAIL", "email", str(exc), "sb email set-password")
+
+
+def _version(cfg: Config) -> Check:
+    """The installed version, and whether a newer one is known (from the daily cache)."""
+    if not update.checks_enabled(cfg.update.check):
+        return Check("ok", "version", f"{__version__} (update check is off)")
+    release = update.cached_latest()
+    if release is None:  # offline, or GitHub did not answer: not a problem with this setup
+        return Check("ok", "version", __version__)
+    if update.is_newer(release.version, __version__):
+        return Check("WARN", "version", f"{release.version} is available, run `sb update`")
+    return Check("ok", "version", f"{__version__} (latest)")
+
+
+AGENTS_DIR = Path("~/Library/LaunchAgents").expanduser()
+
+
+def _installed_plist() -> dict:
+    """The installed LaunchAgent, or {} when there is none (or it cannot be read)."""
+    try:
+        return plistlib.loads((AGENTS_DIR / f"{launchd.LABEL}.plist").read_bytes())
+    except (OSError, ValueError):
+        return {}
+
+
+def _installed_time() -> tuple[int, int] | None:
+    """The (hour, minute) in the installed LaunchAgent, if there is one."""
+    try:
+        at = _installed_plist()["StartCalendarInterval"]
+        return at["Hour"], at["Minute"]
+    except KeyError:
+        return None
+
+
+def _job(cfg: Config) -> Check:
+    try:
+        loaded = launchd.is_loaded(os.getuid(), launchctl=launchd.run_launchctl)
+    except OSError:
+        return Check("WARN", "nightly job", "launchd is not available here")
+    if not loaded:
+        return Check("WARN", "nightly job", "not installed", "sb schedule install")
+    installed = _installed_time()
+    if installed is not None and installed != cfg.nightly_at:
+        return Check(
+            "WARN",
+            "nightly job",
+            f"installed for {installed[0]:02d}:{installed[1]:02d} but config.toml says {cfg.nightly_time}",
+            "sb schedule install",
+        )
+    command = " ".join(map(str, _installed_plist().get("ProgramArguments", [])))
+    if "/Cellar/esbi-cli/" in command:  # `brew upgrade` deletes that folder: the job would stop
+        return Check(
+            "WARN",
+            "nightly job",
+            "points into a versioned Homebrew folder (Cellar) that the next `brew upgrade` removes",
+            "sb schedule install",
+        )
+    prefix = Path(
+        sys.prefix
+    )  # a uv tool or pipx environment can be rebuilt elsewhere by an upgrade
+    if update.install_method(prefix, Path(__file__).resolve().parent) in ("uv-tool", "pipx"):
+        current = launchd.stable_prefix(prefix) / "bin" / "sb"
+        if str(current) not in command:
+            old = re.search(r"\S+/bin/sb\b", command)
+            return Check(
+                "WARN",
+                "nightly job",
+                f"runs {old[0] if old else 'another environment'}, not the {current} that is "
+                "running now",
+                "sb schedule install",
+            )
+    return Check("ok", "nightly job", f"installed and loaded, runs at {cfg.nightly_time}")
+
+
+def _last_run(vault: Path) -> Check:
+    runs = [r for r in RunLog(vault / ".esbi" / "runs.jsonl").runs() if r.trigger == "scheduled"]
+    if not runs:
+        return Check(
+            "WARN",
+            "last run",
+            "no scheduled run yet",
+            "the job does it after the nightly time; or try `sb run --if-due`",
+        )
+    last = runs[-1]
+    text = f"{last.started:%Y-%m-%d %H:%M}: {last.ingested} ingested, {last.failed} failed"
+    if last.stopped_by == "llm_unavailable":
+        return Check(
+            "WARN", "last run", f"{text}; the model was unreachable", "check the model line above"
+        )
+    return Check("ok", "last run", text)
+
+
+def _vault(root: Path, viewer: str = "obsidian") -> list[Check]:
+    if not (root / "SCHEMA.md").is_file() or not (root / "wiki").is_dir():
+        return [
+            Check(
+                "FAIL",
+                "vault",
+                f"{root} is not a esbi-cli vault (SCHEMA.md or wiki/ missing)",
+                "fix [paths].vault in config.toml",
+            )
+        ]
+    checks = [Check("ok", "vault", str(root))]
+    if not (root / ".git").exists():
+        checks.append(
+            Check(
+                "ok", "vault history", "off: git is not installed (optional, `sb init` turns it on)"
+            )
+            if not has_git()
+            else Check(
+                "WARN",
+                "vault history",
+                "no version history (the vault is not a git repository)",
+                f"git -C {root} init",
+            )
+        )
+    if viewer == "none":
+        checks.append(Check("ok", "obsidian", 'not used ([notes].viewer = "none")'))
+    elif not (root / ".obsidian").is_dir():
+        checks.append(
+            Check(
+                "WARN",
+                "obsidian",
+                "this folder was never opened as a vault in Obsidian",
+                f"Obsidian > Open folder as vault > {root}",
+            )
+        )
+    else:
+        checks.append(Check("ok", "obsidian", "opened as a vault"))
+    counts = Queue(root / ".esbi" / "queue.sqlite3").counts()
+    text = ", ".join(f"{n} {state}" for state, n in sorted(counts.items())) or "empty"
+    if counts.get("failed"):
+        checks.append(Check("WARN", "queue", text, "sb status, then `sb retry` or `sb drop`"))
+    else:
+        checks.append(Check("ok", "queue", text))
+    return checks + [_last_run(root)]
+
+
+def run_checks(config_arg: Path | None) -> list[Check]:
+    try:
+        path = find_config(config_arg)
+        cfg = load_config(path)
+    except (FileNotFoundError, ValueError) as exc:
+        return [
+            Check(
+                "FAIL",
+                "config",
+                str(exc),
+                "correct the setting named in the message"
+                if isinstance(exc, ValueError)
+                else "run `sb init`, or point at your file with --config PATH",
+            )
+        ]
+    checks = [
+        Check("ok", "config", str(path)),
+        _version(cfg),
+        Check("ok", "notes language", f"{cfg.language} ({lang.name(cfg.language)})"),
+        *_vault(cfg.vault, cfg.viewer),
+    ]
+    for task in ("summarize", "synthesize", "ask", "private", "embed"):
+        if task in cfg.llm:
+            checks.append(_model(task, cfg))
+            if cfg.llm[task].fallback:
+                checks.append(_model(task, cfg, fallback=True))
+    checks += _ocr(cfg)
+    checks.append(_privacy(cfg))
+    checks += _remote_servers(cfg)
+    checks += [*_network(cfg), _email(cfg), _job(cfg)]
+    sb = shutil.which("sb")
+    project = Path(__file__).resolve().parents[2]
+    checks.append(
+        Check("ok", "global install", sb)
+        if sb
+        else Check(
+            "WARN",
+            "global install",
+            "`sb` is not on your PATH",
+            f"uv tool install --editable {project}",
+        )
+    )
+    return checks

esbi_cli/evaluate.py

@@ -0,0 +1,91 @@
+"""Score retrieval (and answers) on questions whose source is known: the number to beat before
+changing how `sb ask` finds pages."""
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from esbi_cli.ask.answer import answer_question, rewrite_question
+from esbi_cli.ingest.retrieve import find_candidates
+from esbi_cli.llm.adapter import LLM
+from esbi_cli.vault import Vault, fold
+
+
+@dataclass
+class Golden:
+    question: str
+    expect: list[str]  # titles, or the start of titles: long source titles are cut
+
+
+@dataclass
+class Case:
+    question: str
+    expect: list[str]
+    retrieved: list[str]
+    rank: int | None  # 1 = first page returned; None = not among them
+    grounded: bool | None = None  # only when answers were asked for
+    cited: bool | None = None
+
+
+@dataclass
+class EvalReport:
+    k: int
+    cases: list[Case] = field(default_factory=list)
+
+    @property
+    def recall(self) -> float:
+        return sum(c.rank is not None for c in self.cases) / len(self.cases) if self.cases else 0.0
+
+    @property
+    def mrr(self) -> float:
+        return (
+            sum(1 / c.rank for c in self.cases if c.rank) / len(self.cases) if self.cases else 0.0
+        )
+
+    def rate(self, attribute: str) -> float | None:
+        values = [getattr(c, attribute) for c in self.cases if getattr(c, attribute) is not None]
+        return sum(values) / len(values) if values else None
+
+
+def load_golden(path: Path) -> list[Golden]:
+    golden = []
+    for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+            golden.append(Golden(str(row["question"]), [str(e) for e in row["expect"]]))
+        except (ValueError, KeyError, TypeError) as exc:
+            raise ValueError(
+                f'{path.name} line {number}: need {{"question": ..., "expect": [...]}}'
+            ) from exc
+    return golden
+
+
+def _matches(title: str, expect: list[str]) -> bool:
+    return any(fold(title).startswith(fold(e)) for e in expect)
+
+
+def evaluate(
+    vault: Vault,
+    golden: list[Golden],
+    k: int = 6,
+    rewrite_llm: LLM | None = None,
+    answer_llm: LLM | None = None,
+) -> EvalReport:
+    report = EvalReport(k)
+    for item in golden:
+        extra = rewrite_question(rewrite_llm, item.question, vault.language) if rewrite_llm else []
+        titles = [
+            c.title for c in find_candidates(vault, item.question, max_results=k, extra=extra)
+        ]
+        rank = next((i + 1 for i, t in enumerate(titles) if _matches(t, item.expect)), None)
+        case = Case(item.question, item.expect, titles, rank)
+        if answer_llm:
+            answer = answer_question(
+                vault, answer_llm, item.question, rewrite=rewrite_llm is not None
+            )
+            case.grounded = answer.grounded
+            case.cited = any(_matches(c, item.expect) for c in answer.citations)
+        report.cases.append(case)
+    return report

esbi_cli/export.py

@@ -0,0 +1,137 @@
+"""A read-only static copy of the wiki (plain HTML files), for people who browse without Obsidian."""
+
+import html
+import re
+import shutil
+from pathlib import Path
+from urllib.parse import quote
+
+from markdown_it import MarkdownIt
+
+from esbi_cli import lang
+from esbi_cli.vault import Page, Vault, fold, slugify
+
+MARKER = ".esbi-site"
+_EMBED = re.compile(r"!\[\[([^\]|]+?)(?:\|(\d+))?\]\]")
+_LINK = re.compile(r"\[\[([^\]|#]+?)(?:#[^\]|]*)?(?:\|([^\]]+?))?\]\]")
+_STYLE = """
+:root { color-scheme: light dark; --fg: #1f2328; --bg: #ffffff; --muted: #6a737d; --line: #d8dee4; --link: #0b5cad; }
+@media (prefers-color-scheme: dark) { :root { --fg: #e6edf3; --bg: #0d1117; --muted: #8b949e; --line: #30363d; --link: #58a6ff; } }
+body { font: 16px/1.6 system-ui, sans-serif; color: var(--fg); background: var(--bg); max-width: 46rem; margin: 0 auto; padding: 1rem; }
+a { color: var(--link); } nav { border-bottom: 1px solid var(--line); margin-bottom: 1rem; padding-bottom: .5rem; }
+img { max-width: 100%; height: auto; } pre { overflow-x: auto; } blockquote { border-left: 3px solid var(--line); margin-left: 0; padding-left: 1rem; color: var(--muted); }
+li.src small { color: var(--muted); } table { border-collapse: collapse; } td, th { border: 1px solid var(--line); padding: .25rem .5rem; }
+"""
+_MERMAID = (
+    '<script type="module">import mermaid from "https://cdn.jsdelivr.net/npm/mermaid@10.9.3/dist/'
+    'mermaid.esm.min.mjs"; mermaid.initialize({startOnLoad: true});</script>'
+)
+
+
+def _page_html(title: str, body: str, root: str, language: str) -> str:
+    script = _MERMAID if 'class="mermaid"' in body else ""
+    return (
+        f'<!doctype html><html lang="{language}"><head><meta charset="utf-8">'
+        f'<meta name="viewport" content="width=device-width, initial-scale=1">'
+        f"<title>{html.escape(title)}</title><style>{_STYLE}</style></head><body>"
+        f'<nav><a href="{root}index.html">esbi-cli</a></nav>{body}{script}</body></html>'
+    )
+
+
+def export_site(vault: Vault, out: Path) -> int:
+    """Write the wiki as HTML into `out` (replacing an earlier export). Returns the page count."""
+    out = out.expanduser().resolve()
+    if out.exists() and any(out.iterdir()):
+        old_marker = out / ".second-brain-site"  # legacy: the name before esbi-cli
+        if not (out / MARKER).exists() and not old_marker.exists():
+            raise ValueError(
+                f"{out} is not an export of this wiki and has other files: not touching it"
+            )
+        for child in out.iterdir():
+            shutil.rmtree(child) if child.is_dir() else child.unlink()
+    out.mkdir(parents=True, exist_ok=True)
+    (out / MARKER).write_text("written by `sb export`; safe to replace\n", encoding="utf-8")
+
+    pages = list(vault.iter_pages())
+    taken: dict[str, set[str]] = {}
+    where: dict[Page, str] = {}  # page -> "kind/slug.html"
+    names: dict[str, str] = {}
+    for page in pages:
+        slug, used = slugify(page.title), taken.setdefault(page.kind, set())
+        n = 2
+        while slug in used:
+            slug, n = f"{slugify(page.title)}-{n}", n + 1
+        used.add(slug)
+        where[id(page)] = f"{page.kind}/{slug}.html"
+        for name in (page.path.stem, page.title, *page.aliases):
+            names.setdefault(fold(str(name)), where[id(page)])
+
+    # html False: a note is written from untrusted pages and mail, and the export is a website, so
+    # raw HTML in a note is shown as text. Links and images are written as Markdown below.
+    md = MarkdownIt("commonmark", {"html": False}).enable("table")
+    for page in pages:
+
+        def embed(m: re.Match) -> str:
+            return f"![]({quote('../' + m[1].strip(), safe='/')})"
+
+        def link(m: re.Match) -> str:
+            target = names.get(fold(m[1].strip()))
+            label = (m[2] or m[1].strip()).replace("[", "\\[").replace("]", "\\]")
+            return f"[{label}](../{quote(target, safe='/')})" if target else label
+
+        text = _LINK.sub(link, _EMBED.sub(embed, page.body))
+        body = md.render(text)
+        body = body.replace('<pre><code class="language-mermaid">', '<pre class="mermaid"><code>')
+        body = re.sub(
+            r'<pre class="mermaid"><code>(.*?)</code></pre>',
+            r'<pre class="mermaid">\1</pre>',
+            body,
+            flags=re.S,
+        )
+        body = body.replace("<li>[x] ", "<li>☑ ").replace("<li>[ ] ", "<li>☐ ")
+        target = out / where[id(page)]
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(_page_html(page.title, body, "../", vault.language), encoding="utf-8")
+
+    if (vault.root / "attachments").is_dir():
+        shutil.copytree(vault.root / "attachments", out / "attachments", dirs_exist_ok=True)
+    (out / "index.html").write_text(_index(pages, where, vault.language), encoding="utf-8")
+    return len(pages)
+
+
+def _index(pages: list[Page], where: dict, language: str) -> str:
+    def items(selected: list[Page]) -> str:
+        rows = []
+        for p in selected:
+            summary = html.escape(str(p.meta.get("summary") or ""))
+            rows.append(
+                f'<li class="src"><a href="{where[id(p)]}">{html.escape(p.title)}</a>'
+                + (f"<br><small>{summary}</small>" if summary else "")
+                + "</li>"
+            )
+        return f"<ul>{''.join(rows)}</ul>"
+
+    sections = []
+    sources = sorted(
+        (p for p in pages if p.kind == "sources"),
+        key=lambda p: (str(p.meta.get("processed", "")), p.title),
+        reverse=True,
+    )
+    for heading, selected in (
+        (lang.t(language, "sources"), sources),
+        (
+            lang.t(language, "concepts"),
+            sorted((p for p in pages if p.kind == "concepts"), key=lambda p: fold(p.title)),
+        ),
+        (
+            lang.t(language, "entities"),
+            sorted((p for p in pages if p.kind == "entities"), key=lambda p: fold(p.title)),
+        ),
+        (
+            lang.t(language, "syntheses"),
+            sorted((p for p in pages if p.kind == "syntheses"), key=lambda p: fold(p.title)),
+        ),
+    ):
+        if selected:
+            sections.append(f"<h2>{heading} ({len(selected)})</h2>{items(selected)}")
+    return _page_html("esbi-cli", "<h1>esbi-cli</h1>" + "".join(sections), "", language)