esbi-cli 0.2.1__py3-none-any.whl

esbi_cli/report/daily_index.py

@@ -0,0 +1,235 @@
+"""The daily index note: where the day starts. Regenerated from state on every run."""
+
+import re
+from datetime import date
+from pathlib import Path
+
+from esbi_cli import lang
+from esbi_cli.extract import is_url
+from esbi_cli.links import link_targets
+from esbi_cli.queue import Queue, normalize_target
+from esbi_cli.runlog import RunLog
+from esbi_cli.vault import Page, Vault, fold
+
+
+def _sources_where(vault: Vault, key: str, today: date) -> list[Page]:
+    found = [p for p in vault.iter_pages(("sources",)) if p.meta.get(key) == today.isoformat()]
+    return sorted(found, key=lambda p: fold(p.title))
+
+
+def _read_section(vault: Vault, today: date) -> list[str]:
+    pages = _sources_where(vault, "read", today)
+    if not pages:
+        return [lang.t(vault.language, "nothing_read")]
+    return [f"- [[{p.title}]] — {p.meta.get('summary', '')}" for p in pages]
+
+
+def _processed_section(vault: Vault, today: date) -> list[str]:
+    lines = []
+    for page in _sources_where(vault, "processed", today):
+        box = "x" if page.meta.get("status") == "read" else " "
+        lines.append(f"- [{box}] [[{page.title}]] — {page.meta.get('summary', '')}")
+    return lines or [lang.t(vault.language, "nothing_processed")]
+
+
+QUEUE_LIST_LIMIT = 20
+
+
+def _short(target: str) -> str:
+    return target if is_url(target) else Path(target).name
+
+
+def _name(item) -> str:
+    """The most readable name we have: the label (clip title), else the URL or file name."""
+    return item.label or _short(item.target)
+
+
+def _pending(vault: Vault, queue: Queue) -> list:
+    """Queued items not already in the wiki: those are skipped at run time with no model call."""
+    known = {
+        normalize_target(str(p.meta["url"]))
+        for p in vault.iter_pages(("sources",))
+        if p.meta.get("url")
+    }
+    return [i for i in queue.items("queued") if normalize_target(i.target) not in known]
+
+
+def _queue_section(vault: Vault, queue: Queue) -> list[str]:
+    items = _pending(vault, queue)
+    L = vault.language
+    if not items:
+        return [lang.t(L, "queue_empty")]
+    total = len(items)
+    header = lang.t(L, "queue_one" if total == 1 else "queue_many", n=total)
+    if total > QUEUE_LIST_LIMIT:
+        header += lang.t(L, "queue_cap", n=QUEUE_LIST_LIMIT)
+    names = [_name(i) for i in items]
+    return [header + ".", *(f"- {n}" for n in names[:QUEUE_LIST_LIMIT])]
+
+
+def _review_section(vault: Vault, queue: Queue) -> list[str]:
+    notes = sorted((vault.wiki / "review").glob("*.md"))
+    lines = [f"- [[{n.stem}]]" for n in notes]
+    L = vault.language
+    lines += [
+        lang.t(L, "could_not_process", name=_name(i), error=i.error) for i in queue.items("failed")
+    ]
+    lines += [
+        lang.t(L, "retrying", name=_name(i), n=i.attempts, max=queue.max_attempts, error=i.error)
+        for i in queue.items("queued")
+        if i.attempts
+    ]
+    return lines or [lang.t(L, "nothing_pending")]
+
+
+REVISIT_COUNT = 3
+
+
+def _revisit_section(vault: Vault, today: date) -> list[str]:
+    """A few older pages, rotating with the date so each day surfaces different ones."""
+    older = [
+        p
+        for p in vault.iter_pages()
+        if str(p.meta.get("processed") or p.meta.get("updated") or "") < today.isoformat()
+    ]
+    if not older:
+        return [lang.t(vault.language, "no_old_notes")]
+    older.sort(key=lambda p: (p.kind, fold(p.title)))
+    start = (today.toordinal() * REVISIT_COUNT) % len(older)
+    picked = [older[(start + k) % len(older)] for k in range(min(REVISIT_COUNT, len(older)))]
+    return [f"- [[{p.title}]] — {p.meta.get('summary', '')}" for p in picked]
+
+
+def _orphans(pages: list[Page]) -> int:
+    """Pages that no other page links to."""
+    linked: set[str] = set()
+    for page in pages:
+        own = fold(page.title)
+        linked |= {fold(t) for t in link_targets(page.body)} - {own}
+    return sum(fold(p.title) not in linked for p in pages)
+
+
+def _stats_section(vault: Vault, today: date) -> list[str]:
+    pages = list(vault.iter_pages())
+    by_kind = {k: [p for p in pages if p.kind == k] for k in ("sources", "concepts", "entities")}
+    sources = by_kind["sources"]
+    day = today.isoformat()
+    touched = [p for p in by_kind["concepts"] + by_kind["entities"] if p.meta.get("updated") == day]
+    read = sum(p.meta.get("status") == "read" for p in sources)
+    L = vault.language
+    return [
+        lang.t(
+            L,
+            "stat_pages",
+            total=len(pages),
+            sources=len(sources),
+            concepts=len(by_kind["concepts"]),
+            entities=len(by_kind["entities"]),
+        ),
+        lang.t(
+            L,
+            "stat_today",
+            processed=sum(p.meta.get("processed") == day for p in sources),
+            touched=len(touched),
+        ),
+        lang.t(L, "stat_read", read=read, total=len(sources)),
+        lang.t(L, "stat_orphans", n=_orphans(pages)),
+    ]
+
+
+STOP_REASONS = {  # RunRecord.stopped_by -> label key
+    "max_sources": "stop_max_sources",
+    "token_budget": "stop_token_budget",
+    "llm_unavailable": "stop_llm_unavailable",
+}
+
+
+def _runs_section(vault: Vault, today: date) -> list[str]:
+    runs = [
+        r for r in RunLog(vault.root / ".esbi" / "runs.jsonl").runs() if r.started.date() == today
+    ]
+    L = vault.language
+    if not runs:
+        return [lang.t(L, "no_runs")]
+    lines = []
+    for run in runs:
+        minutes = int((run.finished - run.started).total_seconds() // 60)
+        line = lang.t(
+            L,
+            "run_line",
+            at=f"{run.started:%H:%M}",
+            manual=lang.t(L, "run_manual") if run.trigger == "manual" else "",
+            ingested=run.ingested,
+            skipped=run.skipped,
+            failed=run.failed,
+            tokens=run.tokens_used,
+            duration=lang.t(L, "run_minutes", n=minutes)
+            if minutes
+            else lang.t(L, "run_under_minute"),
+        )
+        if run.stopped_by:
+            key = STOP_REASONS.get(run.stopped_by)
+            line += lang.t(L, "run_stopped", reason=lang.t(L, key) if key else run.stopped_by)
+        lines.append(line)
+    return lines
+
+
+HOME_START, HOME_END = "<!-- esbi:start -->", "<!-- esbi:end -->"
+
+
+def _update_home(vault: Vault, queue: Queue, today: date) -> None:
+    """Rewrite only the worker-managed block of Home.md; the rest of the page stays the user's."""
+    unread = sum(p.meta.get("status") == "processed" for p in vault.iter_pages(("sources",)))
+    L = vault.language
+    block = "\n".join(
+        [
+            HOME_START,
+            lang.t(L, "home_today", link=f"[[{today.isoformat()}]]"),
+            lang.t(L, "home_unread", n=unread),
+            lang.t(L, "home_queue", n=len(_pending(vault, queue))),
+            HOME_END,
+        ]
+    )
+    home = vault.root / "Home.md"
+    text = home.read_text(encoding="utf-8") if home.exists() else "# esbi-cli\n"
+    text = text.replace("<!-- second-brain:start -->", HOME_START).replace(  # legacy markers
+        "<!-- second-brain:end -->", HOME_END
+    )
+    pattern = re.compile(re.escape(HOME_START) + r".*?" + re.escape(HOME_END), re.DOTALL)
+    if pattern.search(text):
+        text = pattern.sub(lambda _: block, text)
+    else:
+        text = text.rstrip("\n") + "\n\n" + block + "\n"
+    home.write_text(text, encoding="utf-8")
+
+
+def build_daily_index(vault: Vault, queue: Queue, today: date) -> Path:
+    L = vault.language
+    body = [
+        f"# {lang.t(L, 'daily_title', date=today.isoformat())}",
+        "",
+        f"## {lang.t(L, 'daily_read')}",
+        *_read_section(vault, today),
+        "",
+        f"## {lang.t(L, 'daily_processed')}",
+        *_processed_section(vault, today),
+        "",
+        f"## {lang.t(L, 'daily_queue')}",
+        *_queue_section(vault, queue),
+        "",
+        f"## {lang.t(L, 'daily_review')}",
+        *_review_section(vault, queue),
+        "",
+        f"## {lang.t(L, 'daily_revisit')}",
+        *_revisit_section(vault, today),
+        "",
+        f"## {lang.t(L, 'daily_runs')}",
+        *_runs_section(vault, today),
+        "",
+        f"## {lang.t(L, 'daily_stats')}",
+        *_stats_section(vault, today),
+    ]
+    path = vault.wiki / "daily" / f"{today.isoformat()}.md"
+    vault.write_page(Page(path, {"type": "daily", "date": today.isoformat()}, "\n".join(body)))
+    _update_home(vault, queue, today)
+    return path

esbi_cli/report/index_md.py

@@ -0,0 +1,21 @@
+"""Rebuild index.md deterministically from page frontmatter (no LLM needed)."""
+
+from esbi_cli import lang
+from esbi_cli.vault import Vault, fold
+
+SECTIONS = ("sources", "concepts", "entities", "syntheses")  # page folders, and their label keys
+
+
+def rebuild_index(vault: Vault) -> None:
+    by_kind: dict[str, list[tuple[str, str]]] = {kind: [] for kind in SECTIONS}
+    for page in vault.iter_pages():
+        by_kind[page.kind].append((page.title, str(page.meta.get("summary") or "")))
+    total = sum(len(v) for v in by_kind.values())
+    L = vault.language
+    lines = [f"# {lang.t(L, 'index_title')}", "", lang.t(L, "index_blurb", total=total), ""]
+    for kind in SECTIONS:
+        lines += [f"## {lang.t(L, kind)} ({len(by_kind[kind])})", ""]
+        for title, summary in sorted(by_kind[kind], key=lambda t: fold(t[0])):
+            lines.append(f"- [[{title}]]" + (f" — {summary}" if summary else ""))
+        lines.append("")
+    (vault.root / "index.md").write_text("\n".join(lines), encoding="utf-8")

esbi_cli/report/readstate.py

@@ -0,0 +1,26 @@
+"""Read tracking: ticked checkboxes in the daily index notes mark sources as read."""
+
+import re
+from datetime import date
+
+from esbi_cli.vault import Vault
+
+_TICKED = re.compile(r"^\s*- \[[xX]\] \[\[([^\]|#]+)", re.MULTILINE)
+
+
+def sync_read_state(vault: Vault, today: date) -> list[str]:
+    """Flip ticked sources from processed to read. Returns the titles newly marked read."""
+    ticked: list[str] = []
+    for note in sorted((vault.wiki / "daily").glob("*.md")):
+        ticked += _TICKED.findall(note.read_text(encoding="utf-8"))
+
+    newly_read: list[str] = []
+    for title in dict.fromkeys(t.strip() for t in ticked):
+        page = vault.find_page(title, ("sources",))
+        if page is None or page.meta.get("status") != "processed":
+            continue
+        page.meta["status"] = "read"
+        page.meta["read"] = today.isoformat()
+        vault.write_page(page)
+        newly_read.append(page.title)
+    return newly_read

esbi_cli/run.py

@@ -0,0 +1,100 @@
+"""Drain the ingest queue within limits, isolating failures per item."""
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from esbi_cli.ingest.pipeline import IngestResult
+from esbi_cli.interrupts import Interrupted, handling, interruptible
+from esbi_cli.llm.adapter import LLMError, LLMTimeout
+from esbi_cli.queue import Queue
+
+
+@dataclass
+class RunLimits:
+    max_sources: int
+    max_tokens: int | None = None
+
+
+@dataclass
+class Failure:
+    name: str  # the label (clip title, file name) or the target
+    attempt: int  # which attempt this was
+    error: str
+    parked: bool  # out of attempts: it waits for `sb retry`
+
+
+@dataclass
+class RunSummary:
+    ingested: int = 0
+    skipped: int = 0
+    failed: int = 0
+    failures: list[Failure] = field(default_factory=list)
+    tokens_used: int = 0
+    # "max_sources" | "token_budget" | "llm_unavailable" | "interrupted" | None (queue drained)
+    stopped_by: str | None = None
+    signum: int | None = None  # set when stopped_by == "interrupted"
+    released: int = 0  # items put back in the queue by an interruption
+
+
+def run_queue(
+    queue: Queue,
+    ingest_fn: Callable[[str], IngestResult],
+    limits: RunLimits,
+    tokens_used: Callable[[], int] = lambda: 0,
+) -> RunSummary:
+    summary = RunSummary()
+    start_tokens = tokens_used()
+    tried: set[int] = set()  # a failed item is requeued, but must wait for the next run
+    item = None
+    with handling():
+        try:
+            while True:
+                with interruptible():  # a signal that came between two items stops the run here
+                    pass
+                claimed = queue.claim(1, exclude=tried)
+                if not claimed:
+                    break
+                item = claimed[0]
+                if len(tried) >= limits.max_sources:
+                    queue.release(item.id)
+                    summary.stopped_by = "max_sources"
+                    break
+                if (
+                    limits.max_tokens is not None
+                    and tokens_used() - start_tokens >= limits.max_tokens
+                ):
+                    queue.release(item.id)
+                    summary.stopped_by = "token_budget"
+                    break
+                tried.add(item.id)
+                try:
+                    with interruptible():
+                        result = ingest_fn(item.target)
+                except Exception as exc:  # one bad source must not stop the nightly batch
+                    if isinstance(exc, LLMError) and not isinstance(exc, LLMTimeout):
+                        # the model is the problem, not this source: keep it fresh and stop
+                        queue.release(item.id)
+                        summary.stopped_by = "llm_unavailable"
+                        break
+                    error = f"{type(exc).__name__}: {exc}"
+                    queue.fail(item.id, error)
+                    attempt = item.attempts + 1
+                    summary.failures.append(
+                        Failure(
+                            item.label or item.target, attempt, error, attempt >= queue.max_attempts
+                        )
+                    )
+                    summary.failed += 1
+                    continue
+                queue.complete(item.id)
+                if result.status == "skipped":
+                    summary.skipped += 1
+                else:
+                    summary.ingested += 1
+        except Interrupted as exc:
+            # like an outage: the item goes back untouched. Release only affects a row still
+            # `processing`, so a finished item is not undone.
+            summary.stopped_by, summary.signum = "interrupted", exc.signum
+            summary.released = int(item is not None and queue.release(item.id))
+    summary.tokens_used = tokens_used() - start_tokens
+    return summary

esbi_cli/runlock.py

@@ -0,0 +1,31 @@
+"""Single-run guard: an OS-level file lock that dies with the process, so it can never get stuck."""
+
+import fcntl
+from pathlib import Path
+
+
+class LockBusy(RuntimeError):
+    pass
+
+
+class RunLock:
+    def __init__(self, path: Path):
+        self.path = path
+        self._file = None
+
+    def __enter__(self) -> "RunLock":
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        file = self.path.open("a")
+        try:
+            fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            file.close()
+            raise LockBusy(f"another run holds {self.path}") from None
+        self._file = file
+        return self
+
+    def __exit__(self, *exc_info) -> None:
+        if self._file:
+            fcntl.flock(self._file, fcntl.LOCK_UN)
+            self._file.close()
+            self._file = None

esbi_cli/runlog.py

@@ -0,0 +1,80 @@
+"""Append-only record of every `sb run`: feeds the daily index and the "is the nightly due?" check."""
+
+import json
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+
+
+@dataclass
+class RunRecord:
+    started: datetime
+    finished: datetime
+    ingested: int
+    skipped: int
+    failed: int
+    tokens_used: int
+    stopped_by: str | None  # None = queue drained; see run.RunSummary
+    trigger: str  # "scheduled" (sb run --if-due) or "manual"
+
+
+class RunLog:
+    def __init__(self, path: Path, keep: int = 500):
+        self.path, self.keep = path, keep  # older runs are dropped: only recent ones are ever read
+
+    def record(self, run: RunRecord) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        row = {
+            **asdict(run),
+            "started": run.started.isoformat(),
+            "finished": run.finished.isoformat(),
+        }
+        with self.path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(row) + "\n")
+        lines = self.path.read_text(encoding="utf-8").splitlines()
+        if len(lines) > self.keep:
+            self.path.write_text("\n".join(lines[-self.keep :]) + "\n", encoding="utf-8")
+
+    def runs(self) -> list[RunRecord]:
+        if not self.path.exists():
+            return []
+        rows = [
+            json.loads(line) for line in self.path.read_text(encoding="utf-8").splitlines() if line
+        ]
+        return [
+            RunRecord(
+                **{
+                    **row,
+                    "started": datetime.fromisoformat(row["started"]),
+                    "finished": datetime.fromisoformat(row["finished"]),
+                }
+            )
+            for row in rows
+        ]
+
+
+def is_due(runs: list[RunRecord], now: datetime, at: tuple[int, int] = (3, 0)) -> bool:
+    """Has the nightly run still to happen?
+
+    True until a scheduled run that reached the LLM has started since the most recent `at` (hour, minute)
+    boundary. Runs stopped by an LLM outage or an interruption don't count (retried on the next
+    hourly check) and neither do manual runs (which may be partial).
+    """
+    boundary = now.replace(hour=at[0], minute=at[1], second=0, microsecond=0)
+    if now < boundary:
+        boundary -= timedelta(days=1)
+    return not any(
+        r.trigger == "scheduled"
+        and r.stopped_by not in ("llm_unavailable", "interrupted")
+        and r.started >= boundary
+        for r in runs
+    )
+
+
+def trim_log(path: Path, limit_bytes: int = 1_000_000, keep_bytes: int = 200_000) -> None:
+    """Cut a log that passed `limit_bytes` to its last `keep_bytes`, from a whole line. launchd
+    appends to this file, so it is trimmed in place at the start of a run, never renamed."""
+    if not path.is_file() or path.stat().st_size <= limit_bytes:
+        return
+    tail = path.read_bytes()[-keep_bytes:]
+    path.write_bytes(tail[tail.find(b"\n") + 1 :])

esbi_cli/schedule.py

@@ -0,0 +1,106 @@
+"""launchd integration: `sb run --if-due` at the chosen time (or the next wake), at login, and hourly as a retry net."""
+
+import plistlib
+import shlex
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+LABEL = "com.esbi-cli.nightly"
+OLD_LABEL = "com.secondbrain.nightly"  # legacy: the name before esbi-cli
+CHECK_EVERY_SECONDS = 3600
+
+Launchctl = Callable[[list[str]], subprocess.CompletedProcess]
+
+
+class ScheduleError(RuntimeError):
+    pass
+
+
+def run_launchctl(args: list[str]) -> subprocess.CompletedProcess:
+    return subprocess.run(["launchctl", *args], capture_output=True, text=True)
+
+
+def stable_prefix(prefix: Path) -> Path:
+    """Under Homebrew the environment lives in `<brew prefix>/Cellar/esbi-cli/<version>/libexec`, a
+    folder `brew upgrade` deletes. `<brew prefix>/opt/esbi-cli` is a symlink brew keeps pointing at
+    the current version: that is the path a long-lived job must use. Any other prefix is unchanged."""
+    parts = prefix.parts
+    for i in range(len(parts) - 2):
+        if parts[i : i + 2] == ("Cellar", "esbi-cli"):
+            return Path(*parts[:i], "opt", "esbi-cli", *parts[i + 3 :])
+    return prefix
+
+
+def render_plist(config: Path, log_dir: Path, venv: Path, at: tuple[int, int] = (3, 0)) -> bytes:
+    """`venv` is the environment that runs `sb`; it need not live inside the repo. `at` is the
+    time of day, (hour, minute), the nightly job runs."""
+    venv = stable_prefix(venv)
+    command = (
+        # macOS sometimes flags the venv as hidden, and Python 3.13 then ignores its .pth files
+        f"chflags -R nohidden {shlex.quote(str(venv))} 2>/dev/null; "
+        f"exec /usr/bin/caffeinate -i {shlex.quote(str(venv / 'bin' / 'sb'))} run --if-due "
+        f"--config {shlex.quote(str(config))}"
+    )
+    log = str(log_dir / "nightly.log")
+    return plistlib.dumps(
+        {
+            "Label": LABEL,
+            "ProgramArguments": ["/bin/bash", "-c", command],
+            # Not the repo: it usually lives in ~/Documents, and macOS privacy (TCC) then blocks the
+            # job at start-up. Everything the job needs is passed as an absolute path.
+            "WorkingDirectory": str(Path.home()),
+            "StartInterval": CHECK_EVERY_SECONDS,
+            # cron skips jobs while the Mac sleeps; launchd runs a calendar job at the next wake
+            "StartCalendarInterval": {"Hour": at[0], "Minute": at[1]},
+            "RunAtLoad": True,
+            "StandardOutPath": log,
+            "StandardErrorPath": log,
+            "EnvironmentVariables": {"PATH": "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"},
+        }
+    )
+
+
+def _plist_path(agents_dir: Path) -> Path:
+    return agents_dir / f"{LABEL}.plist"
+
+
+def _remove_old_job(agents_dir: Path, uid: int, launchctl: Launchctl) -> bool:
+    """legacy: a job installed under the old label would run next to the new one."""
+    old = agents_dir / f"{OLD_LABEL}.plist"
+    unloaded = launchctl(["bootout", f"gui/{uid}/{OLD_LABEL}"]).returncode == 0
+    existed = old.exists()
+    old.unlink(missing_ok=True)
+    return unloaded or existed
+
+
+def cron_line(config: Path) -> str:
+    """What to put in cron where there is no launchd (hourly is fine: it runs once a day)."""
+    return f"0 * * * * sb run --if-due --config {config.resolve()}"
+
+
+def install(plist: bytes, agents_dir: Path, uid: int, launchctl: Launchctl = run_launchctl) -> Path:
+    """Write the LaunchAgent and (re)load it into the user's GUI session."""
+    path = _plist_path(agents_dir)
+    # first, so that a system without launchctl fails before anything is written
+    _remove_old_job(agents_dir, uid, launchctl)
+    launchctl(["bootout", f"gui/{uid}/{LABEL}"])  # a previous version may be loaded; ignore errors
+    agents_dir.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(plist)
+    result = launchctl(["bootstrap", f"gui/{uid}", str(path)])
+    if result.returncode != 0:
+        raise ScheduleError(result.stderr.strip() or f"launchctl exited with {result.returncode}")
+    return path
+
+
+def uninstall(agents_dir: Path, uid: int, launchctl: Launchctl = run_launchctl) -> bool:
+    """Unload and delete the LaunchAgent. Returns False if there was nothing to remove."""
+    path = _plist_path(agents_dir)
+    unloaded = launchctl(["bootout", f"gui/{uid}/{LABEL}"]).returncode == 0
+    existed = path.exists()
+    path.unlink(missing_ok=True)
+    return _remove_old_job(agents_dir, uid, launchctl) or unloaded or existed
+
+
+def is_loaded(uid: int, launchctl: Launchctl = run_launchctl) -> bool:
+    return launchctl(["print", f"gui/{uid}/{LABEL}"]).returncode == 0

esbi_cli/templates/SCHEMA.md

@@ -0,0 +1,52 @@
+# SCHEMA — the rules of this wiki
+
+This document defines how the worker (and the LLM) maintains this wiki. It is given to the model as context on every operation. Based on Karpathy's "LLM Wiki" pattern.
+
+## Language
+All notes are written in **{language}**. Verbatim quotes keep the language of their source. The language is set by `[notes].language` in config.toml; if that setting and this paragraph disagree, the setting wins.
+
+## Layers
+1. `raw/`: the original sources, **immutable** (markdown snapshots, PDFs). Never edited.
+2. `wiki/`: pages generated and maintained by the LLM. The human reads; the LLM writes.
+3. `SCHEMA.md`: this file.
+
+## Page types (`wiki/`)
+- `sources/`: one summary per source.
+- `concepts/`: ideas, techniques, topics. One page per concept, which grows with each new source.
+- `entities/`: people, organizations, tools, papers.
+- `syntheses/`: analyses that cross several sources, and useful answers to questions.
+- `daily/YYYY-MM-DD.md`: the daily index.
+- `review/`: the queue of things for the human to check.
+
+## Frontmatter
+Sources (`wiki/sources/`):
+```yaml
+---
+type: source
+title: ""
+url: ""            # or a path in raw/
+kind: article|paper|email|other
+status: processed  # queued | processed | read | failed
+captured: YYYY-MM-DD
+processed: YYYY-MM-DD
+read: null         # YYYY-MM-DD when the human marks it as read
+tags: []
+---
+```
+Concepts and entities: `type`, `title`, `aliases`, `tags`, `sources` (a list of wikilinks), `updated`.
+
+## Links
+- Use wikilinks: `[[Page title]]`.
+- Every source links to the concepts and entities it mentions, and the other way round (a concept lists its sources).
+- Prefer updating an existing page to creating a duplicate. Check aliases.
+
+## Operations
+- **Ingest**: summarize the source, extract concepts and entities, create or update pages, link them, update `index.md`, add a line to `log.md`.
+- **Query**: answer by citing wiki pages; if the answer is valuable, save it in `syntheses/`.
+- **Lint**: find orphans, stale claims, contradictions, concepts without a page, broken links.
+
+## Contradictions and doubts
+When a new source contradicts existing content, do not overwrite: note the contradiction on both pages and create a note in `review/`. Merges, deletions and uncertain links also go to `review/`.
+
+## Log
+`log.md` is append-only. Format: `## [YYYY-MM-DD] ingest | Title`.

esbi_cli/templates/clipper-template.json

@@ -0,0 +1,17 @@
+{
+  "schemaVersion": "0.1.0",
+  "name": "esbi-cli",
+  "behavior": "create",
+  "noteNameFormat": "{{title|safe_name}}",
+  "path": "inbox",
+  "triggers": [],
+  "properties": [
+    { "name": "title", "value": "{{title}}", "type": "text" },
+    { "name": "source", "value": "{{url}}", "type": "text" },
+    { "name": "author", "value": "{{author}}", "type": "text" },
+    { "name": "published", "value": "{{published}}", "type": "text" },
+    { "name": "kind", "value": "article", "type": "text" }
+  ],
+  "noteContentFormat": "{{content}}",
+  "context": ""
+}