dd-chat 0.1.0__py3-none-any.whl

dd_chat/app.py

@@ -0,0 +1,15 @@
+"""Multi-Chat: ask one question, let several LLMs answer side by side. A showcase of GraphLang.
+
+    streamlit run src/app.py        (from the repo root, so .streamlit/config.toml applies)
+"""
+import streamlit as st
+
+st.set_page_config(page_title="Multi-Chat · GraphLang", page_icon="💬", layout="wide")
+
+pages = [
+    st.Page("views/multi_chat.py", title="Chats", icon="💬", default=True),
+    st.Page("views/review.py", title="Review", icon="🔍"),
+    st.Page("views/config_llm.py", title="Settings", icon="⚙️"),
+    st.Page("views/about.py", title="About", icon="ℹ️"),
+]
+st.navigation(pages).run()

dd_chat/chat_db.py

@@ -0,0 +1,250 @@
+"""SQLite persistence for the app: chat sessions, and settings (Chats-page widgets, Settings-page model choices, presets).
+
+The file is `multi_chat.db` under the GraphLang home (`~/.graflang`, or $GRAFLANG_HOME). Tables:
+  settings(key, value)                one JSON value per key
+  sessions(id, title, created, updated, disabled, rev)    disabled = 1: soft-deleted, hidden from the lists but kept;
+                                      rev: bumped by every save, so two browser tabs cannot overwrite each other's chat (see save_session)
+  messages(session_id, bot, pos, role, content, ...)      one row per message; MSG_FIELDS lists the columns
+  judgements(session_id, q, judge, content, seconds)      a model's comparison of the answers to question q
+`bot` is the chatbot number minus one; `pos` the message's place in that chatbot's history.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from contextlib import contextmanager
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Iterator
+
+DB = Path(os.environ.get("GRAFLANG_HOME") or Path.home() / ".graflang") / "multi_chat.db"
+COLUMNS = {             # message field -> column type; every field but role and content may be missing from a message
+    "role": "TEXT NOT NULL", "content": "TEXT NOT NULL", "shown": "TEXT", "q": "INTEGER", "model": "TEXT", "seconds": "REAL",
+    "feedback": "INTEGER DEFAULT 0", "est_tokens": "INTEGER", "ctx_tokens": "INTEGER", "language": "TEXT", "links": "TEXT",
+    "input_tokens": "INTEGER", "output_tokens": "INTEGER", "cost_usd": "REAL", "best": "INTEGER DEFAULT 0", "shared_from": "INTEGER",
+    "temperature": "REAL", "instructions": "TEXT",
+}
+MSG_FIELDS = tuple(COLUMNS)
+JSON_FIELDS = ("links",)
+
+SCHEMA = f"""
+CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS sessions (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, created TEXT NOT NULL, updated TEXT NOT NULL,
+  disabled INTEGER NOT NULL DEFAULT 0, rev INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE IF NOT EXISTS messages (
+  session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, bot INTEGER NOT NULL, pos INTEGER NOT NULL,
+  {", ".join(f"{name} {kind}" for name, kind in COLUMNS.items())}, PRIMARY KEY (session_id, bot, pos));
+CREATE TABLE IF NOT EXISTS judgements (
+  session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, q INTEGER NOT NULL, judge TEXT, content TEXT NOT NULL, seconds REAL,
+  PRIMARY KEY (session_id, q));
+"""
+_ready: set[str] = set()                # databases whose schema this process has already checked
+
+
+def _migrate(con: sqlite3.Connection) -> None:
+    con.executescript(SCHEMA)
+    have = {r["name"] for r in con.execute("PRAGMA table_info(sessions)")}
+    for column, kind in (("disabled", "INTEGER NOT NULL DEFAULT 0"), ("rev", "INTEGER NOT NULL DEFAULT 0")):   # a database from before these
+        if column not in have:
+            con.execute(f"ALTER TABLE sessions ADD COLUMN {column} {kind}")
+    have = {r["name"] for r in con.execute("PRAGMA table_info(messages)")}
+    for column, kind in COLUMNS.items():
+        if column not in have:
+            con.execute(f"ALTER TABLE messages ADD COLUMN {column} {kind.replace(' NOT NULL', '')}")
+
+
+@contextmanager
+def connect() -> Iterator[sqlite3.Connection]:
+    """A connection in one transaction (committed on success, rolled back on an error), closed afterwards."""
+    DB.parent.mkdir(parents=True, exist_ok=True)
+    con = sqlite3.connect(DB, timeout=10)
+    try:
+        con.row_factory = sqlite3.Row
+        con.execute("PRAGMA foreign_keys = ON")
+        if str(DB) not in _ready:
+            with con:
+                _migrate(con)
+            _ready.add(str(DB))
+        with con:
+            yield con
+    finally:
+        con.close()
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+# ---- settings
+
+def get_setting(key: str, default: Any = None) -> Any:
+    with connect() as con:
+        row = con.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
+    try:
+        return json.loads(row["value"]) if row else default
+    except ValueError:
+        return default
+
+
+def set_setting(key: str, value) -> None:
+    with connect() as con:
+        con.execute("INSERT INTO settings(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, json.dumps(value)))
+
+
+def delete_setting(key: str) -> None:
+    with connect() as con:
+        con.execute("DELETE FROM settings WHERE key = ?", (key,))
+
+
+# ---- chat sessions
+
+def _title(histories: dict[int, list[dict]]) -> str:
+    first = next((m.get("shown") or m["content"] for _, h in sorted(histories.items()) for m in h if m["role"] == "user"), "chat")
+    return " ".join(first.split())[:80]
+
+
+def _row(m: dict) -> tuple:
+    return tuple(json.dumps(m[f]) if f in JSON_FIELDS and m.get(f) else m.get(f) for f in MSG_FIELDS)
+
+
+def _write(con: sqlite3.Connection, session_id: int, histories: dict[int, list[dict]], judgements: dict[int, dict] | None) -> None:
+    con.execute("DELETE FROM messages WHERE session_id = ?", (session_id,))
+    con.executemany(
+        f"INSERT INTO messages(session_id, bot, pos, {', '.join(MSG_FIELDS)}) VALUES(?, ?, ?, {', '.join('?' * len(MSG_FIELDS))})",
+        [(session_id, bot, pos, *_row(m)) for bot, history in histories.items() for pos, m in enumerate(history)])
+    if judgements is not None:
+        con.execute("DELETE FROM judgements WHERE session_id = ?", (session_id,))
+        con.executemany("INSERT INTO judgements(session_id, q, judge, content, seconds) VALUES(?, ?, ?, ?, ?)",
+                        [(session_id, int(q), j.get("judge"), j["content"], j.get("seconds")) for q, j in judgements.items()])
+
+
+def save_session(session_id: int | None, histories: dict[int, list[dict]], rev: int = 0,
+                 judgements: dict[int, dict] | None = None) -> tuple[int | None, int, bool]:
+    """Write every chatbot's history (and the judgements) as one session; returns (session id, its new rev, forked).
+
+    `rev` is the revision this browser session last loaded or wrote. The write goes into `session_id` only when nobody else has written
+    to it since and it is not disabled; otherwise the chat is saved as a new session (forked = True), so no tab overwrites another's
+    messages and a chat disabled on the Review page does not keep growing out of sight. Nothing to save: (session_id, rev, False)."""
+    if not any(histories.values()):
+        return session_id, rev, False
+    title = _title(histories)
+    with connect() as con:
+        if session_id is not None:
+            moved = con.execute("UPDATE sessions SET title = ?, updated = ?, rev = rev + 1 WHERE id = ? AND rev = ? AND disabled = 0",
+                                (title, _now(), session_id, rev)).rowcount
+            if moved:
+                _write(con, session_id, histories, judgements)
+                return session_id, rev + 1, False
+        forked = session_id is not None
+        new_id = int(con.execute("INSERT INTO sessions(title, created, updated, rev) VALUES(?, ?, ?, 1)", (title, _now(), _now())).lastrowid or 0)
+        _write(con, new_id, histories, judgements or {})
+        return new_id, 1, forked
+
+
+def _message(r: sqlite3.Row) -> dict:
+    m = {f: r[f] for f in MSG_FIELDS if r[f] is not None}
+    for f in JSON_FIELDS:
+        if f in m:
+            try:
+                m[f] = json.loads(m[f])
+            except ValueError:
+                del m[f]
+    m["feedback"] = r["feedback"] or 0
+    if not m.get("best"):
+        m.pop("best", None)
+    return m
+
+
+def load_session(session_id: int) -> dict[int, list[dict]]:
+    """{bot index: history} for a session (messages as the Chats page keeps them)."""
+    out: dict[int, list[dict]] = {}
+    with connect() as con:
+        for r in con.execute("SELECT * FROM messages WHERE session_id = ? ORDER BY bot, pos", (session_id,)):
+            out.setdefault(r["bot"], []).append(_message(r))
+    return out
+
+
+def load_judgements(session_id: int) -> dict[int, dict]:
+    with connect() as con:
+        return {r["q"]: {"judge": r["judge"], "content": r["content"], "seconds": r["seconds"]}
+                for r in con.execute("SELECT * FROM judgements WHERE session_id = ? ORDER BY q", (session_id,))}
+
+
+def session_rev(session_id: int) -> int:
+    with connect() as con:
+        row = con.execute("SELECT rev FROM sessions WHERE id = ?", (session_id,)).fetchone()
+    return row["rev"] if row else 0
+
+
+def latest_session_id() -> int | None:
+    with connect() as con:
+        row = con.execute("SELECT id FROM sessions WHERE disabled = 0 ORDER BY updated DESC, id DESC LIMIT 1").fetchone()
+    return row["id"] if row else None
+
+
+def _like(text: str) -> str:
+    return "%" + text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+
+
+def list_sessions(include_disabled: bool = False, search: str = "") -> list[dict]:
+    """Newest first: id, title, created, updated, disabled, questions, chatbots, models. Disabled sessions only when asked for;
+    with `search`, only the sessions whose title or any message contains it (any case)."""
+    where, args = "(? OR s.disabled = 0)", [int(include_disabled)]
+    if search.strip():
+        where += (" AND (s.title LIKE ? ESCAPE '\\' OR EXISTS (SELECT 1 FROM messages m WHERE m.session_id = s.id"
+                  " AND (m.content LIKE ? ESCAPE '\\' OR m.shown LIKE ? ESCAPE '\\')))")
+        args += [_like(search.strip())] * 3
+    with connect() as con:
+        rows = con.execute(f"""
+            SELECT s.id, s.title, s.created, s.updated, s.disabled,
+                   (SELECT COUNT(DISTINCT q) FROM messages m WHERE m.session_id = s.id AND m.role = 'user') AS questions,
+                   (SELECT COUNT(DISTINCT bot) FROM messages m WHERE m.session_id = s.id) AS chatbots,
+                   (SELECT GROUP_CONCAT(model, ', ') FROM (SELECT DISTINCT model FROM messages m WHERE m.session_id = s.id AND model IS NOT NULL)) AS models,
+                   (SELECT ROUND(SUM(cost_usd), 4) FROM messages m WHERE m.session_id = s.id) AS cost_usd
+            FROM sessions s WHERE {where} ORDER BY s.updated DESC, s.id DESC""", args).fetchall()
+    return [dict(r) for r in rows]
+
+
+def set_session_disabled(session_id: int, disabled: bool) -> None:
+    """Soft delete: a disabled session is hidden from the Review table (unless asked for) and is not restored by the Chats page; its data stays."""
+    with connect() as con:
+        con.execute("UPDATE sessions SET disabled = ? WHERE id = ?", (int(disabled), session_id))
+
+
+def delete_session(session_id: int) -> None:
+    """Hard delete: the session, its messages and judgements are gone."""
+    with connect() as con:
+        con.execute("DELETE FROM sessions WHERE id = ?", (session_id,))
+
+
+def delete_disabled() -> int:
+    """Hard-delete every disabled session; returns how many."""
+    with connect() as con:
+        return con.execute("DELETE FROM sessions WHERE disabled = 1").rowcount
+
+
+def leaderboard() -> list[dict]:
+    """Per model, over the enabled sessions: answers, 👍, 👎, wins (🏆 picks), contests (questions it answered that got a 🏆 pick),
+    average seconds, output tokens and cost per answer, and the total cost. An answer copied from a twin chatbot is not counted."""
+    with connect() as con:
+        rows = con.execute("""
+            WITH answers AS (
+              SELECT m.session_id, u.q, m.model, COALESCE(m.feedback, 0) AS feedback, COALESCE(m.best, 0) AS best, m.seconds, m.output_tokens, m.cost_usd
+              FROM messages m JOIN sessions s ON s.id = m.session_id
+              JOIN messages u ON u.session_id = m.session_id AND u.bot = m.bot AND u.pos = m.pos - 1      -- the question it answers
+              WHERE s.disabled = 0 AND m.role = 'assistant' AND m.model IS NOT NULL AND m.shared_from IS NULL),
+            picked AS (SELECT DISTINCT session_id, q FROM answers WHERE best = 1)
+            SELECT a.model AS model, COUNT(*) AS answers,
+                   SUM(a.feedback = 1) AS up, SUM(a.feedback = -1) AS down, SUM(a.best = 1) AS wins,
+                   SUM(p.q IS NOT NULL) AS contests,
+                   ROUND(AVG(a.seconds), 1) AS avg_seconds, ROUND(AVG(a.output_tokens)) AS avg_output_tokens,
+                   ROUND(AVG(a.cost_usd), 5) AS avg_cost_usd, ROUND(SUM(a.cost_usd), 4) AS total_cost_usd
+            FROM answers a LEFT JOIN picked p ON p.session_id = a.session_id AND p.q = a.q
+            GROUP BY a.model ORDER BY wins DESC, up - down DESC, answers DESC""").fetchall()
+    out = []
+    for r in rows:
+        d = dict(r)
+        d["win_rate"] = round(d["wins"] / d["contests"], 2) if d["contests"] else None
+        out.append(d)
+    return out

dd_chat/chat_engine.py

@@ -0,0 +1,229 @@
+"""Asking the models: one chatbot's turn, a turn with pictures, and a comparison of answers. No Streamlit here, so it runs in the
+chatbots' threads and in tests (with the `mock` adapter).
+
+A text turn is one run of `chat_turn.graph.yaml`; a comparison is one run of `compare_answers.graph.yaml`. Every run is metered
+(`MeteredLLM`, capped at one model call), and its tokens, cost and model time come back with the reply.
+
+Kernels: the graph's SOLVE nodes run in a Python (Jupyter) kernel. GraphLang starts a kernel for each run and stops it at the end, which
+costs most of a second per chatbot per question. Here a few kernel pools are kept and lent to runs one at a time (`_kernels`); the
+runtime's own pool is swapped for a lent one before the run. This leans on `GraphLangRuntime.kernels`, an attribute GraphLang does not
+document: when it is missing the run simply starts its own kernel as before. The kernels' sockets go in the repo's `tmp/`
+(`GRAFLANG_KERNEL_DIR`; set it yourself to choose another folder).
+"""
+from __future__ import annotations
+
+import atexit
+import base64
+import json
+import os
+import queue
+import time
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Iterator
+
+import yaml
+from graphlang.grafr.adapters import make_llm
+from graphlang.grafr.kernel import KernelPool
+from graphlang.grafr.metering import MeteredLLM
+from graphlang.grafr.runtime import GraphLangRuntime, load_graph
+
+ROOT = Path(__file__).resolve().parent
+# The kernels' Unix sockets go in the repo's tmp/ (git-ignored) when run from a checkout, else in ~/.graflang/tmp (an installed package must not write into site-packages); never the folder Streamlit started in.
+TMP = ROOT.parent / "tmp" if (ROOT.parent / "tests").is_dir() else Path(os.environ.get("GRAFLANG_HOME", Path.home() / ".graflang")) / "tmp"
+os.environ.setdefault("GRAFLANG_KERNEL_DIR", str(TMP))      # read by GraphLang when a kernel starts; an older GraphLang ignores it
+TMP.mkdir(parents=True, exist_ok=True)
+TMP.chmod(0o700)                            # the sockets accept code: only this user may reach them
+GRAPH = ROOT / "chat_turn.graph.yaml"
+COMPARE_GRAPH = ROOT / "compare_answers.graph.yaml"
+SAME_LANGUAGE = "the same language as the most recent user message"
+IMAGE_FORMATS = {"ollama": "openai", "openrouter": "openai", "openai": "openai", "qwen": "openai", "anthropic": "anthropic"}  # adapter -> image format
+CHARS_PER_TOKEN = 4
+
+
+@dataclass
+class Turn:
+    """What one chatbot is asked with: `adapter:model`, recent turns it sees, reply language, its own instructions and temperature."""
+    llm: str
+    last_n: int = 5
+    language: str = ""              # "" = the language of the question
+    instructions: str = ""
+    temperature: float | None = None
+    prices: tuple[float, float] | None = None      # $ per million prompt / completion tokens, to estimate a cost the adapter does not report
+
+
+@dataclass
+class Answer:
+    reply: str
+    seconds: float                  # wall time of the turn, kernel and all
+    model_ms: int = 0               # time inside the model call
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float | None = None   # None: not known (the adapter reports none and the price is not known)
+
+    def fields(self) -> dict:
+        return asdict(self)
+
+
+@lru_cache(maxsize=None)
+def graph_input_default(name: str, graph: Path = GRAPH) -> str:
+    """The default of a graph input, so the image turns use the same text as the graph (e.g. `rules`)."""
+    doc = yaml.safe_load(graph.read_text(encoding="utf-8"))
+    return next(str(i.get("default", "")) for i in doc["workflow"]["inputs"] if i["name"] == name)
+
+
+def reply_language(language: str) -> str:
+    """The phrase that completes "Write the reply in ...": the chatbot's language, or the language of the question when it has none."""
+    return language or SAME_LANGUAGE
+
+
+# ---- kernels lent to runs
+
+class _KeptPool(KernelPool):
+    """A kernel pool that outlives the run: the runtime's `close()` at the end of a run does nothing, `shutdown()` really stops it."""
+
+    def close(self) -> None:
+        pass
+
+    def shutdown(self) -> None:
+        KernelPool.close(self)
+
+
+_idle: "queue.SimpleQueue[_KeptPool]" = queue.SimpleQueue()
+
+
+@contextmanager
+def _kernels() -> Iterator[_KeptPool]:
+    """An idle kept pool, or a new one; given back afterwards. A pool whose run failed is stopped, not reused (its kernel may be broken)."""
+    try:
+        pool = _idle.get_nowait()
+    except queue.Empty:
+        pool = _KeptPool("python3", str(ROOT))
+        atexit.register(pool.shutdown)
+    try:
+        yield pool
+    except BaseException:
+        pool.shutdown()
+        raise
+    _idle.put(pool)
+
+
+def _run(graph: Path, llm: Any, inputs: dict) -> dict:
+    runtime = GraphLangRuntime(load_graph(graph), llm=llm, verbose=False)
+    if not isinstance(getattr(runtime, "kernels", None), KernelPool):     # a GraphLang without the attribute: let it start its own kernel
+        return runtime.execute(inputs)
+    with _kernels() as pool:
+        runtime.kernels = pool
+        return runtime.execute(inputs)
+
+
+# ---- the model, with a chatbot's temperature
+
+class _Tuned:
+    """The run's LLM with this chatbot's temperature (GraphLang takes a temperature only from the node's config)."""
+
+    def __init__(self, inner: Any, temperature: float | None):
+        self.inner, self.temperature = inner, temperature
+
+    def __getattr__(self, attr):
+        if attr == "inner":
+            raise AttributeError(attr)
+        return getattr(self.inner, attr)
+
+    def generate(self, prompt: str, model: str | None = None, **opts: Any) -> str:
+        if self.temperature is not None:
+            opts["temperature"] = self.temperature
+        return self.inner.generate(prompt, model=model, **opts)
+
+
+def _metered(turn: Turn) -> MeteredLLM:
+    adapter, _, model = turn.llm.partition(":")
+    return MeteredLLM(_Tuned(make_llm(adapter, model or None), turn.temperature), max_calls=1)
+
+
+def _cost(reported: float | None, input_tokens: int, output_tokens: int, prices: tuple[float, float] | None) -> float | None:
+    if reported:
+        return reported
+    if prices:
+        return round((input_tokens * prices[0] + output_tokens * prices[1]) / 1_000_000, 6)
+    return None
+
+
+def _answer(reply: str, started: float, meter: MeteredLLM, prices: tuple[float, float] | None) -> Answer:
+    snap = meter.snapshot()
+    reported = None if meter.cost_unreported else snap["cost_usd"]
+    if meter.name == "claude_cli":                  # the CLI reports cost 0 (subscription) and estimated tokens: the cost is not known
+        reported = None
+    return Answer(reply, time.monotonic() - started, snap["ms"], snap["input_tokens"], snap["output_tokens"],
+                  _cost(reported, snap["input_tokens"], snap["output_tokens"], prices))
+
+
+# ---- a chatbot's turn
+
+def ask(question: str, history: list[dict], turn: Turn) -> Answer:
+    """One run of the graph for one chatbot. The graph sees only role and content."""
+    started = time.monotonic()
+    meter = _metered(turn)
+    plain = [{"role": m["role"], "content": m["content"]} for m in history]
+    out = _run(GRAPH, meter, {"history": json.dumps(plain), "question": question, "last_n": turn.last_n,
+                              "language": reply_language(turn.language), "instructions": turn.instructions,
+                              "rules": graph_input_default("rules")})
+    return _answer(out["reply"], started, meter, turn.prices)
+
+
+def system_prompt(turn: Turn) -> str:
+    """The system prompt of an image turn: the graph's prompt, said directly."""
+    return " ".join(filter(None, ["You are a helpful assistant engaged in a multi-turn conversation.", turn.instructions,
+                                  f"Reply helpfully, in {reply_language(turn.language)}.", graph_input_default("rules")]))
+
+
+def image_messages(question: str, history: list[dict], last_n: int, images: list[tuple[str, bytes]], fmt: str) -> list[dict]:
+    """The recent history, then the question with its images, in the adapter's message format."""
+    recent = [{"role": m["role"], "content": m["content"]} for m in history[-2 * last_n:]]
+    encoded = [(mime, base64.b64encode(data).decode("ascii")) for mime, data in images]
+    if fmt == "anthropic":
+        parts = [{"type": "image", "source": {"type": "base64", "media_type": mime, "data": b64}} for mime, b64 in encoded]
+        content = parts + [{"type": "text", "text": question}]
+    else:
+        content = [{"type": "text", "text": question}] + [{"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}} for mime, b64 in encoded]
+    return recent + [{"role": "user", "content": content}]
+
+
+def ask_with_images(question: str, history: list[dict], turn: Turn, images: list[tuple[str, bytes]]) -> Answer:
+    """A turn with pictures, straight to the adapter (the graph passes text only)."""
+    adapter, _, model = turn.llm.partition(":")
+    fmt = IMAGE_FORMATS.get(adapter)
+    if not fmt:
+        raise RuntimeError("this chatbot cannot read images; use an Ollama vision model, or an OpenRouter, OpenAI, Qwen or Anthropic API model")
+    import dd_llm
+    started = time.monotonic()
+    options = {"temperature": turn.temperature} if turn.temperature is not None else {}
+    result = dd_llm.get_adapter(adapter).call(model=model, messages=image_messages(question, history, turn.last_n, images, fmt),
+                                              system=system_prompt(turn), **options)
+    if not result.success:
+        errors = result.error_history or []
+        raise RuntimeError(errors[-1]["error"] if errors else f"{adapter} call failed")
+    tokens_in, tokens_out = int(result.input_tokens or 0), int(result.output_tokens or 0)
+    return Answer(result.content, time.monotonic() - started, int(result.latency_ms or 0), tokens_in, tokens_out,
+                  _cost(result.cost_usd, tokens_in, tokens_out, turn.prices))
+
+
+def estimate_tokens(question: str, history: list[dict], turn: Turn) -> int:
+    """A rough count of the prompt a turn sends (four characters a token): the recent history, the question, and the graph's own text."""
+    recent = history[-2 * turn.last_n:]
+    chars = sum(len(m["content"]) + 30 for m in recent) + len(question) + len(turn.instructions) + len(graph_input_default("rules")) + 300
+    return chars // CHARS_PER_TOKEN
+
+
+# ---- comparing answers
+
+def compare(question: str, answers: list[tuple[str, str]], turn: Turn) -> Answer:
+    """One run of the comparison graph: `answers` are (label, text)."""
+    started = time.monotonic()
+    meter = _metered(turn)
+    text = "\n\n".join(f"### {label}\n{reply}" for label, reply in answers)
+    out = _run(COMPARE_GRAPH, meter, {"question": question, "answers": text,
+                                      "language": turn.language or "the same language as the question"})
+    return _answer(out["comparison"], started, meter, turn.prices)

dd_chat/chat_export.py

@@ -0,0 +1,168 @@
+"""Export and import of a chat: YAML (complete, can be read back) and Markdown (to read or share).
+
+The YAML is organised by question. Each question has its text (`question`, as the page shows it), the full prompt when attached files made
+it longer (`prompt`), the answers (chatbot number, `<provider>:<model-id>`, language, seconds, tokens, cost, feedback 1 / -1 / 0, `best`,
+temperature, instructions, links, `shared_from` when a twin chatbot's reply was copied, and the response) and the comparison, if one was
+made. Older exports (just chatbot, llm, language, seconds, feedback, response) still import.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+import yaml
+
+from chat_models import ADAPTER_PROVIDER, PROVIDERS
+
+ANSWER_FIELDS = ("seconds", "input_tokens", "output_tokens", "cost_usd", "temperature", "instructions", "links", "shared_from",
+                 "est_tokens", "ctx_tokens")
+
+
+class _Dumper(yaml.SafeDumper):
+    pass
+
+
+def _text(dumper: yaml.SafeDumper, value: str):
+    """Multi-line text as a literal block (`|`), so a response reads as it was written; PyYAML falls back to quoting when it must."""
+    return dumper.represent_scalar("tag:yaml.org,2002:str", value, style="|" if "\n" in value else None)
+
+
+_Dumper.add_representer(str, _text)
+
+
+def _clean(text: str) -> str:
+    """Line ends unified and control characters (which YAML cannot hold) removed."""
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    return "".join(c for c in text if c in "\n\t" or (ord(c) >= 32 and ord(c) != 127))
+
+
+def by_question(histories: dict[int, list[dict]]) -> dict[int, dict]:
+    """{question number: {question, prompt, answers: [(chatbot index, assistant message)]}} across the chatbots."""
+    out: dict[int, dict] = {}
+    for i, history in sorted(histories.items()):
+        asked, pending = 0, None
+        for m in history:
+            if m["role"] == "user":
+                asked += 1
+                pending = m
+            elif pending is not None:
+                q = pending.get("q", asked)
+                entry = out.setdefault(q, {"question": pending.get("shown") or pending["content"], "prompt": pending["content"], "answers": []})
+                entry["answers"].append((i, m))
+                pending = None
+    return dict(sorted(out.items()))
+
+
+def provider_llm(model: str | None) -> str:
+    adapter, _, name = (model or ":").partition(":")
+    return f"{ADAPTER_PROVIDER.get(adapter, adapter)}:{name}"
+
+
+def export_yaml(histories: dict[int, list[dict]], judgements: dict[int, dict] | None = None, session: dict | None = None) -> str:
+    questions = []
+    for q, entry in by_question(histories).items():
+        item: dict = {"number": q, "question": _clean(entry["question"])}
+        if entry["prompt"] != entry["question"]:
+            item["prompt"] = _clean(entry["prompt"])
+        answers = []
+        for i, m in entry["answers"]:
+            a: dict = {"chatbot": i + 1, "llm": provider_llm(m.get("model")), "language": m.get("language") or "",
+                       "feedback": m.get("feedback", 0), "best": bool(m.get("best"))}
+            for f in ANSWER_FIELDS:
+                if m.get(f) not in (None, "", {}):
+                    a[f] = round(m[f], 2) if f == "seconds" else m[f]
+            a["response"] = _clean(m["content"])
+            answers.append(a)
+        item["answers"] = answers
+        if judgements and q in judgements:
+            j = judgements[q]
+            item["comparison"] = {"judge": provider_llm(j.get("judge")), "seconds": j.get("seconds"), "content": _clean(j["content"])}
+        questions.append(item)
+    doc: dict = {"exported": datetime.now().isoformat(timespec="seconds"), "format": 2}
+    if session:
+        doc["session"] = session
+    doc["questions"] = questions
+    return yaml.dump(doc, Dumper=_Dumper, sort_keys=False, allow_unicode=True, width=10_000)
+
+
+def export_markdown(histories: dict[int, list[dict]], judgements: dict[int, dict] | None = None, title: str = "Multi-Chat") -> str:
+    lines = [f"# {title}", "", f"Exported {datetime.now():%Y-%m-%d %H:%M}", ""]
+    for q, entry in by_question(histories).items():
+        lines += [f"## [{q}] {' '.join(entry['question'].split())[:120]}", ""]
+        if len(entry["question"]) > 120 or "\n" in entry["question"]:
+            lines += ["> " + line for line in entry["question"].split("\n")] + [""]
+        for i, m in entry["answers"]:
+            facts = [provider_llm(m.get("model"))]
+            if m.get("seconds") is not None:
+                facts.append(f"{m['seconds']:.1f}s")
+            if m.get("cost_usd") is not None:
+                facts.append(f"${m['cost_usd']:.4f}")
+            facts += [s for s, on in (("🏆 best", m.get("best")), ("👍", m.get("feedback") == 1), ("👎", m.get("feedback") == -1)) if on]
+            lines += [f"### Chatbot {i + 1} · " + " · ".join(facts), "", m["content"].strip(), ""]
+        if judgements and q in judgements:
+            lines += [f"### Comparison · {provider_llm(judgements[q].get('judge'))}", "", judgements[q]["content"].strip(), ""]
+    return "\n".join(lines)
+
+
+def _int(value, default: int) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _float(value) -> float | None:
+    try:
+        return float(value) if value is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _model(llm) -> str:
+    """`provider:model` in the file -> `adapter:model`, as the page keeps it."""
+    provider, _, name = str(llm or "").partition(":")
+    return f"{PROVIDERS.get(provider, provider)}:{name}" if name else str(llm or "")
+
+
+def parse_export(data: bytes) -> tuple[dict[int, list[dict]], dict[int, dict]]:
+    """The chat in an export file: ({chatbot index: history} like the Chats page keeps it, {question number: judgement}).
+    Raises ValueError if it is not an export."""
+    try:
+        doc = yaml.safe_load(data)
+    except yaml.YAMLError as e:
+        raise ValueError(f"not valid YAML: {e}") from e
+    if not isinstance(doc, dict) or not isinstance(doc.get("questions"), list):
+        raise ValueError("this does not look like a Multi-Chat export (no `questions` list)")
+    histories: dict[int, list[dict]] = {}
+    judgements: dict[int, dict] = {}
+    for n, q in enumerate(doc["questions"], 1):
+        if not isinstance(q, dict) or "question" not in q:
+            continue
+        number = _int(q.get("number"), n)
+        question = str(q["question"])
+        prompt = str(q.get("prompt") or question)
+        for a in q.get("answers") or []:
+            if not isinstance(a, dict):
+                continue
+            bot = max(0, _int(a.get("chatbot"), 1) - 1)
+            user = {"role": "user", "content": prompt, "q": number}
+            if prompt != question:
+                user["shown"] = question
+            reply = {"role": "assistant", "content": str(a.get("response") or ""), "model": _model(a.get("llm")),
+                     "language": str(a["language"]) if a.get("language") else None, "feedback": _int(a.get("feedback"), 0) if a.get("feedback") in (1, -1, 0, "1", "-1", "0") else 0}
+            if a.get("best"):
+                reply["best"] = 1
+            for f in ("seconds", "cost_usd", "temperature"):
+                if _float(a.get(f)) is not None:
+                    reply[f] = _float(a.get(f))
+            for f in ("input_tokens", "output_tokens", "est_tokens", "ctx_tokens", "shared_from"):
+                if a.get(f) is not None and _int(a.get(f), -1) >= 0:
+                    reply[f] = _int(a.get(f), 0)
+            if isinstance(a.get("instructions"), str) and a["instructions"]:
+                reply["instructions"] = a["instructions"]
+            if isinstance(a.get("links"), dict):
+                reply["links"] = {str(k): str(v) for k, v in a["links"].items()}
+            histories.setdefault(bot, []).extend([user, reply])
+        c = q.get("comparison")
+        if isinstance(c, dict) and c.get("content"):
+            judgements[number] = {"judge": _model(c.get("judge")), "content": str(c["content"]), "seconds": _float(c.get("seconds"))}
+    return histories, judgements