okf-kit 0.1.0__py3-none-any.whl

okf_kit/__init__.py

@@ -0,0 +1,3 @@
+"""okf-kit — turn any website into a portable, agent-ready OKF bundle."""
+
+__version__ = "0.1.0"

okf_kit/bundle_nav.py

@@ -0,0 +1,97 @@
+"""Bundle navigation primitives shared by chat and the MCP server.
+
+These are the tools an agent uses to walk an OKF bundle — the same
+list_directory / read_concept pair proven to let independent agents navigate
+calknowledge bundles, plus a no-index keyword search. All are
+path-traversal-guarded.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from .config import STATE_DIRNAME
+from .okf import RESERVED
+
+MAX_FILE_CHARS = 12000
+_TOKEN = re.compile(r"[a-z0-9]{2,}")
+
+
+def _safe(bundle_dir: Path, path: str) -> Path | None:
+    target = (bundle_dir / path.strip().lstrip("/")).resolve()
+    root = bundle_dir.resolve()
+    if target == root or root in target.parents:
+        return target
+    return None
+
+
+def list_directory(bundle_dir, path: str = "/") -> str:
+    bundle_dir = Path(bundle_dir)
+    target = _safe(bundle_dir, path)
+    if not target or not target.is_dir():
+        return f"error: {path} is not a directory in this bundle. Try /index.md."
+    entries = sorted(
+        ("dir:  " if p.is_dir() else "file: ") + p.name
+        for p in target.iterdir()
+        if p.name != STATE_DIRNAME
+    )
+    return "\n".join(entries) or "(empty)"
+
+
+def read_concept(bundle_dir, path: str) -> str:
+    bundle_dir = Path(bundle_dir)
+    target = _safe(bundle_dir, path)
+    if not target or not target.is_file():
+        return f"error: {path} is not a file in this bundle. Read /index.md to see what exists."
+    return target.read_text(encoding="utf8")[:MAX_FILE_CHARS]
+
+
+def _concept_files(bundle_dir):
+    for md in (Path(bundle_dir) / "pages").rglob("*.md"):
+        if md.name not in RESERVED:
+            yield md
+
+
+def _split_frontmatter(text: str) -> tuple[str | None, str]:
+    """Return (title, body-without-frontmatter)."""
+    title = None
+    body = text
+    if text.startswith("---\n"):
+        end = text.find("\n---\n", 4)
+        if end != -1:
+            fm = text[4:end]
+            m = re.search(r"^title:\s*(.+)$", fm, re.MULTILINE)
+            if m:
+                title = m.group(1).strip()
+            body = text[end + 5 :]
+    return title, body
+
+
+def search_bundle(bundle_dir, query: str, limit: int = 5) -> list[dict]:
+    """Keyword search over concept files (title-boosted). No index needed."""
+    bundle_dir = Path(bundle_dir)
+    terms = set(_TOKEN.findall(query.lower()))
+    if not terms:
+        return []
+    results = []
+    for md in _concept_files(bundle_dir):
+        title, body = _split_frontmatter(md.read_text(encoding="utf8"))
+        body_l = body.lower()
+        title_l = (title or "").lower()
+        score = sum(body_l.count(t) + 3 * title_l.count(t) for t in terms)
+        if score:
+            rel = "/" + str(md.relative_to(bundle_dir))
+            results.append({"path": rel, "title": title, "score": score,
+                            "snippet": _snippet(body, terms)})
+    results.sort(key=lambda r: -r["score"])
+    return results[:limit]
+
+
+def _snippet(text: str, terms: set[str], width: int = 220) -> str:
+    lower = text.lower()
+    pos = min((lower.find(t) for t in terms if lower.find(t) >= 0), default=-1)
+    if pos < 0:
+        return ""
+    start = max(0, pos - width // 3)
+    return re.sub(r"\s+", " ", text[start : start + width]).strip()

okf_kit/chat/__init__.py

@@ -0,0 +1 @@
+"""Chat with a bundle (M3): providers, REPL, local history."""

okf_kit/chat/agent.py

@@ -0,0 +1,92 @@
+"""The bundle-navigation agent: answer a question by walking the OKF bundle.
+
+Ported from calknowledge's OKF-mode agent (verified to navigate large bundles
+correctly with independent LLMs): seed the root index, descend directory
+indexes to the most specific concept, answer only from what was read.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..bundle_nav import list_directory, read_concept
+
+MAX_STEPS = 16
+
+SYSTEM = """\
+You are a knowledge agent answering questions from an OKF bundle: a directory
+of markdown concept files with YAML frontmatter. Every directory has an
+index.md listing its contents.
+
+Navigate with the tools:
+- list_directory(path): list a directory (e.g. "/", "/pages", "/pages/docs")
+- read_concept(path): read a file (e.g. "/pages/docs/intro.md")
+
+Strategy: the root index is provided below. Descend through directory indexes
+to the MOST SPECIFIC concept for the question — don't stop at a general page
+when a dedicated one exists. If a tool errors, read the nearest parent
+index.md. Answer ONLY from concept files you have read; if the bundle doesn't
+contain the answer, say so. Cite the bundle paths you used."""
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "list_directory",
+            "description": "List entries in a directory of the OKF bundle.",
+            "parameters": {
+                "type": "object",
+                "properties": {"path": {"type": "string", "description": "Bundle path, e.g. /pages"}},
+                "required": ["path"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "read_concept",
+            "description": "Read a markdown concept file from the OKF bundle.",
+            "parameters": {
+                "type": "object",
+                "properties": {"path": {"type": "string", "description": "Bundle path, e.g. /index.md"}},
+                "required": ["path"],
+            },
+        },
+    },
+]
+
+
+def ask(bundle_dir, question: str, provider, *, max_steps: int = MAX_STEPS) -> dict:
+    """Run the navigation loop; returns {answer, steps, sources}."""
+    bundle_dir = Path(bundle_dir)
+    root_index = read_concept(bundle_dir, "/index.md")
+    messages = [
+        {"role": "system", "content": SYSTEM},
+        {"role": "user", "content": f"Question: {question}\n\nBundle root index (/index.md):\n{root_index}"},
+    ]
+    steps: list[dict] = []
+    read_paths: list[str] = []
+    answer = None
+
+    for _ in range(max_steps):
+        turn = provider.complete(messages, TOOLS)
+        messages.append(provider.assistant_message(turn))
+        if not turn.tool_calls:
+            answer = turn.text
+            break
+        for call in turn.tool_calls:
+            path = call.arguments.get("path", "/")
+            if call.name == "read_concept":
+                result = read_concept(bundle_dir, path)
+                if not result.startswith("error:") and path not in read_paths:
+                    read_paths.append(path)
+            else:
+                result = list_directory(bundle_dir, path)
+            steps.append({"tool": call.name, "path": path})
+            messages.append(provider.tool_result_message(call, result))
+
+    if answer is None:
+        messages.append({"role": "user", "content": "Stop exploring and answer from what you've read."})
+        answer = provider.complete(messages, []).text
+
+    return {"answer": answer or "", "steps": steps, "sources": read_paths}

okf_kit/chat/history.py

@@ -0,0 +1,51 @@
+"""Local chat history: one JSONL session file per conversation.
+
+Stored under ~/.okf/chats/<bundle>/<timestamp>.jsonl — nothing leaves the
+machine except the provider call itself.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..config import chats_dir
+from ..model import utcnow_iso
+
+
+class History:
+    def __init__(self, bundle_name: str, session: str | None = None):
+        self.dir = chats_dir() / bundle_name
+        self.dir.mkdir(parents=True, exist_ok=True)
+        if session:
+            self.path = self.dir / f"{session}.jsonl"
+        else:
+            self.path = self.dir / f"{utcnow_iso().replace(':', '-')}.jsonl"
+
+    def append(self, role: str, content: str, meta: dict | None = None) -> None:
+        rec = {"ts": utcnow_iso(), "role": role, "content": content}
+        if meta:
+            rec["meta"] = meta
+        with self.path.open("a", encoding="utf8") as fh:
+            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
+
+    def load(self) -> list[dict]:
+        if not self.path.exists():
+            return []
+        return [json.loads(line) for line in self.path.read_text(encoding="utf8").splitlines() if line.strip()]
+
+    @classmethod
+    def latest(cls, bundle_name: str) -> "History | None":
+        d = chats_dir() / bundle_name
+        sessions = sorted(d.glob("*.jsonl")) if d.exists() else []
+        if not sessions:
+            return None
+        return cls(bundle_name, session=sessions[-1].stem)
+
+    @classmethod
+    def list_sessions(cls, bundle_name: str) -> list[tuple[str, int]]:
+        d = chats_dir() / bundle_name
+        out = []
+        for f in sorted(d.glob("*.jsonl")) if d.exists() else []:
+            turns = sum(1 for line in f.read_text(encoding="utf8").splitlines() if '"role": "user"' in line)
+            out.append((f.stem, turns))
+        return out

okf_kit/chat/providers.py

@@ -0,0 +1,161 @@
+"""LLM provider abstraction for chat — deliberately tiny (no LangChain).
+
+One protocol, two implementations:
+  OpenAICompatProvider  — any OpenAI-compatible endpoint (OpenAI, Ollama,
+                          vLLM, LM Studio, OpenRouter). okf-kit[chat].
+  AnthropicProvider     — Claude with tool use. okf-kit[anthropic].
+
+`make_provider` returns None when no provider/key is available, so the caller
+falls back to zero-key retrieval.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+
+# Preset endpoints for OpenAI-compatible servers.
+_PRESETS = {
+    "openai": {"base_url": None, "key_env": "OPENAI_API_KEY", "default_model": "gpt-4o-mini"},
+    "ollama": {"base_url": "http://localhost:11434/v1", "key_env": None, "default_model": "llama3.1"},
+    "openrouter": {"base_url": "https://openrouter.ai/api/v1", "key_env": "OPENROUTER_API_KEY",
+                   "default_model": "openai/gpt-4o-mini"},
+}
+
+
+@dataclass
+class ToolCall:
+    id: str
+    name: str
+    arguments: dict
+
+
+@dataclass
+class Turn:
+    text: str | None = None
+    tool_calls: list[ToolCall] = field(default_factory=list)
+
+
+class OpenAICompatProvider:
+    def __init__(self, model: str, base_url: str | None, api_key: str):
+        try:
+            from openai import OpenAI
+        except ImportError as exc:  # noqa: TRY003
+            raise SystemExit("Chat needs the chat extra:  pip install 'okf-kit[chat]'") from exc
+        self.model = model
+        self._client = OpenAI(base_url=base_url, api_key=api_key)
+
+    def complete(self, messages: list[dict], tools: list[dict]) -> Turn:
+        resp = self._client.chat.completions.create(
+            model=self.model, messages=messages, tools=tools or None
+        )
+        msg = resp.choices[0].message
+        calls = [
+            ToolCall(id=tc.id, name=tc.function.name, arguments=_loads(tc.function.arguments))
+            for tc in (msg.tool_calls or [])
+        ]
+        return Turn(text=msg.content, tool_calls=calls)
+
+    def assistant_message(self, turn: Turn) -> dict:
+        m: dict = {"role": "assistant", "content": turn.text or ""}
+        if turn.tool_calls:
+            m["tool_calls"] = [
+                {"id": c.id, "type": "function",
+                 "function": {"name": c.name, "arguments": _dumps(c.arguments)}}
+                for c in turn.tool_calls
+            ]
+        return m
+
+    @staticmethod
+    def tool_result_message(call: ToolCall, content: str) -> dict:
+        return {"role": "tool", "tool_call_id": call.id, "content": content}
+
+
+class AnthropicProvider:
+    def __init__(self, model: str, api_key: str):
+        try:
+            from anthropic import Anthropic
+        except ImportError as exc:  # noqa: TRY003
+            raise SystemExit("Claude chat needs:  pip install 'okf-kit[anthropic]'") from exc
+        self.model = model
+        self._client = Anthropic(api_key=api_key)
+        self._system = None
+
+    def complete(self, messages: list[dict], tools: list[dict]) -> Turn:
+        system, msgs = _split_system(messages)
+        atools = [
+            {"name": t["function"]["name"], "description": t["function"]["description"],
+             "input_schema": t["function"]["parameters"]}
+            for t in tools
+        ]
+        resp = self._client.messages.create(
+            model=self.model, max_tokens=2048, system=system or "", messages=msgs, tools=atools or []
+        )
+        text_parts, calls = [], []
+        for block in resp.content:
+            if block.type == "text":
+                text_parts.append(block.text)
+            elif block.type == "tool_use":
+                calls.append(ToolCall(id=block.id, name=block.name, arguments=dict(block.input)))
+        return Turn(text="".join(text_parts) or None, tool_calls=calls)
+
+    def assistant_message(self, turn: Turn) -> dict:
+        content: list[dict] = []
+        if turn.text:
+            content.append({"type": "text", "text": turn.text})
+        for c in turn.tool_calls:
+            content.append({"type": "tool_use", "id": c.id, "name": c.name, "input": c.arguments})
+        return {"role": "assistant", "content": content}
+
+    @staticmethod
+    def tool_result_message(call: ToolCall, content: str) -> dict:
+        return {"role": "user", "content": [
+            {"type": "tool_result", "tool_use_id": call.id, "content": content}]}
+
+
+def make_provider(provider: str | None, model: str | None, base_url: str | None):
+    """Return a provider instance, or None to use the zero-key retrieval fallback."""
+    if provider in (None, "none"):
+        return None
+    if provider == "anthropic":
+        key = os.environ.get("ANTHROPIC_API_KEY")
+        if not key:
+            return None
+        return AnthropicProvider(model or "claude-sonnet-5", key)
+    if provider in _PRESETS or provider == "custom":
+        preset = _PRESETS.get(provider, {})
+        url = base_url or preset.get("base_url")
+        key_env = preset.get("key_env")
+        key = os.environ.get(key_env) if key_env else None
+        if provider == "ollama":
+            key = key or "ollama"  # Ollama ignores the key but the SDK requires one
+        if not key:
+            return None
+        return OpenAICompatProvider(model or preset.get("default_model", "gpt-4o-mini"), url, key)
+    raise SystemExit(f"Unknown provider '{provider}'. Use openai/ollama/openrouter/anthropic/custom.")
+
+
+def _loads(s):
+    import json
+
+    try:
+        return json.loads(s or "{}")
+    except json.JSONDecodeError:
+        return {}
+
+
+def _dumps(d):
+    import json
+
+    return json.dumps(d)
+
+
+def _split_system(messages):
+    system = None
+    msgs = []
+    for m in messages:
+        if m["role"] == "system":
+            system = m["content"]
+        else:
+            msgs.append(m)
+    return system, msgs

okf_kit/chat/repl.py

@@ -0,0 +1,65 @@
+"""Terminal chat loop for `okf chat`."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..registry import resolve_bundle
+from . import agent, retrieval
+from .history import History
+from .providers import make_provider
+
+
+def run_chat(
+    name_or_dir: str,
+    *,
+    provider: str | None = None,
+    model: str | None = None,
+    base_url: str | None = None,
+    trace: bool = False,
+    resume: bool = False,
+    show_history: bool = False,
+) -> int:
+    bundle_dir = resolve_bundle(name_or_dir)
+    bundle_name = Path(bundle_dir).name
+
+    if show_history:
+        sessions = History.list_sessions(bundle_name)
+        if not sessions:
+            print("No saved chats for this bundle.")
+        for name, turns in sessions:
+            print(f"{name}  ({turns} turns)")
+        return 0
+
+    prov = make_provider(provider, model, base_url)
+    history = History.latest(bundle_name) if resume else History(bundle_name)
+    if resume and history is None:
+        history = History(bundle_name)
+
+    if prov is None:
+        print("No LLM provider configured — retrieval-only mode.")
+        print("For a synthesized answer:  --provider ollama  (offline)  or  --provider openai\n")
+    else:
+        print(f"Chatting with '{bundle_name}' via {provider or 'default'} — Ctrl-D to exit.\n")
+
+    while True:
+        try:
+            question = input("you> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print()
+            return 0
+        if not question:
+            continue
+
+        history.append("user", question)
+        result = retrieval.answer(bundle_dir, question) if prov is None else agent.ask(
+            bundle_dir, question, prov
+        )
+        print(f"\n{result['answer']}\n")
+        if trace and result["steps"]:
+            print("  trace: " + " → ".join(f"{s['tool']} {s['path']}" for s in result["steps"]))
+        if result["sources"]:
+            print("  sources: " + ", ".join(result["sources"]))
+        print()
+        history.append("assistant", result["answer"],
+                       meta={"steps": result["steps"], "sources": result["sources"]})

okf_kit/chat/retrieval.py

@@ -0,0 +1,35 @@
+"""Zero-key fallback: answer from keyword retrieval with citations.
+
+When no LLM provider is configured, `okf chat` still does something useful —
+it returns the most relevant concepts with their matching passages quoted and
+cited, clearly labeled as retrieval-only. (Adopted from the deterministic
+`ask` in bushans/okfgen; see docs/ECOSYSTEM.md.)
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..bundle_nav import search_bundle
+
+
+def answer(bundle_dir, question: str, *, limit: int = 3) -> dict:
+    bundle_dir = Path(bundle_dir)
+    hits = search_bundle(bundle_dir, question, limit=limit)
+    if not hits:
+        return {
+            "answer": "No matching content found in this bundle (retrieval-only, no LLM configured).",
+            "steps": [{"tool": "search_bundle", "path": question}],
+            "sources": [],
+        }
+    lines = ["Retrieval-only answer (no LLM configured) — most relevant concepts:\n"]
+    for i, h in enumerate(hits, 1):
+        title = h["title"] or h["path"]
+        lines.append(f"{i}. {title}  [{h['path']}]")
+        lines.append(f"   {h['snippet']}\n")
+    lines.append("Configure a provider for a synthesized answer:  okf chat <bundle> --provider ollama")
+    return {
+        "answer": "\n".join(lines),
+        "steps": [{"tool": "search_bundle", "path": question}],
+        "sources": [h["path"] for h in hits],
+    }