deeptutor-knowledge 1.7.0__py3-none-any.whl

deeptutor/api/routers/marginnote4.py

@@ -0,0 +1,278 @@
+"""HTTP bridge for MarginNote 4 Add-on devices.
+
+The MN4 Add-on (JavaScript running inside MarginNote 4) calls these endpoints
+to pair with this DeepTutor instance and push synced study data.
+
+Authentication layers:
+* ``/pair``, ``/devices``, ``/status`` -- DeepTutor session auth (the logged-in
+  user manages their own devices). The router is mounted with ``_auth`` in
+  ``main.py``.
+* ``/sync``, ``/heartbeat`` -- device-token auth via
+  ``Authorization: MarginNote <device_id>:<token>``. The Add-on stores the
+  token received at pairing time.
+
+Phase 1 scope: device pairing, incremental sync, heartbeat. Write-back to MN4
+(propose / apply / verify) is Phase 2.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from deeptutor.api.routers.auth import require_auth
+from deeptutor.capabilities.marginnote4.models import MarginNoteObject, SyncBatch
+from deeptutor.capabilities.marginnote4.store import MarginNoteStore, resolve_db_path
+from deeptutor.services.path_service import PathService
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+_auth = [Depends(require_auth)]
+
+# Upper bound on objects (and tombstones) accepted from one sync request.
+MAX_SYNC_BATCH = 2000
+
+
+def _requested_kb(request: Request) -> str:
+    """The MN4 library this request addresses.
+
+    In Phase 1 a single default store serves all devices; ``X-MN4-KB`` selects
+    a dedicated database instead. The header is only a *selector* — the device
+    token is the credential, and it is checked against whichever store the
+    header names, so naming another library grants nothing.
+    """
+    return request.headers.get("x-mn4-kb", "default")
+
+
+def _device_db_path(kb_name: str) -> Path:
+    """The database the device-token endpoints address.
+
+    Those endpoints carry no session, so ``get_path_service()`` would hand them
+    whatever the ambient context happens to be — the default workspace in
+    practice. Naming it explicitly keeps the sync path from drifting away from
+    the store pairing wrote to, and gives :func:`pair_device` something to
+    check itself against.
+    """
+    return resolve_db_path(kb_name, path_service=PathService.get_instance())
+
+
+def _store_for(request: Request) -> MarginNoteStore:
+    """Resolve (creating if absent) the store for a session-authenticated call."""
+    return MarginNoteStore(resolve_db_path(_requested_kb(request)))
+
+
+def _auth_device(request: Request, authorization: str | None) -> tuple[str, MarginNoteStore]:
+    """Validate the device token and return ``(device_id, store)``.
+
+    Reached with no session, so nothing here may create state from
+    caller-supplied input: ``open_existing`` keeps an unauthenticated request
+    from materialising a directory and a schema'd database per distinct
+    ``X-MN4-KB`` value.
+    """
+    if not authorization or not authorization.startswith("MarginNote "):
+        raise HTTPException(401, "Missing or malformed Authorization header.")
+    raw = authorization[len("MarginNote ") :]
+    if ":" not in raw:
+        raise HTTPException(401, "Invalid Authorization format.")
+    device_id, token = raw.split(":", 1)
+    store = MarginNoteStore.open_existing(_device_db_path(_requested_kb(request)))
+    if store is None or not store.verify_token(device_id, token):
+        raise HTTPException(403, "Invalid device credentials.")
+    store.touch_device(device_id)
+    return device_id, store
+
+
+# -- request / response models ---------------------------------------------
+
+
+class PairRequest(BaseModel):
+    device_name: str = Field("", max_length=128)
+    device_kind: str = Field("macos", max_length=32)
+
+
+class PairResponse(BaseModel):
+    device_id: str
+    token: str
+    device_name: str
+    device_kind: str
+
+
+class SyncObjectIn(BaseModel):
+    object_id: str
+    object_type: str
+    title: str = ""
+    content: str = ""
+    excerpt: str | None = None
+    document_id: str | None = None
+    document_title: str | None = None
+    page: int | None = None
+    tags: list[str] = Field(default_factory=list)
+    links: list[str] = Field(default_factory=list)
+    color: str | None = None
+    created_at: str = ""
+    updated_at: str = ""
+    raw: dict[str, Any] = Field(default_factory=dict)
+
+
+class SyncRequest(BaseModel):
+    # One batch, not one library: the Add-on pages through its backlog, so an
+    # unbounded list only ever meant a token holder could pin the event loop
+    # and the database on a single request.
+    cursor: str = Field("", max_length=256)
+    objects: list[SyncObjectIn] = Field(default_factory=list, max_length=MAX_SYNC_BATCH)
+    deleted_ids: list[str] = Field(default_factory=list, max_length=MAX_SYNC_BATCH)
+
+
+class SyncResponse(BaseModel):
+    stored: int
+    updated: int
+    deleted: int
+    new_cursor: str
+
+
+class DeviceInfo(BaseModel):
+    device_id: str
+    device_name: str
+    device_kind: str
+    paired_at: str
+    last_seen: str
+    active: bool
+
+
+# -- session-authenticated endpoints (DeepTutor user) -----------------------
+
+
+@router.post("/pair", response_model=PairResponse, dependencies=_auth)
+async def pair_device(body: PairRequest, request: Request) -> PairResponse:
+    """Pair a new MN4 device. Requires a DeepTutor session.
+
+    Returns a one-time token the Add-on stores and presents on every sync.
+    """
+    kb_name = _requested_kb(request)
+    if resolve_db_path(kb_name) != _device_db_path(kb_name):
+        # Pairing runs under a session and resolves the caller's own
+        # workspace; /sync does not and resolves the default one. Where those
+        # differ, pairing would hand out a token that 403s on every sync
+        # forever, so refuse instead of issuing a dead credential.
+        raise HTTPException(
+            501,
+            "MN4 device sync is not available for this account yet: pairing and "
+            "sync would resolve different workspaces.",
+        )
+    store = _store_for(request)
+    device, token = store.pair_device(device_name=body.device_name, device_kind=body.device_kind)
+    logger.info("Paired MN4 device %s (%s)", device.device_id, device.device_name)
+    return PairResponse(
+        device_id=device.device_id,
+        token=token,
+        device_name=device.device_name,
+        device_kind=device.device_kind,
+    )
+
+
+@router.get("/devices", response_model=list[DeviceInfo], dependencies=_auth)
+async def list_devices(request: Request) -> list[DeviceInfo]:
+    """List all paired devices."""
+    store = _store_for(request)
+    return [
+        DeviceInfo(
+            device_id=d.device_id,
+            device_name=d.device_name,
+            device_kind=d.device_kind,
+            paired_at=d.paired_at,
+            last_seen=d.last_seen,
+            active=d.active,
+        )
+        for d in store.list_devices()
+    ]
+
+
+@router.delete("/devices/{device_id}", dependencies=_auth)
+async def revoke_device(device_id: str, request: Request) -> dict[str, str]:
+    """Revoke a paired device."""
+    store = _store_for(request)
+    if not store.revoke_device(device_id):
+        raise HTTPException(404, f"Device {device_id} not found.")
+    return {"status": "revoked", "device_id": device_id}
+
+
+@router.get("/status", dependencies=_auth)
+async def status(request: Request) -> dict[str, Any]:
+    """Health check and summary stats."""
+    store = _store_for(request)
+    return {
+        "status": "ok",
+        "devices": len(store.list_devices()),
+        "objects": store.count(),
+    }
+
+
+# -- device-token-authenticated endpoints (MN4 Add-on) ---------------------
+
+
+@router.post("/sync", response_model=SyncResponse)
+async def sync_objects(
+    body: SyncRequest,
+    request: Request,
+    authorization: str | None = Header(None),
+) -> SyncResponse:
+    """Receive an incremental sync batch from a paired MN4 device."""
+    device_id, store = _auth_device(request, authorization)
+    objects = [
+        MarginNoteObject(
+            object_id=o.object_id,
+            object_type=o.object_type,
+            title=o.title,
+            content=o.content,
+            excerpt=o.excerpt,
+            document_id=o.document_id,
+            document_title=o.document_title,
+            page=o.page,
+            tags=o.tags,
+            links=o.links,
+            color=o.color,
+            created_at=o.created_at,
+            updated_at=o.updated_at,
+            device_id=device_id,
+            raw=o.raw,
+        )
+        for o in body.objects
+    ]
+    batch = SyncBatch(
+        device_id=device_id,
+        cursor=body.cursor,
+        objects=objects,
+        deleted_ids=body.deleted_ids,
+    )
+    result = store.ingest(batch)
+    logger.info(
+        "MN4 sync from %s: +%d ~%d -%d",
+        device_id,
+        result.stored,
+        result.updated,
+        result.deleted,
+    )
+    return SyncResponse(
+        stored=result.stored,
+        updated=result.updated,
+        deleted=result.deleted,
+        new_cursor=result.new_cursor,
+    )
+
+
+@router.post("/heartbeat")
+async def heartbeat(
+    request: Request,
+    authorization: str | None = Header(None),
+) -> dict[str, Any]:
+    """Lightweight liveness check. Updates last_seen for the device."""
+    device_id, store = _auth_device(request, authorization)
+    return {
+        "status": "ok",
+        "device_id": device_id,
+        "object_count": store.count(device_id=device_id),
+    }

deeptutor/capabilities/ima/__init__.py

@@ -0,0 +1,6 @@
+"""IMA loop capability — inventory, source reading and authoring over a live library."""
+
+from deeptutor.capabilities.ima.capability import ImaCapability
+from deeptutor.capabilities.ima.tools import IMA_TOOL_NAMES, IMA_TOOL_TYPES
+
+__all__ = ["IMA_TOOL_NAMES", "IMA_TOOL_TYPES", "ImaCapability"]

deeptutor/capabilities/ima/binding.py

@@ -0,0 +1,118 @@
+"""Resolve which connected Tencent IMA libraries the current turn can reach.
+
+The binding is derived once per turn from the user's selected knowledge bases:
+every selection whose KB metadata is ``type == ima`` becomes a library the IMA
+tools may operate on. Unlike the Obsidian binding (one live vault per turn) all
+selected IMA libraries are kept, because a turn can legitimately browse one and
+read from another — the tools take an explicit ``kb_name`` when there is more
+than one.
+
+Credentials are deliberately *not* part of a binding. What a tool call carries is
+the KB reference; the credential pair is loaded from ``kb_config.json`` at call
+time by :func:`resolve_client`, which also re-runs the per-user access check. So
+no secret is ever placed in tool kwargs (where a trace or a log could pick it up),
+and a model cannot reach a library by naming one that was not selected.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from deeptutor.core.context import UnifiedContext
+from deeptutor.knowledge.kb_types import IMA_KB_TYPE
+
+# Cached per extension: a tuple of bindings, empty once we have looked and
+# found none. Absence of the key means "not resolved yet".
+_CACHE_KEY = "_ima_bindings"
+
+
+@dataclass(frozen=True, slots=True)
+class ImaBinding:
+    """One selected knowledge base that points at a Tencent IMA library."""
+
+    kb_ref: str
+    """The reference as the user selected it (name or id) — what tools pass back."""
+
+    name: str
+    """Display name, used to match a model-supplied ``kb_name``."""
+
+    knowledge_base_id: str
+    """Which IMA library this KB reads. Not a secret (it is shown in the UI)."""
+
+
+def ima_bindings(context: UnifiedContext) -> tuple[ImaBinding, ...]:
+    """Every connected IMA library among this turn's selected knowledge bases."""
+    state = context.extension("ima")
+    cached = state.get(_CACHE_KEY)
+    if cached is not None:
+        return tuple(cached)
+    resolved = _resolve(context)
+    state[_CACHE_KEY] = resolved
+    return resolved
+
+
+def select_binding(
+    bindings: tuple[ImaBinding, ...],
+    kb_name: str | None = None,
+) -> ImaBinding | None:
+    """The binding a tool call targets, chosen from what the turn made available.
+
+    With one selected library the argument is optional (and ignored when it does
+    not match, since there is no ambiguity to resolve). With several, an
+    unmatched name resolves to ``None`` so the tool can ask for a valid one
+    instead of silently operating on the wrong library. A pure function over the
+    turn's bindings — the model can only ever reach a library that was selected.
+    """
+    if not bindings:
+        return None
+    requested = str(kb_name or "").strip()
+    if not requested:
+        return bindings[0] if len(bindings) == 1 else None
+    for binding in bindings:
+        if requested in {binding.kb_ref, binding.name, binding.knowledge_base_id}:
+            return binding
+    return bindings[0] if len(bindings) == 1 else None
+
+
+def resolve_client(kb_ref: str, *, for_write: bool = False):
+    """An :class:`ImaClient` for *kb_ref*, after re-checking the user's access.
+
+    Raises ``ImaNotConfiguredError`` when the KB (or the account settings) lacks
+    a complete credential pair, and ``fastapi.HTTPException`` when the reference
+    is not accessible to the current user — writes require write access, since
+    they modify the user's own IMA library.
+    """
+    from deeptutor.multi_user.knowledge_access import resolve_kb
+    from deeptutor.services.rag.pipelines.ima.client import ImaClient
+    from deeptutor.services.rag.pipelines.ima.config import resolve_kb_config
+    from deeptutor.services.rag.provider_binding import load_kb_config_entry
+
+    resource = resolve_kb(str(kb_ref), require_write=for_write)
+    entry = load_kb_config_entry(resource.base_dir, resource.name)
+    return ImaClient(resolve_kb_config(entry))
+
+
+def _resolve(context: UnifiedContext) -> tuple[ImaBinding, ...]:
+    from deeptutor.multi_user.knowledge_access import resolve_kb_metadata
+
+    bindings: list[ImaBinding] = []
+    seen: set[str] = set()
+    for raw in context.knowledge_bases or []:
+        ref = str(raw).strip()
+        if not ref or ref in seen:
+            continue
+        seen.add(ref)
+        meta = resolve_kb_metadata(ref)
+        if not meta or meta.get("type") != IMA_KB_TYPE:
+            continue
+        bindings.append(
+            ImaBinding(
+                kb_ref=ref,
+                name=str(meta.get("name") or ref),
+                knowledge_base_id=str(meta.get("knowledge_base_id") or ""),
+            )
+        )
+    return tuple(bindings)
+
+
+__all__ = ["ImaBinding", "ima_bindings", "resolve_client", "select_binding"]

deeptutor/capabilities/ima/capability.py

@@ -0,0 +1,92 @@
+"""IMA loop capability — inventory, source reading and authoring over a live library.
+
+Active whenever one of the turn's selected knowledge bases is a connected Tencent
+IMA library (resolved by :mod:`deeptutor.capabilities.ima.binding`).
+
+Unlike the Obsidian and subagent capabilities this is a *plain*
+:class:`~deeptutor.capabilities.protocol.LoopExtension`, not a
+:class:`~deeptutor.capabilities.protocol.KnowledgeCapability`: it **adds** its
+tools to chat's normal surface instead of replacing it. That difference is not
+stylistic — it follows from what the KB actually is. An Obsidian vault has no
+index at all, so ``rag`` is useless against it and the capability must own the
+turn. An IMA library is genuinely searchable over HTTP: the ``ima`` RAG provider
+answers ``rag`` calls for it. Retrieval therefore stays where it belongs, and
+these tools add only what retrieval cannot do — enumerate the library, read a
+whole source, search notes by recency, and write back.
+
+Keeping the surface additive also means a turn that attaches an IMA library
+alongside other knowledge bases (or that needs web search) does not lose those
+abilities, which an exclusive capability would strip.
+"""
+
+from __future__ import annotations
+
+from importlib import resources
+from typing import Any
+
+from deeptutor.capabilities.ima.binding import ima_bindings
+from deeptutor.capabilities.ima.tools import BINDINGS_KWARG, IMA_TOOL_NAMES
+from deeptutor.capabilities.protocol import PromptBlock
+from deeptutor.core.context import UnifiedContext
+from deeptutor.services.prompt.language import is_chinese as _is_zh
+from deeptutor.services.prompt.lookup import prompt_text as _prompt_text
+
+
+class ImaCapability:
+    """Turn-scoped integration for connected Tencent IMA knowledge bases."""
+
+    name = "ima"
+    owned_tools = IMA_TOOL_NAMES
+
+    def is_active(self, context: UnifiedContext) -> bool:
+        return bool(ima_bindings(context))
+
+    def system_block(
+        self,
+        context: UnifiedContext,
+        *,
+        language: str,
+        prompts: dict[str, Any],
+    ) -> PromptBlock | None:
+        bindings = ima_bindings(context)
+        if not bindings:
+            return None
+        override = _prompt_text(prompts, ("ima", "system"))
+        content = override or _load_system_prompt(language)
+        names = (
+            "、".join(binding.name for binding in bindings)
+            if _is_zh(language)
+            else ", ".join(binding.name for binding in bindings)
+        )
+        return PromptBlock("ima", content.replace("{kb_names}", names))
+
+    def augment_kwargs(
+        self,
+        tool_name: str,
+        kwargs: dict[str, Any],
+        context: UnifiedContext,
+    ) -> dict[str, Any]:
+        if tool_name not in IMA_TOOL_NAMES:
+            return kwargs
+        bindings = ima_bindings(context)
+        if not bindings:
+            return kwargs
+        # Server-owned: overwrite any model-supplied value so a turn can only
+        # reach the libraries the user actually attached. Credentials are not
+        # part of this payload — the tools load them per call.
+        updated = dict(kwargs)
+        updated[BINDINGS_KWARG] = bindings
+        return updated
+
+    def pre_loop_seed(self, context: UnifiedContext) -> str:
+        _ = context
+        return ""
+
+
+def _load_system_prompt(language: str) -> str:
+    lang = "zh" if _is_zh(language) else "en"
+    prompt = resources.files(__package__).joinpath("prompts", lang, "system.md")
+    return prompt.read_text(encoding="utf-8").strip()
+
+
+__all__ = ["ImaCapability"]

deeptutor/capabilities/ima/prompts/en/system.md

@@ -0,0 +1,33 @@
+# Tencent IMA Library
+
+The user has attached their Tencent IMA knowledge base **{kb_names}** to this
+turn. Its documents live in IMA, not on this machine: everything below is an API
+call against the user's own library. Your normal tools stay available — the IMA
+tools are additional.
+
+## Which tool answers which question
+
+- **"What does the material say about X?"** → `rag` with this knowledge base.
+  That is IMA's semantic search, and it stays the way to find content.
+- **"What's in here?" / "Is document X in it?" / "What's in that folder?"** →
+  `ima_list`. Retrieval only reports what a query happened to match, so it can
+  never establish what the library contains. Never infer the inventory from
+  retrieved passages, and never claim the document list cannot be read.
+- **"Give me the whole document" / a retrieved snippet is too thin** →
+  `ima_read` with the item's `media_id` (from `ima_list` or a citation).
+- **"My latest notes" / "what did I write about X?"** → `ima_note_search`.
+  Notes carry created/updated timestamps; knowledge-base documents do not, so
+  recency questions about documents can only be answered by the order `ima_list`
+  returns them in — say so rather than inventing dates.
+
+## Writing (only when asked)
+
+`ima_add_url` collects web pages into the library and `ima_write_note` saves a
+note. Both modify the user's own IMA account, so use them only when the user
+asked to save, collect, or record something — never as a side effect of
+answering. Appending to an existing note cannot be undone: only append to a note
+the user actually named, otherwise create a new one. Nothing you can call
+deletes or overwrites their material.
+
+Answer grounded in what you read, citing document or note titles. If the library
+does not cover something, say so instead of guessing.

deeptutor/capabilities/ima/prompts/zh/system.md

@@ -0,0 +1,24 @@
+# 腾讯 IMA 知识库
+
+用户已把腾讯 IMA 知识库 **{kb_names}** 挂载到本轮对话。它的文档存放在 IMA 而不在本机，
+下面每一项都是对用户自己那个库的接口调用。你原有的工具依然可用，IMA 工具是额外增加的。
+
+## 什么问题用什么工具
+
+- **「资料里怎么说 X」** → 对该知识库调用 `rag`。那就是 IMA 的语义检索，找内容仍然走它。
+- **「这里面有什么」「有没有某个文档」「那个文件夹里是什么」** → `ima_list`。
+  检索只能反映某次查询命中了什么，永远无法说明库里到底有哪些东西。
+  不要用检索到的片段去推断文档构成，也不要声称读不到文档清单。
+- **「把整篇给我」或检索片段太薄** → 用条目的 `media_id`（来自 `ima_list` 或引用）调 `ima_read`。
+- **「我最近的笔记」「我写过什么关于 X 的」** → `ima_note_search`。
+  笔记带创建/更新时间，知识库文档没有时间字段；因此关于文档「最新」的问题只能依据
+  `ima_list` 的返回顺序，如实说明，不要编造日期。
+
+## 写入（仅在用户要求时）
+
+`ima_add_url` 把网页收进知识库，`ima_write_note` 保存笔记。两者都会改动用户自己的 IMA 账号，
+所以只有在用户明确要求保存、收藏、记录时才使用，绝不作为回答的副作用。
+往已有笔记追加内容不可撤销：只追加到用户明确指名的那篇，否则新建一篇。
+你能调用的工具都不会删除或覆盖用户的资料。
+
+回答要基于你读到的内容，并标注文档或笔记标题。库里没有覆盖的内容，如实说明，不要猜。