memu-cli 0.1.0__cp313-abi3-win_amd64.whl

memu/__init__.py

@@ -0,0 +1,9 @@
+from memu._core import hello_from_bin
+from memu.app.service import MemoryService
+
+# Public alias used in documentation examples
+MemUService = MemoryService
+
+
+def _rust_entry() -> str:
+    return hello_from_bin()

memu/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m memu`` to invoke the CLI (used by the npm launcher)."""
+
+import sys
+
+from memu.cli import main
+
+sys.exit(main())

memu/_core.pyi

@@ -0,0 +1 @@
+def hello_from_bin() -> str: ...

memu/app/__init__.py

@@ -0,0 +1,37 @@
+from memu.app.service import MemoryService
+from memu.app.settings import (
+    BlobConfig,
+    DatabaseConfig,
+    DefaultUserModel,
+    EmbeddingConfig,
+    EmbeddingProfilesConfig,
+    LLMConfig,
+    LLMProfilesConfig,
+    MemorizeConfig,
+    RetrieveConfig,
+    UserConfig,
+)
+from memu.workflow.runner import (
+    LocalWorkflowRunner,
+    WorkflowRunner,
+    register_workflow_runner,
+    resolve_workflow_runner,
+)
+
+__all__ = [
+    "BlobConfig",
+    "DatabaseConfig",
+    "DefaultUserModel",
+    "EmbeddingConfig",
+    "EmbeddingProfilesConfig",
+    "LLMConfig",
+    "LLMProfilesConfig",
+    "LocalWorkflowRunner",
+    "MemorizeConfig",
+    "MemoryService",
+    "RetrieveConfig",
+    "UserConfig",
+    "WorkflowRunner",
+    "register_workflow_runner",
+    "resolve_workflow_runner",
+]

memu/app/agentic.py

@@ -0,0 +1,394 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel
+
+from memu.blob.folder import infer_modality
+from memu.vector import cosine_topk
+
+if TYPE_CHECKING:
+    from memu.app.service import Context
+    from memu.app.settings import RetrieveWorkspaceConfig
+    from memu.database.interfaces import Database
+    from memu.database.models import RecallFile, Resource
+
+
+class AgenticMixin:
+    if TYPE_CHECKING:
+        _get_context: Callable[[], Context]
+        _get_database: Callable[[], Database]
+        _normalize_where: Callable[[Mapping[str, Any] | None], dict[str, Any]]
+        _model_dump_without_embeddings: Callable[[BaseModel], dict[str, Any]]
+        _ensure_categories_ready: Callable[[Context, Database, Mapping[str, Any] | None], Awaitable[None]]
+        _get_embedding_client: Callable[..., Any]
+        retrieve_workspace_config: RetrieveWorkspaceConfig
+        user_model: type[BaseModel]
+
+    async def list_all_recall_files(
+        self,
+        where: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """List RecallFiles across every track without workflow orchestration.
+
+        Unlike :meth:`CRUDMixin.list_recall_files`, this does not force the
+        ``track="memory"`` filter (ADR 0006), so skill-track files are included,
+        and it queries the repository directly instead of running a workflow.
+        """
+        store = self._get_database()
+        where_filters = self._normalize_where(where)
+        categories = store.recall_file_repo.list_categories(where_filters)
+        categories_list = [self._model_dump_without_embeddings(category) for category in categories.values()]
+        return {"categories": categories_list}
+
+    async def progressive_retrieve(
+        self,
+        query: str,
+        where: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Single-shot, LLM-free retrieval over the segment/file/resource layers.
+
+        This mirrors :meth:`RetrieveWorkspaceMixin.retrieve_workspace` but runs no
+        workflow — the four stages are executed sequentially in-line, the same way
+        :meth:`commit_results` inverts ``memorize_workspace``. The query is embedded
+        once and used to rank two layers by vector similarity; no intention routing,
+        sufficiency checks, or summarization:
+
+        * ``segments``: :class:`RecallFileSegment` slices ranked by embedding,
+          ``file.top_k`` of them.
+        * ``files``: the :class:`RecallFile`\\ s pointed to by those segments — not
+          a ranked search, just a roll-up. Each file's score is the max score of
+          the segments that point to it.
+        * ``resources``: workspace-track resources ranked by embedding,
+          ``resource.top_k`` of them.
+
+        Returns ``segments``, ``files``, and ``resources``.
+        """
+        if not query or not query.strip():
+            raise ValueError("empty_query")
+        store = self._get_database()
+        where_filters = self._normalize_where(where)
+        config = self.retrieve_workspace_config
+        embed_client = self._get_embedding_client("embedding")
+        query_vector = (await embed_client.embed([query]))[0]
+
+        segment_hits, segment_pool = self._recall_segments(
+            store=store, where_filters=where_filters, query_vector=query_vector, enabled=config.file.enabled
+        )
+        file_hits, file_pool, file_resource_urls = self._collect_files(
+            store=store, where_filters=where_filters, segment_hits=segment_hits, segment_pool=segment_pool
+        )
+        resource_hits, resource_pool = self._recall_resources(
+            store=store, where_filters=where_filters, query_vector=query_vector, enabled=config.resource.enabled
+        )
+
+        files = self._materialize_hits(file_hits, file_pool)
+        for file in files:
+            file["resource_urls"] = file_resource_urls.get(file["id"], [])
+        return {
+            "segments": self._materialize_hits(segment_hits, segment_pool),
+            "files": files,
+            "resources": self._materialize_hits(resource_hits, resource_pool),
+        }
+
+    def _recall_segments(
+        self,
+        *,
+        store: Database,
+        where_filters: dict[str, Any],
+        query_vector: list[float],
+        enabled: bool,
+    ) -> tuple[list[tuple[str, float]], dict[str, Any]]:
+        """Rank :class:`RecallFileSegment` slices by embedding similarity.
+
+        Mirrors :meth:`RetrieveWorkspaceMixin._ws_recall_segments`. The segment repo
+        has no vector search, so the stored segment embeddings are ranked directly,
+        optionally scoped to the configured tracks via the denormalized ``track``.
+        """
+        if not enabled:
+            return [], {}
+
+        segment_where = dict(where_filters)
+        tracks = self.retrieve_workspace_config.file.tracks
+        if tracks:
+            segment_where["track__in"] = list(tracks)
+        segment_pool = {seg.id: seg for seg in store.recall_file_segment_repo.list_segments(segment_where)}
+        segment_hits = cosine_topk(
+            query_vector,
+            [(sid, seg.embedding) for sid, seg in segment_pool.items()],
+            k=self.retrieve_workspace_config.file.top_k,
+        )
+        return segment_hits, segment_pool
+
+    def _collect_files(
+        self,
+        *,
+        store: Database,
+        where_filters: dict[str, Any],
+        segment_hits: list[tuple[str, float]],
+        segment_pool: dict[str, Any],
+    ) -> tuple[list[tuple[str, float]], dict[str, Any], dict[str, list[str]]]:
+        """Roll the ranked segments up to their files (no ranked file search).
+
+        Mirrors :meth:`RetrieveWorkspaceMixin._ws_collect_files`. Every file pointed
+        to by a top segment is returned; a file's score is the max score across the
+        segments that point to it.
+        """
+        file_pool = store.recall_file_repo.list_categories(where_filters)
+
+        file_scores: dict[str, float] = {}
+        for seg_id, score in segment_hits:
+            seg = segment_pool.get(seg_id)
+            if seg is None:
+                continue
+            fid = seg.recall_file_id
+            if fid not in file_pool:
+                continue
+            score = float(score)
+            if fid not in file_scores or score > file_scores[fid]:
+                file_scores[fid] = score
+
+        # Preserve descending-score order so the response reads best-first.
+        file_hits = sorted(file_scores.items(), key=lambda kv: kv[1], reverse=True)
+        file_resource_urls = self._collect_file_resource_urls(store, where_filters, file_pool)
+        return file_hits, file_pool, file_resource_urls
+
+    @staticmethod
+    def _collect_file_resource_urls(
+        store: Database, where_filters: dict[str, Any], file_pool: dict[str, Any]
+    ) -> dict[str, list[str]]:
+        """Map each file id to the URLs of the resources linked to it.
+
+        Mirrors :meth:`RetrieveWorkspaceMixin._ws_collect_file_resource_urls`. Resolves
+        the ``RecallFileResource`` link table (file -> resource) and the resource
+        records (resource -> url) within the current scope, surfacing url strings only.
+        """
+        resources = store.resource_repo.list_resources(where_filters)
+        file_resource_urls: dict[str, list[str]] = {}
+        for rel in store.recall_file_resource_repo.list_relations(where_filters):
+            if rel.file_id not in file_pool:
+                continue
+            resource = resources.get(rel.resource_id)
+            if resource is None:
+                continue
+            file_resource_urls.setdefault(rel.file_id, []).append(resource.url)
+        return file_resource_urls
+
+    def _recall_resources(
+        self,
+        *,
+        store: Database,
+        where_filters: dict[str, Any],
+        query_vector: list[float],
+        enabled: bool,
+    ) -> tuple[list[tuple[str, float]], dict[str, Any]]:
+        """Rank workspace-track resources by embedding similarity.
+
+        Mirrors :meth:`RetrieveWorkspaceMixin._ws_recall_resources`. Only resources
+        ingested by ``memorize_workspace`` (``track="workspace"``) are surfaced; other
+        tracks are excluded.
+        """
+        if not enabled:
+            return [], {}
+
+        resource_where = {**where_filters, "track": "workspace"}
+        resource_pool = store.resource_repo.list_resources(resource_where)
+        resource_hits = store.resource_repo.vector_search_resources(
+            query_vector, self.retrieve_workspace_config.resource.top_k, where=resource_where
+        )
+        return resource_hits, resource_pool
+
+    def _materialize_hits(self, hits: Sequence[tuple[str, float]], pool: dict[str, Any]) -> list[dict[str, Any]]:
+        """Expand ``(id, score)`` hits into scored, embedding-free dicts.
+
+        Mirrors :meth:`RetrieveWorkspaceMixin._materialize_hits`.
+        """
+        out = []
+        for _id, score in hits:
+            obj = pool.get(_id)
+            if not obj:
+                continue
+            data = self._model_dump_without_embeddings(obj)
+            data["score"] = float(score)
+            out.append(data)
+        return out
+
+    async def commit_results(
+        self,
+        *,
+        recall_files: list[dict[str, Any]] | None = None,
+        resource: list[dict[str, Any]] | None = None,
+        user: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Persist externally-prepared resources and recall files into the store.
+
+        This mirrors the persistence half of :meth:`MemorizeWorkspaceMixin.memorize_workspace`
+        but takes items that were already preprocessed/synthesized off-service (see
+        :mod:`memu.hosts.bridging.pipeline`), so it runs no ingest/preprocess/LLM steps and
+        no workflow — just create-or-update straight into storage:
+
+        - ``resource`` — a list of ``{path, description}`` records. Each is a
+          :class:`Resource` keyed by ``url`` (``= path``); the description becomes the
+          embedded caption used for INDEX/resource recall.
+        - ``recall_files`` — a list of ``{name, track, description, content}`` records. Each
+          is a :class:`RecallFile` keyed by ``name`` within its ``track`` (``memory``/``skill``),
+          with the same track-specific segment (re)generation as the workspace path.
+        """
+        store = self._get_database()
+        user_scope = self.user_model(**user).model_dump() if user is not None else None
+        embed_client = self._get_embedding_client("embedding")
+
+        committed_resources = await self._commit_resources(
+            resource or [], store=store, user_scope=user_scope, embed_client=embed_client
+        )
+        committed_files = await self._commit_recall_files(
+            recall_files or [], store=store, user_scope=user_scope, embed_client=embed_client
+        )
+        return {
+            "resources": [self._model_dump_without_embeddings(r) for r in committed_resources],
+            "recall_files": [self._model_dump_without_embeddings(f) for f in committed_files],
+        }
+
+    async def _commit_resources(
+        self,
+        resources: list[dict[str, Any]],
+        *,
+        store: Database,
+        user_scope: dict[str, Any] | None,
+        embed_client: Any,
+    ) -> list[Resource]:
+        """Create-or-update each ``{path, description}`` as a ``Resource`` keyed by url.
+
+        ``ResourceRepo`` has no in-place update, so an "update" is a delete-then-create:
+        any existing resource sharing this url (and its file provenance links) is dropped
+        before the fresh record is created. Mirrors
+        :meth:`MemorizeWorkspaceMixin._create_resource_with_caption`.
+        """
+        where = user_scope or None
+        committed: list[Resource] = []
+        for item in resources:
+            url = (item.get("path") or "").strip()
+            if not url:
+                continue
+            caption = (item.get("description") or "").strip() or None
+
+            # Create-or-update keyed by url: drop any prior resource for this url first.
+            stale = [res for res in store.resource_repo.list_resources(where=where).values() if res.url == url]
+            for res in stale:
+                store.recall_file_resource_repo.unlink_resource(res.id)
+                store.resource_repo.delete_resource(res.id)
+
+            caption_embedding = (await embed_client.embed([caption]))[0] if caption else None
+            res = store.resource_repo.create_resource(
+                url=url,
+                modality=infer_modality(url) or "document",
+                local_path=url,
+                caption=caption,
+                embedding=caption_embedding,
+                user_data=dict(user_scope or {}),
+            )
+            committed.append(res)
+        return committed
+
+    async def _commit_recall_files(
+        self,
+        recall_files: list[dict[str, Any]],
+        *,
+        store: Database,
+        user_scope: dict[str, Any] | None,
+        embed_client: Any,
+    ) -> list[RecallFile]:
+        """Create-or-update each ``{name, track, description, content}`` as a ``RecallFile``.
+
+        Keyed by ``name`` within the record's ``track`` (``memory``/``skill``). New files embed
+        their ``name: description`` for file-level recall; existing files keep their embedding
+        and only take the new content. Segments are then reconciled per track. Mirrors the
+        persist path of :meth:`MemorizeWorkspaceMixin._synthesize_file_ops`.
+        """
+        user_data = dict(user_scope or {})
+        committed: list[RecallFile] = []
+        for item in recall_files:
+            name = (item.get("name") or "").strip()
+            if not name:
+                continue
+            file_track = item.get("track") or "memory"
+            description = (item.get("description") or "").strip()
+            content = (item.get("content") or "").strip()
+
+            existing = store.recall_file_repo.list_categories(where={**user_data, "track": file_track})
+            file = {f.name: f for f in existing.values()}.get(name)
+            if file is None:
+                emb_text = f"{name}: {description}" if description else name
+                embedding = (await embed_client.embed([emb_text]))[0]
+                file = store.recall_file_repo.get_or_create_category(
+                    name=name,
+                    description=description,
+                    embedding=embedding,
+                    user_data=user_data,
+                    track=file_track,
+                )
+            file = store.recall_file_repo.update_category(category_id=file.id, content=content)
+            await self._commit_sync_file_segments(
+                file=file,
+                file_track=file_track,
+                store=store,
+                user_scope=user_data,
+                embed_client=embed_client,
+            )
+            committed.append(file)
+        return committed
+
+    @staticmethod
+    def _commit_segment_texts_for_file(file: RecallFile, file_track: str) -> list[str]:
+        """Compute a file's searchable segment texts (ADR 0007 L2 items), track-specific.
+
+        Mirrors :meth:`MemorizeWorkspaceMixin._segment_texts_for_file`:
+
+        - ``skill``: a single ``name: ...\\ndescription: ...`` segment for the whole skill.
+        - ``memory``: one segment per content line, skipping blank lines and markdown
+          headings, de-duplicated while preserving order.
+        """
+        if file_track == "skill":
+            return [f"name: {file.name}\ndescription: {file.description}"]
+
+        texts: list[str] = []
+        for line in (file.content or "").split("\n"):
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            texts.append(stripped)
+        return list(dict.fromkeys(texts))
+
+    async def _commit_sync_file_segments(
+        self,
+        *,
+        file: RecallFile,
+        file_track: str,
+        store: Database,
+        user_scope: dict[str, Any],
+        embed_client: Any,
+    ) -> None:
+        """Reconcile a file's stored segments with its freshly computed segment texts.
+
+        Drop-and-add on the difference only: segments whose text disappeared are deleted and
+        only genuinely new texts are embedded and inserted, so unchanged lines keep their
+        embedding. Mirrors :meth:`MemorizeWorkspaceMixin._sync_file_segments` (single file).
+        """
+        new_texts = self._commit_segment_texts_for_file(file, file_track)
+        existing = store.recall_file_segment_repo.list_segments_for_file(file.id)
+        existing_texts = {seg.text for seg in existing}
+        new_set = set(new_texts)
+
+        for seg in existing:
+            if seg.text not in new_set:
+                store.recall_file_segment_repo.delete_segment(seg.id)
+
+        to_add = [text for text in new_texts if text not in existing_texts]
+        if not to_add:
+            return
+        vecs = await embed_client.embed(to_add)
+        for text, vec in zip(to_add, vecs, strict=True):
+            store.recall_file_segment_repo.create_segment(
+                recall_file_id=file.id, track=file_track, text=text, embedding=vec, user_data=dict(user_scope)
+            )

memu/app/client_pool.py

@@ -0,0 +1,63 @@
+"""Generic lazy client pool keyed by profile name.
+
+``MemoryService`` needs three structurally identical caches: one each for the
+chat (LLM), vision (VLM) and embedding clients. Each maps a profile name to a
+lazily-built, cached client. This pool factors out that duplicated bookkeeping
+so adding a fourth capability (e.g. rerank/STT) is a single instantiation rather
+than another copy of the get/cache/build dance.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+
+
+class ClientPool[TConfig, TClient]:
+    """Lazily build and cache one client per named profile.
+
+    Args:
+        profiles: Mapping of profile name -> config object.
+        builder: Factory turning a config into a concrete client.
+        label: Human-readable capability name used in error messages
+            (e.g. ``"llm"``, ``"vlm"``, ``"embedding"``).
+        default_profile: Profile name used when ``get()`` is called with ``None``.
+    """
+
+    def __init__(
+        self,
+        *,
+        profiles: Mapping[str, TConfig],
+        builder: Callable[[TConfig], TClient],
+        label: str,
+        default_profile: str = "default",
+    ) -> None:
+        self._profiles = profiles
+        self._builder = builder
+        self._label = label
+        self._default_profile = default_profile
+        self._cache: dict[str, TClient] = {}
+
+    def config(self, profile: str | None = None) -> TConfig | None:
+        """Return the config for ``profile`` (or the default), if present."""
+        return self._profiles.get(profile or self._default_profile)
+
+    def get(self, profile: str | None = None) -> TClient:
+        """Return the cached client for ``profile``, building it on first use.
+
+        Raises:
+            KeyError: if no profile with that name is configured.
+        """
+        name = profile or self._default_profile
+        cached = self._cache.get(name)
+        if cached is not None:
+            return cached
+        cfg = self._profiles.get(name)
+        if cfg is None:
+            msg = f"Unknown {self._label} profile '{name}'"
+            raise KeyError(msg)
+        client = self._builder(cfg)
+        self._cache[name] = client
+        return client
+
+
+__all__ = ["ClientPool"]