memorius 0.1.0__py3-none-any.whl

memorius/__init__.py

@@ -0,0 +1,29 @@
+"""memorius — self-contained memory vault for any AI agent.
+
+Architecture:
+
+  Storage Layer:
+    chroma_db:     Vector store for semantic search (ChromaDB)
+    sqlite_store:  Metadata store for vaults, shelves, folders, notes, and diaries (SQLite)
+  Hierarchy:       Vault > Shelf > Folder > Note
+
+  Vault Layer:
+    vault:    Vault > Shelf > Folder > Note hierarchy
+    diary:    Session diary entries with timestamps
+    mine:     Extract memories from conversations/transcripts
+    search:   Multi-modal search (vector + metadata + temporal)
+
+  Server Layer:
+    mcp:      MCP protocol server (primary interface for tool-calling agents)
+    rest:     FastAPI REST server (alternative interface for web/curl)
+
+  Integration Layer:
+    hooks:       Agent-agnostic hook lifecycle system
+    plugin_gen:  Per-agent plugin manifest generator
+    normalizers: Conversation format importers (Discord, Telegram, WhatsApp)
+
+  CLI Layer:
+    memorius:  Main CLI — init, mine, search, diary, status, serve, hook
+"""
+
+__version__ = "0.1.0"

memorius/cli/main.py

@@ -0,0 +1,306 @@
+"""memorius CLI — interact with your memory vault from the terminal.
+
+Usage:
+  memorius init              Initialize a new vault
+  memorius status            Show vault status
+  memorius store <text>      Store a memory
+  memorius search <query>    Semantic search across memories
+  memorius mine <file>       Mine memories from a transcript file
+  memorius diary <session>   Write a diary entry (interactive)
+  memorius diaries           List recent diary entries
+  memorius ls [vault]        Explore vault hierarchy
+  memorius serve             Start the MCP server (stdio)
+  memorius serve-rest        Start the REST API server
+  memorius config            Show current config
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+from memorius import __version__
+from memorius.config import load_config, DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_PATH
+from memorius.vault import VaultEngine
+
+logging.basicConfig(
+    level=logging.WARNING,
+    format="%(levelname)s [%(name)s] %(message)s",
+)
+logger = logging.getLogger("memorius.cli")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        "memorius",
+        description="Memory vault for any AI agent — store, search, and organize memories.",
+    )
+    parser.add_argument("--version", action="store_true", help="Show version")
+    parser.add_argument("--config", default=None, help="Path to config file")
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    subparsers.add_parser("init", help="Initialize a new vault")
+    subparsers.add_parser("status", help="Show vault status")
+
+    store_p = subparsers.add_parser("store", help="Store a memory")
+    store_p.add_argument("content", nargs="?", default=None, help="Memory content")
+    store_p.add_argument("--vault", default="main", help="Vault name")
+    store_p.add_argument("--shelf", default="default", help="Shelf name")
+    store_p.add_argument("--folder", default="default", help="Folder name")
+    store_p.add_argument("--note", default="default", help="Note name")
+
+    search_p = subparsers.add_parser("search", help="Semantic search")
+    search_p.add_argument("query", nargs="?", default=None, help="Search query")
+    search_p.add_argument("--n", type=int, default=10, help="Number of results")
+    search_p.add_argument("--vault", default=None, help="Filter by vault")
+    search_p.add_argument("--shelf", default=None, help="Filter by shelf")
+
+    mine_p = subparsers.add_parser("mine", help="Mine memories from a transcript")
+    mine_p.add_argument("file", nargs="?", default=None, help="Transcript file path")
+    mine_p.add_argument("--vault", default="main", help="Vault name")
+    mine_p.add_argument("--text", default=None, help="Transcript text (inline)")
+
+    diary_p = subparsers.add_parser("diary", help="Write a diary entry")
+    diary_p.add_argument("session_id", nargs="?", default=None, help="Session ID")
+    diary_p.add_argument("--title", default="", help="Diary title")
+    diary_p.add_argument("--summary", default="", help="Diary summary")
+    diary_p.add_argument("--content", default="", help="Diary content")
+    diary_p.add_argument("--vault", default="main", help="Vault name")
+
+    subparsers.add_parser("diaries", help="List recent diary entries")
+    p = subparsers.add_parser("ls", help="Explore vault hierarchy")
+    p.add_argument("--vault", default=None, help="Vault to explore (default: all)")
+
+    serve_p = subparsers.add_parser("serve", help="Start MCP server (stdio)")
+    serve_p.add_argument("--port", type=int, default=8911, help="Not used for stdio")
+
+    serve_rest_p = subparsers.add_parser("serve-rest", help="Start REST API server")
+    serve_rest_p.add_argument("--port", type=int, default=8912, help="Port")
+    serve_rest_p.add_argument("--host", default="127.0.0.1", help="Host")
+
+    config_p = subparsers.add_parser("config", help="Show configuration")
+    config_p.add_argument("--show", action="store_true", default=True, help="Show config")
+    config_p.add_argument("--path", action="store_true", help="Show config file path")
+
+    args = parser.parse_args()
+
+    if args.version:
+        print(f"memorius v{__version__}")
+        return
+
+    if args.debug:
+        logging.getLogger("memorius").setLevel(logging.DEBUG)
+
+    if args.command is None:
+        parser.print_help()
+        return
+
+    # Load config and create engine
+    config = load_config(args.config)
+    engine = VaultEngine(config)
+
+    # Dispatch
+    commands = {
+        "init": cmd_init,
+        "status": cmd_status,
+        "store": cmd_store,
+        "search": cmd_search,
+        "mine": cmd_mine,
+        "diary": cmd_diary,
+        "diaries": cmd_diaries,
+        "ls": cmd_ls,
+        "serve": cmd_serve,
+        "serve-rest": cmd_serve_rest,
+        "config": cmd_config,
+    }
+    handler = commands.get(args.command)
+    if handler:
+        handler(engine, args, config)
+    else:
+        print(f"Unknown command: {args.command}")
+
+
+def cmd_init(engine, args, config):
+    """Initialize the vault — ensures config dir and storage exists."""
+    DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    if not DEFAULT_CONFIG_PATH.exists():
+        from memorius.config import DEFAULT_CONFIG_YAML
+        DEFAULT_CONFIG_PATH.write_text(DEFAULT_CONFIG_YAML)
+        print(f"Created config: {DEFAULT_CONFIG_PATH}")
+    else:
+        print(f"Config exists: {DEFAULT_CONFIG_PATH}")
+
+    engine._meta.ensure_vault("main", "Main vault")
+    print("Vault initialized: main")
+    print(f"Storage: {config.get('storage', {}).get('path', '~/.memorius/data')}")
+    print(f"Embeddings: {config.get('embeddings', {}).get('provider', 'chroma-default')}")
+
+
+def cmd_status(engine, args, config):
+    """Show vault status."""
+    status = engine.status()
+    print(f"  Vaults:       {status['vaults']}")
+    print(f"  Memories:     {status['memories']}")
+    print(f"  Embeddings:   {status['embedding_provider']} (dim={status['embedding_dimension']})")
+    print()
+    diaries = engine._meta.list_diaries(limit=5)
+    if diaries:
+        print("Recent diaries:")
+        for d in diaries:
+            print(f"  [{d['created_at'][:19]}] {d.get('title', 'untitled')} ({d['session_id']})")
+    else:
+        print("No diaries yet.")
+
+
+def cmd_store(engine, args, config):
+    """Store a memory. Reads from stdin if no content arg."""
+    content = args.content
+    if not content:
+        content = sys.stdin.read().strip()
+    if not content:
+        print("Error: content required (pass as argument or pipe to stdin)")
+        return
+
+    memory = engine.store(
+        content=content,
+        vault=args.vault,
+        shelf=args.shelf,
+        folder=args.folder,
+        note=args.note,
+    )
+    print(f"Stored: {memory.id}")
+    print(f"  Path: {memory.vault}/{memory.shelf}/{memory.folder}/{memory.note}")
+
+
+def cmd_search(engine, args, config):
+    """Semantic search across memories."""
+    query = args.query
+    if not query:
+        query = sys.stdin.read().strip()
+    if not query:
+        print("Error: query required (pass as argument or pipe to stdin)")
+        return
+
+    results = engine.search(
+        query=query,
+        vault=args.vault,
+        shelf=args.shelf,
+        limit=args.n,
+    )
+    print(f'Search: "{query}"')
+    print(f"Results: {len(results)}")
+    print()
+    for i, m in enumerate(results, 1):
+        print(f"{i}. [{m.vault}/{m.shelf}/{m.folder}/{m.note}]")
+        print(f"   {m.content[:200]}")
+        if len(m.content) > 200:
+            print("   ...")
+        print()
+
+
+def cmd_mine(engine, args, config):
+    """Mine memories from a transcript."""
+    text = args.text
+    if not text and args.file:
+        text = Path(args.file).read_text()
+    if not text:
+        text = sys.stdin.read().strip()
+    if not text:
+        print("Error: transcript required (--text, <file>, or stdin)")
+        return
+
+    memories = engine.mine(
+        text=text,
+        vault=args.vault,
+    )
+    print(f"Mined {len(memories)} memories into {args.vault}/conversations/mined/transcript")
+
+
+def cmd_diary(engine, args, config):
+    """Write a diary entry."""
+    session_id = args.session_id or input("Session ID: ").strip()
+    if not session_id:
+        print("Error: session_id required")
+        return
+    title = args.title or input("Title: ").strip()
+    summary = args.summary or input("Summary: ").strip()
+
+    entry = engine.write_diary(
+        session_id=session_id,
+        vault=args.vault,
+        title=title,
+        summary=summary,
+        content=args.content,
+    )
+    print(f"Diary written: {entry['id']}")
+
+
+def cmd_diaries(engine, args, config):
+    """List recent diary entries."""
+    diaries = engine._meta.list_diaries(limit=10)
+    if not diaries:
+        print("No diary entries.")
+        return
+    for d in diaries:
+        print(f"[{d['created_at'][:19]}] {d['title'] or 'untitled'}")
+        print(f"  Session: {d['session_id']} | Vault: {d['vault']}")
+        if d['summary']:
+            print(f"  Summary: {d['summary'][:200]}")
+        print()
+
+
+def cmd_ls(engine, args, config):
+    """Explore vault hierarchy."""
+    vaults = engine._meta.list_vaults()
+    if not vaults:
+        print("No vaults. Run: memorius init")
+        return
+    for v in vaults:
+        print(f"{v['name']}/")
+        shelves = engine._meta.list_shelves(v['name'])
+        for sh in shelves:
+            print(f"  {sh['name']}/")
+            folders = engine._meta.list_folders(v['name'], sh['name'])
+            for f in folders[:5]:
+                print(f"    {f['name']}/")
+                notes = engine._meta.list_notes(v['name'], sh['name'], f['name'])
+                for n in notes:
+                    print(f"      {n['name']} ({n['memory_count']} memories)")
+            if len(folders) > 5:
+                print(f"    ... and {len(folders) - 5} more folders")
+
+
+def cmd_serve(engine, args, config):
+    """Start the MCP server (stdio)."""
+    from memorius.mcp_server import McpServer
+    server = McpServer(engine)
+    print("memorius MCP server starting (stdio)...", file=sys.stderr)
+    server.run()
+
+
+def cmd_serve_rest(engine, args, config):
+    """Start the REST API server."""
+    from memorius.rest_server import run_rest_server
+    config_server = config.get("server", {})
+    host = args.host or config_server.get("host", "127.0.0.1")
+    port = args.port or config_server.get("rest_port", 8912)
+    run_rest_server(engine, host=host, port=port)
+
+
+def cmd_config(engine, args, config):
+    """Show current configuration."""
+    if args.path:
+        path = DEFAULT_CONFIG_PATH
+        print(path)
+        print(f"  Exists: {path.exists()}")
+        return
+    print(json.dumps(config, indent=2, default=str))
+
+
+if __name__ == "__main__":
+    main()

memorius/config.py

@@ -0,0 +1,133 @@
+"""Configuration loading for memorius — YAML config with env var overrides."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+
+DEFAULT_CONFIG_DIR = Path.home() / ".memorius"
+DEFAULT_CONFIG_PATH = DEFAULT_CONFIG_DIR / "config.yaml"
+
+DEFAULT_CONFIG_YAML = """
+# memorius configuration
+# Path: ~/.memorius/config.yaml
+
+storage:
+  type: chroma  # chroma | sqlite (chroma is always primary for vectors)
+  path: ~/.memorius/data
+
+embeddings:
+  provider: chroma-default  # chroma-default | sentence-transformers | openai
+  model: all-MiniLM-L6-v2
+  # openai:
+  #   api_key: ...
+  #   model: text-embedding-3-small
+
+server:
+  mcp_port: 8911
+  rest_port: 8912
+  host: "127.0.0.1"
+
+vault:
+  default: "main"
+  max_note_size: 1000
+
+hooks:
+  enabled: true
+  config: ~/.memorius/hooks.yaml
+"""
+
+
+def load_config(path: str | Path | None = None) -> dict[str, Any]:
+    """Load config from YAML file, merging with env var overrides.
+
+    Env var overrides (prefixed with MEMORIUS_):
+      MEMORIUS_STORAGE_PATH     → storage.path
+      MEMORIUS_EMBEDDINGS_PROVIDER → embeddings.provider
+      MEMORIUS_OPENAI_API_KEY   → embeddings.openai.api_key
+      MEMORIUS_MCP_PORT         → server.mcp_port
+      MEMORIUS_REST_PORT        → server.rest_port
+      MEMORIUS_HOST             → server.host
+    """
+    import yaml
+
+    config_path = Path(path or _find_config()).expanduser()
+
+    if config_path.exists():
+        with open(config_path) as f:
+            config = yaml.safe_load(f) or {}
+    else:
+        config = _default_config()
+
+    _apply_env_overrides(config)
+    _ensure_defaults(config)
+    return config
+
+
+def _find_config() -> Path:
+    """Find config file: cwd ./memorius.yaml, then ~/.memorius/config.yaml."""
+    cwd_config = Path.cwd() / "memorius.yaml"
+    if cwd_config.exists():
+        return cwd_config
+    return DEFAULT_CONFIG_PATH
+
+
+def _default_config() -> dict[str, Any]:
+    import yaml
+    return yaml.safe_load(DEFAULT_CONFIG_YAML)
+
+
+def _ensure_defaults(config: dict[str, Any]):
+    """Fill in missing keys with defaults."""
+    storage = config.setdefault("storage", {})
+    storage.setdefault("type", "chroma")
+    storage.setdefault("path", "~/.memorius/data")
+
+    embeddings = config.setdefault("embeddings", {})
+    embeddings.setdefault("provider", "chroma-default")
+    embeddings.setdefault("model", "all-MiniLM-L6-v2")
+
+    server = config.setdefault("server", {})
+    server.setdefault("mcp_port", 8911)
+    server.setdefault("rest_port", 8912)
+    server.setdefault("host", "127.0.0.1")
+
+    vault_cfg = config.setdefault("vault", {})
+    vault_cfg.setdefault("default", "main")
+    vault_cfg.setdefault("max_note_size", 1000)
+
+    hooks = config.setdefault("hooks", {})
+    hooks.setdefault("enabled", True)
+    hooks.setdefault("config", "~/.memorius/hooks.yaml")
+
+
+def _apply_env_overrides(config: dict[str, Any]):
+    """Override config values from environment variables."""
+    overrides = {
+        "MEMORIUS_STORAGE_PATH": ("storage", "path"),
+        "MEMORIUS_EMBEDDINGS_PROVIDER": ("embeddings", "provider"),
+        "MEMORIUS_MCP_PORT": ("server", "mcp_port"),
+        "MEMORIUS_REST_PORT": ("server", "rest_port"),
+        "MEMORIUS_HOST": ("server", "host"),
+        "MEMORIUS_DEFAULT_VAULT": ("vault", "default"),
+    }
+    for env_key, (section, key) in overrides.items():
+        val = os.environ.get(env_key)
+        if val is not None:
+            config.setdefault(section, {})[key] = _coerce(val)
+
+    # Special handling for OpenAI key
+    openai_key = os.environ.get("MEMORIUS_OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY")
+    if openai_key:
+        emb = config.setdefault("embeddings", {})
+        emb.setdefault("openai", {})["api_key"] = openai_key
+
+
+def _coerce(val: str) -> int | str:
+    """Coerce string to int if possible, else return as-is."""
+    try:
+        return int(val)
+    except ValueError:
+        return val

memorius/embeddings.py

@@ -0,0 +1,157 @@
+"""Embedding providers for memorius — abstracted vector embedding interface."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, ClassVar
+
+
+class EmbeddingProvider(ABC):
+    """Abstract embedding provider. Returns normalized float vectors."""
+
+    @abstractmethod
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed a list of texts into vectors."""
+        ...
+
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        """Return the embedding vector dimension."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def from_config(cls, config: dict[str, Any]) -> "EmbeddingProvider":
+        """Create provider from config dict."""
+        ...
+
+
+class SentenceTransformerProvider(EmbeddingProvider):
+    """Local sentence-transformers embeddings (offline, no API key needed)."""
+
+    _model_key: ClassVar[str] = "all-MiniLM-L6-v2"
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        self._model_name = model_name
+        self._model = None  # lazy load
+        self._dim = 384  # all-MiniLM-L6-v2 default
+
+    def _lazy_load(self):
+        if self._model is not None:
+            return
+        try:
+            from sentence_transformers import SentenceTransformer
+            self._model = SentenceTransformer(self._model_name)
+            self._dim = self._model.get_sentence_embedding_dimension()
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers not installed. "
+                "Install: pip install memorius[local-embeddings]"
+            )
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        self._lazy_load()
+        embeddings = self._model.encode(texts, normalize_embeddings=True)
+        return [emb.tolist() for emb in embeddings]
+
+    @property
+    def dimension(self) -> int:
+        self._lazy_load()
+        return self._dim
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "SentenceTransformerProvider":
+        model = config.get("model", cls._model_key)
+        return cls(model_name=model)
+
+
+class OpenAIEmbeddingProvider(EmbeddingProvider):
+    """OpenAI API-based embeddings."""
+
+    def __init__(self, api_key: str, model: str = "text-embedding-3-small"):
+        self._api_key = api_key
+        self._model = model
+        self._dim = 1536 if "3-small" in model else 3072
+        self._client = None
+
+    def _lazy_load(self):
+        if self._client is not None:
+            return
+        try:
+            from openai import OpenAI
+            self._client = OpenAI(api_key=self._api_key)
+        except ImportError:
+            raise ImportError(
+                "openai package not installed. "
+                "Install: pip install memorius[openai]"
+            )
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        self._lazy_load()
+        resp = self._client.embeddings.create(input=texts, model=self._model)
+        # Sort by index to preserve order
+        sorted_data = sorted(resp.data, key=lambda d: d.index)
+        return [d.embedding for d in sorted_data]
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "OpenAIEmbeddingProvider":
+        openai_cfg = config.get("openai", {})
+        api_key = openai_cfg.get("api_key", config.get("api_key", ""))
+        model = openai_cfg.get("model", config.get("model", "text-embedding-3-small"))
+        if not api_key:
+            raise ValueError("OpenAI API key required. Set MEMORIUS_OPENAI_API_KEY or OPENAI_API_KEY")
+        return cls(api_key=api_key, model=model)
+
+
+class ChromaDefaultProvider(EmbeddingProvider):
+    """ChromaDB's built-in embedding function (ONNX all-MiniLM-L6-v2).
+    
+    No extra dependencies — ChromaDB ships its own ONNX runtime.
+    """
+    def __init__(self):
+        import chromadb.utils.embedding_functions as ef
+        self._fn = ef.DefaultEmbeddingFunction()
+        self._dim = 384
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        result = self._fn(texts)
+        return result
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "ChromaDefaultProvider":
+        return cls()
+
+
+class EmbeddingFactory:
+    """Create embedding providers from config."""
+
+    _registry: dict[str, type[EmbeddingProvider]] = {
+        "chroma-default": ChromaDefaultProvider,
+        "sentence-transformers": SentenceTransformerProvider,
+        "openai": OpenAIEmbeddingProvider,
+    }
+
+    @classmethod
+    def create(cls, config: dict[str, Any]) -> EmbeddingProvider:
+        provider_name = config.get("provider", "sentence-transformers")
+        provider_cls = cls._registry.get(provider_name)
+        if provider_cls is None:
+            raise ValueError(
+                f"Unknown embedding provider: {provider_name}. "
+                f"Available: {list(cls._registry.keys())}"
+            )
+        return provider_cls.from_config(config)
+
+    @classmethod
+    def register(cls, name: str, provider_cls: type[EmbeddingProvider]):
+        """Register a custom embedding provider."""
+        cls._registry[name] = provider_cls