cli-memory-os 0.1.0__py3-none-any.whl

infrastructure/health.py

@@ -0,0 +1,338 @@
+"""
+Infrastructure: Service health checks.
+
+Individual check functions for Neo4j, Qdrant, SQLite, Groq, Composio,
+embedding model, and connectors. Used by the ``doctor`` command.
+"""
+
+import os
+import sys
+import logging
+
+logger = logging.getLogger("infrastructure.health")
+
+
+def check_python() -> tuple[bool, str]:
+    """Check Python version."""
+    version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    ok = sys.version_info >= (3, 12)
+    return ok, version
+
+
+def check_neo4j() -> tuple[bool, str]:
+    """Attempt a Neo4j bolt connection."""
+    try:
+        from neo4j import GraphDatabase
+        uri = os.getenv("NEO4J_URI", "bolt://localhost:7687")
+        user = os.getenv("NEO4J_USER", "neo4j")
+        password = os.getenv("NEO4J_PASSWORD", "password")
+        driver = GraphDatabase.driver(uri, auth=(user, password))
+        driver.verify_connectivity()
+        driver.close()
+        return True, "Healthy"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_qdrant() -> tuple[bool, str]:
+    """Ping the Qdrant HTTP health endpoint."""
+    try:
+        import urllib.request
+        port = os.getenv("QDRANT_PORT", "6333")
+        url = f"http://localhost:{port}/healthz"
+        req = urllib.request.urlopen(url, timeout=5)
+        if req.status == 200:
+            return True, "Healthy"
+        return False, f"HTTP {req.status}"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_sqlite(db_path: str | None = None) -> tuple[bool, str]:
+    """Verify a SQLite database file is valid."""
+    import sqlite3
+    if db_path is None:
+        db_path = os.getenv("MEMORY_OS_DB_PATH", "memory.db")
+    if not os.path.exists(db_path):
+        return False, f"File not found: {db_path}"
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("PRAGMA integrity_check")
+        result = cursor.fetchone()
+        conn.close()
+        if result and result[0] == "ok":
+            return True, "Healthy"
+        return False, f"Integrity check: {result}"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_groq_api() -> tuple[bool, str]:
+    """Make a minimal Groq API test call."""
+    try:
+        from groq import Groq
+        key = os.getenv("GROQ_API_KEY")
+        if not key:
+            return False, "No API key"
+        client = Groq(api_key=key)
+        # List models as a lightweight connectivity test
+        client.models.list()
+        return True, "Connected"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_composio() -> tuple[bool, str]:
+    """Verify Composio toolkit status."""
+    try:
+        from composio import Composio
+        c = Composio()
+        s = c.create(user_id=os.getenv("COMPOSIO_USER_ID", "user_123"))
+        toolkits = s.toolkits()
+        count = len(toolkits.items) if toolkits and toolkits.items else 0
+        return True, f"Connected ({count} toolkits)"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_embedding_model() -> tuple[bool, str]:
+    """Check if the sentence-transformers model is cached locally."""
+    try:
+        from sentence_transformers import SentenceTransformer
+        model_name = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+        # Try loading from cache only
+        SentenceTransformer(model_name, local_files_only=True)
+        return True, model_name
+    except Exception:
+        model_name = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+        return False, f"{model_name} (not cached)"
+
+
+def check_connector(name: str) -> tuple[bool, str]:
+    """Check if a Composio connector (github, gmail, notion) is active."""
+    try:
+        from composio import Composio
+        c = Composio()
+        s = c.create(user_id=os.getenv("COMPOSIO_USER_ID", "user_123"))
+        toolkits = s.toolkits()
+        tk = next((t for t in toolkits.items if t.slug == name), None)
+        if tk and tk.connection and tk.connection.is_active:
+            return True, "Connected"
+        return False, "Not connected"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_workspace() -> tuple[bool, str]:
+    """Check if the workspace directory exists and is writable."""
+    from infrastructure.workspace import get_workspace_root
+    root = get_workspace_root()
+    if not root.exists():
+        return False, "Not initialized"
+    # Check writable
+    test_file = root / ".write_test"
+    try:
+        test_file.write_text("test", encoding="utf-8")
+        test_file.unlink()
+        return True, str(root)
+    except Exception as e:
+        return False, f"Not writable: {e}"
+
+
+def check_disk_usage() -> tuple[bool, str]:
+    """Calculate workspace disk usage."""
+    from infrastructure.workspace import get_workspace_root, get_workspace_size
+    root = get_workspace_root()
+    if not root.exists():
+        return False, "Workspace not found"
+    size_bytes = get_workspace_size()
+    if size_bytes < 1024 * 1024:
+        size_str = f"{size_bytes / 1024:.0f} KB"
+    elif size_bytes < 1024 * 1024 * 1024:
+        size_str = f"{size_bytes / (1024 * 1024):.0f} MB"
+    else:
+        size_str = f"{size_bytes / (1024 * 1024 * 1024):.1f} GB"
+    return True, size_str
+
+
+def check_memory_usage() -> tuple[bool, str]:
+    """Report current process memory usage."""
+    try:
+        import psutil
+        process = psutil.Process(os.getpid())
+        mem = process.memory_info().rss
+        if mem < 1024 * 1024:
+            return True, f"{mem / 1024:.0f} KB"
+        return True, f"{mem / (1024 * 1024):.0f} MB"
+    except ImportError:
+        # psutil not available, use a rough estimate
+        return True, "N/A (psutil not installed)"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_config_validation() -> tuple[bool, str]:
+    """Verify configuration validation."""
+    from infrastructure.config import get_config, DEFAULTS
+    try:
+        config = get_config()
+        for key in DEFAULTS:
+            if key not in config:
+                return False, f"Missing section '{key}'"
+            if isinstance(DEFAULTS[key], dict):
+                if not isinstance(config[key], dict):
+                    return False, f"Section '{key}' must be a dictionary"
+                for subkey in DEFAULTS[key]:
+                    if subkey not in config[key]:
+                        return False, f"Missing subkey '{key}.{subkey}'"
+        return True, "Valid"
+    except Exception as e:
+        return False, str(e)
+
+
+def check_active_plugins() -> tuple[bool, str]:
+    """Verify connector plugins registration and connection status."""
+    try:
+        from connectors.registry import discover_connectors
+        active = []
+        for conn in discover_connectors():
+            if conn.authenticate():
+                active.append(conn.name)
+        if active:
+            return True, f"{len(active)} active ({', '.join(active)})"
+        return False, "0 active"
+    except Exception as e:
+        return False, str(e)
+
+
+def run_all_checks() -> list[tuple[str, bool, str]]:
+    """Run all health checks and return a list of (name, passed, detail)."""
+    from infrastructure.docker import check_docker_installed, check_docker_compose_installed
+
+    checks = []
+
+    # Python
+    ok, detail = check_python()
+    checks.append(("Python", ok, detail))
+
+    # Docker
+    ok, detail = check_docker_installed()
+    checks.append(("Docker", ok, detail))
+
+    # Docker Compose
+    ok, detail = check_docker_compose_installed()
+    checks.append(("Docker Compose", ok, detail))
+
+    # SQLite
+    ok, detail = check_sqlite()
+    checks.append(("SQLite", ok, detail))
+
+    # Neo4j
+    ok, detail = check_neo4j()
+    checks.append(("Neo4j", ok, detail))
+
+    # Qdrant
+    ok, detail = check_qdrant()
+    checks.append(("Qdrant", ok, detail))
+
+    # Embedding Model
+    ok, detail = check_embedding_model()
+    checks.append(("Embedding Model", ok, detail))
+
+    # Groq
+    ok, detail = check_groq_api()
+    checks.append(("Groq", ok, detail))
+
+    # Composio
+    ok, detail = check_composio()
+    checks.append(("Composio", ok, detail))
+
+    # Config Validation
+    ok, detail = check_config_validation()
+    checks.append(("Config Validation", ok, detail))
+
+    # Active Plugins
+    ok, detail = check_active_plugins()
+    checks.append(("Active Plugins", ok, detail))
+
+    # Connectors
+    for name in ["github", "gmail", "notion"]:
+        ok, detail = check_connector(name)
+        checks.append((name.capitalize(), ok, detail))
+
+    # Workspace
+    ok, detail = check_workspace()
+    checks.append(("Workspace", ok, detail))
+
+    # Disk Usage
+    ok, detail = check_disk_usage()
+    checks.append(("Disk Usage", ok, detail))
+
+    # Memory Usage
+    ok, detail = check_memory_usage()
+    checks.append(("Memory Usage", ok, detail))
+
+    # Version
+    checks.append(("Version", True, _get_version()))
+
+    # Workspace Profile & Database Counts
+    try:
+        from infrastructure.workspace import get_active_profile
+        checks.append(("Workspace Profile", True, get_active_profile()))
+    except Exception:
+        pass
+
+    try:
+        from core.vector_store import get_vector_index_stats
+        vectors = get_vector_index_stats().get("vectors", 0)
+        checks.append(("Vector Count", True, f"{vectors:,}"))
+    except Exception:
+        checks.append(("Vector Count", True, "0"))
+
+    try:
+        from storage.db import get_repo_count, get_email_count
+        repos = get_repo_count()
+        emails = get_email_count()
+        checks.append(("Repositories", True, str(repos)))
+        checks.append(("Emails", True, str(emails)))
+    except Exception:
+        checks.append(("Repositories", True, "0"))
+        checks.append(("Emails", True, "0"))
+
+    try:
+        from storage.graph import GraphStore
+        nodes = 0
+        rels = 0
+        graph = GraphStore()
+        if graph.is_fallback:
+            from storage.db import get_connection
+            conn = get_connection()
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM graph_nodes")
+            nodes = cursor.fetchone()[0]
+            cursor.execute("SELECT COUNT(*) FROM graph_relationships")
+            rels = cursor.fetchone()[0]
+            conn.close()
+        else:
+            with graph.driver.session() as session:
+                r_nodes = session.run("MATCH (n) RETURN count(n) AS c")
+                nodes = r_nodes.single()["c"]
+                r_rels = session.run("MATCH ()-[r]->() RETURN count(r) AS c")
+                rels = r_rels.single()["c"]
+        checks.append(("Graph Nodes", True, str(nodes)))
+        checks.append(("Graph Relationships", True, str(rels)))
+    except Exception:
+        checks.append(("Graph Nodes", True, "0"))
+        checks.append(("Graph Relationships", True, "0"))
+
+    return checks
+
+
+def _get_version() -> str:
+    """Get the Memory-OS package version."""
+    try:
+        from importlib.metadata import version
+        return version("memory-os")
+    except Exception:
+        return "0.1.0"

infrastructure/observability.py

@@ -0,0 +1,160 @@
+"""
+Infrastructure: Observability and Performance Monitoring.
+
+Parses logs from the active log file (or tracks metrics) to measure:
+- Sync execution time
+- Embedding generation latency
+- Qdrant upload time
+- Neo4j insertion time
+- LLM call count and durations
+- RAG query execution time
+"""
+
+import os
+import re
+from pathlib import Path
+from infrastructure.workspace import get_workspace_root
+
+
+def get_log_file_path() -> Path:
+    """Return path to active memory_os.log."""
+    return get_workspace_root() / "logs" / "memory_os.log"
+
+
+def parse_observability_metrics() -> dict:
+    """Scan the memory_os.log file and parse latency/observability metrics."""
+    log_path = get_log_file_path()
+    
+    # Fallback to local logs directory if not in home workspace yet
+    if not log_path.exists():
+        log_path = Path("logs/memory_os.log")
+        
+    metrics = {
+        "sync_times": [],
+        "embedding_times": [],  # (duration, num_chunks)
+        "qdrant_uploads": [],   # (duration, num_vectors)
+        "neo4j_inserts": [],    # (duration, nodes, relationships)
+        "llm_calls": [],
+        "retrievals": [],
+        "rag_times": [],
+    }
+    
+    if not log_path.exists():
+        return metrics
+
+    # Regex definitions matching existing logs in core/storage/main
+    r_sync = re.compile(r"Sync completed in (\d+\.\d+)s")
+    r_rebuild = re.compile(r"Full rebuild completed in (\d+\.\d+)s")
+    r_embed = re.compile(r"Embedding generation complete for (\d+) chunks in (\d+\.\d+)s")
+    r_upload = re.compile(r"Vector upload complete for (\d+) vectors in (\d+\.\d+)s")
+    r_graph = re.compile(r"Graph Sync Complete.*Nodes: (\d+), Relationships: (\d+)\. Duration: (\d+\.\d+)s")
+    r_llm = re.compile(r"Groq LLM generation finished in (\d+\.\d+)s")
+    r_retrieval = re.compile(r"Hybrid retrieval finished for query:.*Retrieved (\d+) items in (\d+\.\d+)s")
+    r_rag = re.compile(r"RAG pipeline complete for question:.*in (\d+\.\d+)s")
+
+    try:
+        with open(log_path, "r", encoding="utf-8", errors="ignore") as f:
+            for line in f:
+                # Sync
+                m = r_sync.search(line) or r_rebuild.search(line)
+                if m:
+                    metrics["sync_times"].append(float(m.group(1)))
+                    continue
+                
+                # Embeddings
+                m = r_embed.search(line)
+                if m:
+                    metrics["embedding_times"].append((float(m.group(2)), int(m.group(1))))
+                    continue
+                
+                # Qdrant Uploads
+                m = r_upload.search(line)
+                if m:
+                    metrics["qdrant_uploads"].append((float(m.group(2)), int(m.group(1))))
+                    continue
+                
+                # Neo4j Graph Sync
+                m = r_graph.search(line)
+                if m:
+                    metrics["neo4j_inserts"].append((float(m.group(3)), int(m.group(1)), int(m.group(2))))
+                    continue
+                
+                # LLM Calls
+                m = r_llm.search(line)
+                if m:
+                    metrics["llm_calls"].append(float(m.group(1)))
+                    continue
+                
+                # Hybrid Retrieval
+                m = r_retrieval.search(line)
+                if m:
+                    metrics["retrievals"].append(float(m.group(2)))
+                    continue
+                
+                # RAG Times
+                m = r_rag.search(line)
+                if m:
+                    metrics["rag_times"].append(float(m.group(1)))
+                    continue
+    except Exception:
+        pass
+        
+    return metrics
+
+
+def get_performance_summary() -> dict:
+    """Format metrics into a user-friendly statistics summary."""
+    data = parse_observability_metrics()
+    
+    def avg(lst):
+        return sum(lst) / len(lst) if lst else 0.0
+
+    # Sync
+    syncs = data["sync_times"]
+    last_sync = syncs[-1] if syncs else 0.0
+    avg_sync = avg(syncs)
+
+    # Embeddings
+    embeds = data["embedding_times"]
+    total_embed_time = sum(t for t, _ in embeds)
+    total_chunks = sum(c for _, c in embeds)
+    avg_chunk_latency = (total_embed_time / total_chunks) if total_chunks > 0 else 0.0
+
+    # Qdrant uploads
+    uploads = data["qdrant_uploads"]
+    total_upload_time = sum(t for t, _ in uploads)
+    total_vectors = sum(v for _, v in uploads)
+    avg_vector_latency = (total_upload_time / total_vectors) if total_vectors > 0 else 0.0
+
+    # Neo4j
+    inserts = data["neo4j_inserts"]
+    avg_neo4j = avg([t for t, _, _ in inserts])
+    total_nodes = sum(n for _, n, _ in inserts)
+    total_rels = sum(r for _, _, r in inserts)
+
+    # LLM
+    llms = data["llm_calls"]
+    total_llm_calls = len(llms)
+    avg_llm = avg(llms)
+
+    # Retrieval & RAG
+    retrievals = data["retrievals"]
+    avg_retrieval = avg(retrievals)
+    
+    rags = data["rag_times"]
+    avg_rag = avg(rags)
+
+    return {
+        "sync_count": len(syncs),
+        "last_sync_duration": last_sync,
+        "avg_sync_duration": avg_sync,
+        "avg_chunk_latency": avg_chunk_latency,
+        "avg_vector_latency": avg_vector_latency,
+        "avg_neo4j_sync": avg_neo4j,
+        "total_nodes_synced": total_nodes,
+        "total_rels_synced": total_rels,
+        "llm_call_count": total_llm_calls,
+        "avg_llm_duration": avg_llm,
+        "avg_retrieval_duration": avg_retrieval,
+        "avg_rag_duration": avg_rag,
+    }

infrastructure/workspace.py

@@ -0,0 +1,152 @@
+"""
+Infrastructure: Workspace directory and profile management.
+
+Manages the ~/.memory-os/ directory tree and workspace profiles
+(default, work, personal, etc.).
+"""
+
+import os
+import shutil
+from pathlib import Path
+
+
+def get_workspace_root() -> Path:
+    """Return the root workspace directory (~/.memory-os/)."""
+    root = os.getenv("MEMORY_OS_WORKSPACE", "~/.memory-os")
+    return Path(os.path.expanduser(root))
+
+
+def ensure_workspace(profile: str = "default"):
+    """Create the full workspace directory tree for a profile.
+
+    Creates:
+        ~/.memory-os/
+        ├── config.toml         (not created here, handled by config.py)
+        ├── active_workspace
+        ├── workspaces/
+        │   └── <profile>/
+        │       ├── workspace.db  (created by SQLite on first use)
+        │       ├── qdrant/
+        │       ├── neo4j/
+        │       └── embeddings/
+        ├── logs/
+        ├── cache/
+        └── backups/
+    """
+    root = get_workspace_root()
+
+    # Top-level shared directories
+    (root / "logs").mkdir(parents=True, exist_ok=True)
+    (root / "cache").mkdir(parents=True, exist_ok=True)
+    (root / "backups").mkdir(parents=True, exist_ok=True)
+    (root / "workspaces").mkdir(parents=True, exist_ok=True)
+
+    # Profile-specific directories
+    profile_dir = root / "workspaces" / profile
+    (profile_dir / "qdrant").mkdir(parents=True, exist_ok=True)
+    (profile_dir / "neo4j").mkdir(parents=True, exist_ok=True)
+    (profile_dir / "embeddings").mkdir(parents=True, exist_ok=True)
+
+    # Set active workspace if no active_workspace file exists
+    active_file = root / "active_workspace"
+    if not active_file.exists():
+        active_file.write_text(profile, encoding="utf-8")
+
+
+def get_active_profile() -> str:
+    """Read the currently active workspace profile name."""
+    active_file = get_workspace_root() / "active_workspace"
+    if active_file.exists():
+        return active_file.read_text(encoding="utf-8").strip()
+    return "default"
+
+
+def set_active_profile(name: str):
+    """Set the active workspace profile."""
+    root = get_workspace_root()
+    profile_dir = root / "workspaces" / name
+    if not profile_dir.exists():
+        raise FileNotFoundError(f"Workspace profile '{name}' does not exist.")
+    active_file = root / "active_workspace"
+    active_file.write_text(name, encoding="utf-8")
+
+
+def list_profiles() -> list[str]:
+    """List all workspace profile names."""
+    workspaces_dir = get_workspace_root() / "workspaces"
+    if not workspaces_dir.exists():
+        return []
+    return sorted([
+        d.name for d in workspaces_dir.iterdir()
+        if d.is_dir()
+    ])
+
+
+def create_profile(name: str):
+    """Create a new workspace profile."""
+    root = get_workspace_root()
+    profile_dir = root / "workspaces" / name
+    if profile_dir.exists():
+        raise FileExistsError(f"Workspace profile '{name}' already exists.")
+    ensure_workspace(profile=name)
+
+
+def delete_profile(name: str):
+    """Delete a workspace profile directory."""
+    if name == get_active_profile():
+        raise ValueError(f"Cannot delete the active workspace profile '{name}'. Switch first.")
+    profile_dir = get_workspace_root() / "workspaces" / name
+    if not profile_dir.exists():
+        raise FileNotFoundError(f"Workspace profile '{name}' does not exist.")
+    shutil.rmtree(profile_dir)
+
+
+def get_profile_path(name: str | None = None) -> Path:
+    """Return the path for a specific profile (or the active one)."""
+    if name is None:
+        name = get_active_profile()
+    return get_workspace_root() / "workspaces" / name
+
+
+def get_db_path(profile: str | None = None) -> Path:
+    """Return the SQLite database path for a workspace profile."""
+    return get_profile_path(profile) / "workspace.db"
+
+
+def get_qdrant_path(profile: str | None = None) -> Path:
+    """Return the Qdrant storage path for a workspace profile."""
+    return get_profile_path(profile) / "qdrant"
+
+
+def get_neo4j_path(profile: str | None = None) -> Path:
+    """Return the Neo4j data path for a workspace profile."""
+    return get_profile_path(profile) / "neo4j"
+
+
+def get_logs_path() -> Path:
+    """Return the shared logs directory path."""
+    return get_workspace_root() / "logs"
+
+
+def get_backups_path() -> Path:
+    """Return the shared backups directory path."""
+    return get_workspace_root() / "backups"
+
+
+def get_cache_path() -> Path:
+    """Return the shared cache directory path."""
+    return get_workspace_root() / "cache"
+
+
+def get_workspace_size(profile: str | None = None) -> int:
+    """Calculate total disk usage in bytes for a workspace profile."""
+    path = get_profile_path(profile) if profile else get_workspace_root()
+    total = 0
+    for dirpath, dirnames, filenames in os.walk(path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            try:
+                total += os.path.getsize(fp)
+            except OSError:
+                pass
+    return total

models/__init__.py

@@ -0,0 +1 @@
+# models package

models/memory.py

@@ -0,0 +1,28 @@
+class Repository:
+    def __init__(self, repo_name: str, description: str, language: str, visibility: str, stars: int, forks: int, open_issues: int, default_branch: str, updated_at: str, url: str):
+        self.repo_name = repo_name
+        self.description = description
+        self.language = language
+        self.visibility = visibility
+        self.stars = stars
+        self.forks = forks
+        self.open_issues = open_issues
+        self.default_branch = default_branch
+        self.updated_at = updated_at
+        self.url = url
+
+class RepositoryDocument:
+    def __init__(self, repo_name: str, file_name: str, content: str, source: str, synced_at: str):
+        self.repo_name = repo_name
+        self.file_name = file_name
+        self.content = content
+        self.source = source
+        self.synced_at = synced_at
+
+class Email:
+    def __init__(self, message_id: str, subject: str, sender: str, snippet: str, received_at: str):
+        self.message_id = message_id
+        self.subject = subject
+        self.sender = sender
+        self.snippet = snippet
+        self.received_at = received_at

storage/__init__.py

@@ -0,0 +1 @@
+# storage package