mnemo-dev 0.1.0__py3-none-any.whl

mnemo/memory.py

@@ -0,0 +1,250 @@
+"""Persistent memory store for decisions, context, and notes."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+from .config import CONTEXT_FILE, DECISIONS_FILE, MEMORY_FILE, mnemo_path
+from .retrieval import semantic_query
+from .storage import Collections, get_storage
+
+MAX_OUTPUT_CHARS = 75000  # Hard limit for MCP response
+RECALL_BUDGET = 12000  # Target: keep recall small (~3K tokens)
+MAX_MEMORY_ENTRIES = 50  # Keep last 50 entries, summarize older ones
+
+
+def _refresh_rule(repo_root: Path) -> None:
+    """Update installed client context files with latest context."""
+    from .init import refresh_context_files
+
+    refresh_context_files(repo_root)
+
+
+def _as_list(data: Any) -> list[dict[str, Any]]:
+    return data if isinstance(data, list) else []
+
+
+def _as_dict(data: Any) -> dict[str, Any]:
+    return data if isinstance(data, dict) else {}
+
+
+def _compact_memory(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """When memory exceeds limit, compress old entries into a summary."""
+    if len(entries) <= MAX_MEMORY_ENTRIES:
+        return entries
+
+    keep = entries[-30:]
+    old = entries[:-30]
+    summary_lines = [entry["content"] for entry in old[-20:] if entry.get("content")]
+    summary = "Previous context: " + "; ".join(summary_lines)
+
+    summary_entry = {
+        "id": 0,
+        "timestamp": old[-1]["timestamp"],
+        "category": "summary",
+        "content": summary[:500],
+    }
+
+    return [summary_entry] + keep
+
+
+def add_memory(repo_root: Path, content: str, category: str = "general") -> dict[str, Any]:
+    """Add a memory entry and refresh installed context files."""
+    storage = get_storage(repo_root)
+    entries = _as_list(storage.read_collection(Collections.MEMORY))
+    entry = {
+        "id": len(entries) + 1,
+        "timestamp": time.time(),
+        "category": category,
+        "content": content,
+    }
+    entries.append(entry)
+    storage.write_collection(Collections.MEMORY, _compact_memory(entries))
+    _refresh_rule(repo_root)
+    return entry
+
+
+def add_decision(repo_root: Path, decision: str, reasoning: str = "") -> dict[str, Any]:
+    """Record a decision and refresh installed context files."""
+    storage = get_storage(repo_root)
+    entries = _as_list(storage.read_collection(Collections.DECISIONS))
+    entry = {
+        "id": len(entries) + 1,
+        "timestamp": time.time(),
+        "decision": decision,
+        "reasoning": reasoning,
+    }
+    entries.append(entry)
+    storage.write_collection(Collections.DECISIONS, entries)
+    _refresh_rule(repo_root)
+    return entry
+
+
+def save_context(repo_root: Path, context: dict[str, Any]) -> None:
+    """Save project context and refresh installed context files."""
+    storage = get_storage(repo_root)
+    existing = _as_dict(storage.read_collection(Collections.CONTEXT))
+    existing.update(context)
+    existing["last_updated"] = time.time()
+    storage.write_collection(Collections.CONTEXT, existing)
+    _refresh_rule(repo_root)
+
+
+def lookup(repo_root: Path, query: str) -> str:
+    """Look up detailed info for a specific file or folder - parses on demand."""
+    from .repo_map import MAX_FILE_SIZE, SUPPORTED_EXTENSIONS, _extract_file, _should_ignore
+    from .chunking import make_code_chunks
+    from .retrieval import index_chunks
+
+    query_lower = query.lower().strip("/")
+    matches: list[tuple[str, dict[str, Any]]] = []
+    discovered_chunks = []
+
+    for ext, language in SUPPORTED_EXTENSIONS.items():
+        for filepath in repo_root.rglob(f"*{ext}"):
+            if _should_ignore(filepath) or filepath.stat().st_size > MAX_FILE_SIZE:
+                continue
+            rel = str(filepath.relative_to(repo_root))
+            if query_lower not in rel.lower():
+                continue
+            try:
+                source = filepath.read_bytes()
+            except (OSError, PermissionError):
+                continue
+            info = _extract_file(source, language)
+            if info:
+                matches.append((rel, info))
+                discovered_chunks.extend(make_code_chunks(rel, language, info))
+
+    # Feed discovered code into the index for future queries
+    if discovered_chunks:
+        try:
+            index_chunks(repo_root, "code", discovered_chunks)
+        except Exception:
+            pass
+
+    if not matches:
+        return f"No files matching '{query}' found."
+
+    lines = [f"# Details for '{query}'\n"]
+    for filepath, info in sorted(matches):
+        lines.append(f"## {filepath}")
+        if info.get("imports"):
+            lines.append("**Imports:** " + ", ".join(info["imports"]))
+        for cls in info.get("classes", []):
+            impl = f" : {cls['implements']}" if cls.get("implements") else ""
+            lines.append(f"### `{cls['name']}{impl}`")
+            for method in cls.get("methods", []):
+                lines.append(f"- {method}")
+        for function in info.get("functions", []):
+            lines.append(f"- {function}")
+        lines.append("")
+
+    result = "\n".join(lines)
+    if len(result) > MAX_OUTPUT_CHARS:
+        result = result[:MAX_OUTPUT_CHARS]
+        last_nl = result.rfind("\n")
+        if last_nl > 0:
+            result = result[:last_nl]
+        result += "\n... (narrow your query for more details)"
+    return result
+
+
+def recall(repo_root: Path) -> str:
+    """Recall project memory as a compact markdown document."""
+    from .repo_map import CHANGELOG_FILE, has_changes, save_repo_map
+
+    base = mnemo_path(repo_root)
+    if not base.exists():
+        return ""
+
+    if has_changes(repo_root):
+        save_repo_map(repo_root, index=False)
+
+    storage = get_storage(repo_root)
+    sections: list[str] = []
+
+    context = dict(_as_dict(storage.read_collection(Collections.CONTEXT)))
+    context.pop("last_updated", None)
+    if context:
+        sections.append("# Project Context")
+        for key, value in context.items():
+            sections.append(f"- **{key}**: {value}")
+        sections.append("")
+
+    decisions = _as_list(storage.read_collection(Collections.DECISIONS))
+    if decisions:
+        sections.append("# Decisions")
+        for decision in decisions:
+            reasoning = f" - {decision['reasoning']}" if decision.get("reasoning") else ""
+            sections.append(f"- {decision['decision']}{reasoning}")
+        sections.append("")
+
+    memory = _as_list(storage.read_collection(Collections.MEMORY))
+    if memory:
+        sections.append("# Memory")
+        for item in memory:
+            cat = f" [{item['category']}]" if item.get("category") != "general" else ""
+            sections.append(f"- {item['content']}{cat}")
+        sections.append("")
+
+    tasks = _as_list(storage.read_collection(Collections.TASKS))
+    active_tasks = [task for task in tasks if task.get("status") == "active"]
+    if active_tasks:
+        sections.append("# Active Task Context")
+        active = active_tasks[-1]
+        sections.append(f"- **{active.get('task_id', '')}**: {active.get('description', '')}")
+        task_query = " ".join(
+            filter(
+                None,
+                [
+                    str(active.get("task_id", "")),
+                    str(active.get("description", "")),
+                    " ".join(active.get("files", [])),
+                    str(active.get("notes", "")),
+                ],
+            )
+        )
+        hits = semantic_query(repo_root, "code", task_query, limit=5)
+        for hit in hits:
+            meta = hit.get("metadata", {})
+            sections.append(f"- Relevant: `{meta.get('path', '')}` :: `{meta.get('symbol', '')}`")
+        sections.append("")
+
+    changelog_path = base / CHANGELOG_FILE
+    if changelog_path.exists():
+        changelog = json.loads(changelog_path.read_text(encoding="utf-8"))
+        if changelog:
+            sections.append("# Recent Changes")
+            for entry in changelog[-5:]:
+                if entry.get("added"):
+                    sections.append(f"- Added: {', '.join(entry['added'])}")
+                if entry.get("modified"):
+                    sections.append(f"- Modified: {', '.join(entry['modified'])}")
+                if entry.get("deleted"):
+                    sections.append(f"- Deleted: {', '.join(entry['deleted'])}")
+                if entry.get("renamed"):
+                    for new, old in entry["renamed"].items():
+                        sections.append(f"- Renamed: {old} -> {new}")
+            sections.append("")
+
+    summary_path = base / "summary.md"
+    if summary_path.exists():
+        summary = summary_path.read_text(encoding="utf-8")
+        sections.append("# Repo Map")
+        sections.append("(use mnemo_lookup for method-level details)\n")
+
+        header_size = len("\n".join(sections))
+        budget = RECALL_BUDGET - header_size
+        if len(summary) > budget:
+            summary = summary[:budget]
+            last_nl = summary.rfind("\n")
+            if last_nl > 0:
+                summary = summary[:last_nl]
+            summary += "\n... (use mnemo_lookup for details)"
+        sections.append(summary)
+
+    return "\n".join(sections)

mnemo/onboarding/__init__.py

@@ -0,0 +1,86 @@
+"""Onboarding Mode — generate a complete project overview for new developers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..config import mnemo_path
+from ..intelligence import detect_patterns, detect_dependencies, detect_service_calls
+from ..api_discovery import discover_apis
+
+
+def generate_onboarding(repo_root: Path) -> str:
+    """Generate a comprehensive onboarding document for new team members."""
+    lines = ["# Project Onboarding Guide\n"]
+
+    # Project basics
+    lines.append("## Overview")
+    lines.append(f"Repository: `{repo_root.name}`")
+
+    # Detect services
+    services = set()
+    for f in repo_root.rglob("Program.cs"):
+        from ..config import IGNORE_DIRS
+        if not any(part in IGNORE_DIRS for part in f.relative_to(repo_root).parts):
+            services.add(f.relative_to(repo_root).parts[0])
+    if services:
+        lines.append(f"Services: {len(services)} microservices")
+        for svc in sorted(services):
+            lines.append(f"- **{svc}**")
+    lines.append("")
+
+    # Patterns
+    patterns = detect_patterns(repo_root)
+    if patterns:
+        lines.append("## Tech Stack & Patterns")
+        for p in patterns:
+            lines.append(f"- {p}")
+        lines.append("")
+
+    # Architecture
+    graph = detect_service_calls(repo_root)
+    if graph:
+        lines.append("## Service Architecture")
+        lines.append("How services communicate:")
+        for svc, targets in sorted(graph.items()):
+            lines.append(f"- **{svc}** calls → {', '.join(targets)}")
+        lines.append("")
+
+    # Key dependencies
+    deps = detect_dependencies(repo_root)
+    if deps:
+        lines.append("## Key Dependencies")
+        for project, pkgs in sorted(deps.items()):
+            key_pkgs = [p for p in pkgs if any(k in p for k in ["Cosmos", "Azure", "Identity", "Polly", "Swagger"])]
+            if key_pkgs:
+                lines.append(f"**{project}:** {', '.join(key_pkgs[:5])}")
+        lines.append("")
+
+    # API endpoints summary
+    lines.append("## API Endpoints")
+    lines.append("Use `mnemo_discover_apis` for full details. Key services:")
+    for svc in sorted(services):
+        lines.append(f"- {svc}")
+    lines.append("")
+
+    # Getting started
+    lines.append("## Getting Started")
+    lines.append("1. Clone the repo")
+    lines.append("2. Run `mnemo init` to set up project memory")
+    lines.append("3. Ask Amazon Q: \"What does this project do?\"")
+    lines.append("4. Ask: \"Show me the architecture\"")
+    lines.append("5. Ask: \"I need to add a new [feature], show me similar implementations\"")
+    lines.append("")
+
+    # Knowledge base
+    kb_path = mnemo_path(repo_root) / "knowledge"
+    if kb_path.exists():
+        kb_files = list(kb_path.rglob("*.md"))
+        if kb_files:
+            lines.append("## Team Knowledge")
+            lines.append("Check `.mnemo/knowledge/` for:")
+            for f in kb_files:
+                lines.append(f"- {f.name}")
+            lines.append("")
+
+    return "\n".join(lines)

mnemo/repo_map.py

@@ -0,0 +1,357 @@
+"""Lightweight repo map — stores a hash index and pre-built markdown summary."""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from pathlib import Path
+from typing import Any
+
+from .config import IGNORE_DIRS, SUPPORTED_EXTENSIONS, mnemo_path, REPO_MAP_FILE
+from .chunking import make_code_chunks
+from .retrieval import index_chunks
+from .storage import Collections, get_storage
+
+CHANGELOG_FILE = "changelog.json"
+HASH_INDEX_FILE = "hashes.json"
+MAX_FILE_SIZE = 100_000
+
+
+def _should_ignore(path: Path) -> bool:
+    return any(part in IGNORE_DIRS for part in path.parts)
+
+
+def _file_hash(path: Path) -> str:
+    return hashlib.md5(path.read_bytes()).hexdigest()
+
+
+def _get_parser(language: str):
+    import tree_sitter_python
+    import tree_sitter_javascript
+    import tree_sitter_typescript
+    import tree_sitter_go
+    import tree_sitter_c_sharp
+    from tree_sitter import Language, Parser
+
+    lang_map = {
+        "python": tree_sitter_python.language(),
+        "javascript": tree_sitter_javascript.language(),
+        "typescript": tree_sitter_typescript.language_typescript(),
+        "go": tree_sitter_go.language(),
+        "csharp": tree_sitter_c_sharp.language(),
+    }
+    lang = lang_map.get(language)
+    if not lang:
+        return None
+    return Parser(Language(lang))
+
+
+def _get_node_text(node) -> str:
+    return node.text.decode() if node else ""
+
+
+# --- Extractors (return compact dict per file) ---
+
+def _extract_csharp(source: bytes, parser) -> dict:
+    tree = parser.parse(source)
+    result: dict = {"imports": [], "classes": []}
+
+    def extract_class(node) -> dict:
+        name = _get_node_text(node.child_by_field_name("name"))
+        base = ""
+        methods = []
+        for child in node.children:
+            if child.type == "base_list":
+                base = child.text.decode().lstrip(": ").strip()
+            if child.type == "declaration_list":
+                for member in child.children:
+                    if member.type in ("method_declaration", "constructor_declaration"):
+                        mname = _get_node_text(member.child_by_field_name("name"))
+                        if not mname:
+                            continue
+                        params = ""
+                        ret = ""
+                        for part in member.children:
+                            if part.type == "parameter_list":
+                                params = part.text.decode()
+                            elif part.type in ("predefined_type", "identifier", "generic_name",
+                                              "nullable_type", "array_type", "void_keyword", "qualified_name"):
+                                name_node = member.child_by_field_name("name")
+                                if name_node and part.end_byte < name_node.start_byte:
+                                    ret = part.text.decode()
+                        sig = f"{ret + ' ' if ret else ''}{mname}{params}".strip()
+                        methods.append(sig)
+        entry = {"name": name}
+        if base:
+            entry["implements"] = base
+        if methods:
+            entry["methods"] = methods
+        return entry
+
+    def walk(node):
+        for child in node.children:
+            if child.type == "using_directive":
+                result["imports"].append(child.text.decode().strip())
+            elif child.type in ("class_declaration", "interface_declaration",
+                                "struct_declaration", "record_declaration"):
+                result["classes"].append(extract_class(child))
+            elif child.type in ("file_scoped_namespace_declaration",
+                                "namespace_declaration", "declaration_list"):
+                walk(child)
+
+    walk(tree.root_node)
+    return {k: v for k, v in result.items() if v}
+
+
+def _extract_python(source: bytes, parser) -> dict:
+    tree = parser.parse(source)
+    root = tree.root_node
+    result: dict = {"imports": [], "classes": [], "functions": []}
+
+    for node in root.children:
+        if node.type in ("import_statement", "import_from_statement"):
+            result["imports"].append(node.text.decode().strip())
+
+        actual = node
+        decs = []
+        if node.type == "decorated_definition":
+            for child in node.children:
+                if child.type == "decorator":
+                    decs.append(child.text.decode().strip())
+                elif child.type in ("function_definition", "class_definition"):
+                    actual = child
+
+        if actual.type == "function_definition":
+            name = _get_node_text(actual.child_by_field_name("name"))
+            params = _get_node_text(actual.child_by_field_name("parameters"))
+            ret = _get_node_text(actual.child_by_field_name("return_type"))
+            sig = f"def {name}{params}"
+            if ret:
+                sig += f" -> {ret}"
+            if decs:
+                sig = f"{decs[0]} {sig}"
+            result["functions"].append(sig)
+
+        elif actual.type == "class_definition":
+            cname = _get_node_text(actual.child_by_field_name("name"))
+            methods = []
+            body = actual.child_by_field_name("body")
+            if body:
+                for child in body.children:
+                    fn = child
+                    if child.type == "decorated_definition":
+                        for sub in child.children:
+                            if sub.type == "function_definition":
+                                fn = sub
+                    if fn.type == "function_definition":
+                        mname = _get_node_text(fn.child_by_field_name("name"))
+                        mparams = _get_node_text(fn.child_by_field_name("parameters"))
+                        methods.append(f"{mname}{mparams}")
+            entry = {"name": cname}
+            if methods:
+                entry["methods"] = methods
+            result["classes"].append(entry)
+
+    return {k: v for k, v in result.items() if v}
+
+
+def _extract_js(source: bytes, parser) -> dict:
+    tree = parser.parse(source)
+    result: dict = {"functions": [], "classes": []}
+
+    def walk(node):
+        if node.type == "function_declaration":
+            name = _get_node_text(node.child_by_field_name("name"))
+            if name:
+                result["functions"].append(name)
+        elif node.type == "lexical_declaration":
+            text = node.text.decode(errors="replace")
+            if "=>" in text:
+                result["functions"].append(text.split("=")[0].replace("const", "").replace("let", "").replace("var", "").strip())
+        elif node.type == "method_definition":
+            name = _get_node_text(node.child_by_field_name("name"))
+            if name:
+                result["functions"].append(name)
+        elif node.type == "class_declaration":
+            cname = _get_node_text(node.child_by_field_name("name")) or "AnonymousClass"
+            methods = []
+            for child in node.children:
+                if child.type == "class_body":
+                    for member in child.children:
+                        if member.type == "method_definition":
+                            mname = _get_node_text(member.child_by_field_name("name"))
+                            if mname:
+                                methods.append(mname)
+            result["classes"].append({"name": cname, "methods": methods})
+        for child in node.children:
+            walk(child)
+
+    walk(tree.root_node)
+    result["functions"] = sorted(set(result["functions"]))
+    return {k: v for k, v in result.items() if v}
+
+
+def _extract_go(source: bytes, parser) -> dict:
+    tree = parser.parse(source)
+    result: dict = {"functions": [], "classes": []}
+    for node in tree.root_node.children:
+        if node.type == "function_declaration":
+            name = _get_node_text(node.child_by_field_name("name"))
+            if name:
+                result["functions"].append(name)
+        elif node.type == "method_declaration":
+            name = _get_node_text(node.child_by_field_name("name"))
+            receiver = _get_node_text(node.child_by_field_name("receiver"))
+            if name:
+                symbol = f"{receiver}.{name}" if receiver else name
+                result["functions"].append(symbol)
+        elif node.type == "type_declaration":
+            text = node.text.decode(errors="replace")
+            for line in text.splitlines():
+                line = line.strip()
+                if line.startswith("type ") and " struct" in line:
+                    parts = line.split()
+                    if len(parts) >= 2:
+                        result["classes"].append({"name": parts[1], "methods": []})
+    return {k: v for k, v in result.items() if v}
+
+
+def _extract_file(source: bytes, language: str) -> dict | None:
+    parser = _get_parser(language)
+    if not parser:
+        return None
+    try:
+        if language == "csharp":
+            return _extract_csharp(source, parser)
+        elif language == "python":
+            return _extract_python(source, parser)
+        elif language in ("javascript", "typescript"):
+            return _extract_js(source, parser)
+        elif language == "go":
+            return _extract_go(source, parser)
+        return None
+    except Exception:
+        return None
+
+
+# --- Hash index for change detection ---
+
+def _load_hashes(repo_root: Path) -> dict[str, str]:
+    data = get_storage(repo_root).read_collection(Collections.HASHES)
+    if not isinstance(data, dict):
+        return {}
+    return {str(path): str(file_hash) for path, file_hash in data.items()}
+
+
+def _save_hashes(repo_root: Path, hashes: dict[str, str]):
+    get_storage(repo_root).write_collection(Collections.HASHES, hashes)
+
+
+def has_changes(repo_root: Path) -> bool:
+    """Quick check if any files changed since last map generation."""
+    old_hashes = _load_hashes(repo_root)
+    if not old_hashes:
+        return True
+
+    for ext, language in SUPPORTED_EXTENSIONS.items():
+        for filepath in repo_root.rglob(f"*{ext}"):
+            if _should_ignore(filepath) or filepath.stat().st_size > MAX_FILE_SIZE:
+                continue
+            rel = str(filepath.relative_to(repo_root))
+            try:
+                h = _file_hash(filepath)
+            except (OSError, PermissionError):
+                continue
+            if old_hashes.get(rel) != h:
+                return True
+
+    # Check for deletions
+    for rel in old_hashes:
+        if not (repo_root / rel).exists():
+            return True
+
+    return False
+
+
+# --- Summary generation ---
+
+def generate_summary(repo_root: Path, index: bool = True) -> str:
+    """Generate a compact markdown summary of the repo structure."""
+    tree: dict[str, dict[str, list[str]]] = {}
+    hashes: dict[str, str] = {}
+    all_chunks = []
+
+    # Try Roslyn for C# if .NET SDK is available
+    from .analyzers import roslyn_available, run_roslyn_analyzer, roslyn_to_mnemo_format
+    roslyn_data: dict[str, dict] = {}
+    if roslyn_available(repo_root):
+        results = run_roslyn_analyzer(repo_root)
+        if results:
+            roslyn_data = roslyn_to_mnemo_format(results, repo_root)
+
+    for ext, language in SUPPORTED_EXTENSIONS.items():
+        for filepath in repo_root.rglob(f"*{ext}"):
+            if _should_ignore(filepath) or filepath.stat().st_size > MAX_FILE_SIZE:
+                continue
+            try:
+                source = filepath.read_bytes()
+            except (OSError, PermissionError):
+                continue
+
+            rel = str(filepath.relative_to(repo_root))
+            hashes[rel] = hashlib.md5(source).hexdigest()
+
+            parts = rel.split("/")
+            module = parts[0] if len(parts) > 1 else "."
+            submodule = parts[1] if len(parts) > 2 else "_root"
+            if module not in tree:
+                tree[module] = {}
+            if submodule not in tree[module]:
+                tree[module][submodule] = []
+
+            # Use Roslyn data if available for this file, else tree-sitter
+            info = roslyn_data.get(rel) or _extract_file(source, language)
+            rel_short = "/".join(parts[1:]) or rel
+            if info:
+                all_chunks.extend(make_code_chunks(rel, language, info))
+
+            if info and info.get("classes"):
+                class_names = []
+                for cls in info["classes"]:
+                    name = cls["name"]
+                    impl = f" : {cls['implements']}" if cls.get("implements") else ""
+                    class_names.append(f"`{name}{impl}`")
+                tree[module][submodule].append(f"  {rel_short} → {', '.join(class_names)}")
+            elif info and info.get("functions"):
+                tree[module][submodule].append(f"  {rel_short} ({len(info['functions'])} functions)")
+            else:
+                tree[module][submodule].append(f"  {rel_short}")
+
+    # Save hashes
+    _save_hashes(repo_root, hashes)
+
+    # Build markdown
+    lines = []
+    for module in sorted(tree.keys()):
+        lines.append(f"**{module}/**")
+        for submodule in sorted(tree[module].keys()):
+            if submodule != "_root":
+                lines.append(f"  - {submodule}/")
+            for entry in sorted(tree[module][submodule]):
+                lines.append(entry)
+    if index and all_chunks:
+        index_chunks(repo_root, "code", all_chunks)
+    return "\n".join(lines)
+
+
+def save_summary(repo_root: Path, index: bool = True) -> Path:
+    """Generate and save the markdown summary."""
+    summary = generate_summary(repo_root, index=index)
+    out = mnemo_path(repo_root) / "summary.md"
+    out.write_text(summary, encoding="utf-8")
+    return out
+
+
+def save_repo_map(repo_root: Path, index: bool = True) -> Path:
+    """Generate summary (replaces old JSON map)."""
+    return save_summary(repo_root, index=index)