code-context-control 2.28.0__py3-none-any.whl

oracle/services/insight_engine.py

@@ -0,0 +1,307 @@
+"""LLM-powered analysis engine for Oracle."""
+
+import json
+import re
+from pathlib import Path
+
+from oracle.services.cross_memory import CrossMemory
+from oracle.services.memory_reader import MemoryReader
+from oracle.services.ollama_bridge import OllamaBridge
+
+_SYSTEM_PROMPT = """You are Oracle, an AI memory analyst for software projects.
+You analyze project memory facts and identify patterns, risks, and opportunities.
+Always respond with valid JSON. No markdown fences, no extra text."""
+
+
+class InsightEngine:
+    """LLM-powered cross-project analysis."""
+
+    def __init__(self, bridge: OllamaBridge, reader: MemoryReader, cross_memory: CrossMemory):
+        self.bridge = bridge
+        self.reader = reader
+        self.cross_memory = cross_memory
+
+    def analyze_project(self, project_path: str, max_facts: int = 100) -> dict:
+        """LLM reviews a project's top facts. Returns analysis dict."""
+        facts = self.reader.read_facts(project_path)
+        if not facts:
+            return {"project": project_path, "analysis": "No facts found.", "suggestions": []}
+
+        # Sort by relevance (most accessed first), take top N
+        facts.sort(key=lambda f: int(f.get("relevance_count", 0)), reverse=True)
+        top_facts = facts[:max_facts]
+
+        fact_summary = "\n".join(
+            f"- [{f.get('category', 'general')}] {f.get('fact', '')[:200]}"
+            for f in top_facts
+        )
+
+        prompt = f"""Analyze these memory facts from project "{Path(project_path).name}":
+
+{fact_summary}
+
+Return JSON with:
+{{
+  "health_narrative": "1-2 sentence summary of memory health",
+  "key_themes": ["theme1", "theme2"],
+  "suggestions": [
+    {{"type": "merge|archive|investigate", "description": "what to do", "fact_ids": []}}
+  ]
+}}"""
+
+        response = self.bridge.generate(prompt, system=_SYSTEM_PROMPT, max_tokens=1024)
+        if not response:
+            return {"project": project_path, "analysis": "LLM unavailable", "suggestions": []}
+
+        parsed = self._parse_json(response)
+        return {
+            "project": project_path,
+            "analysis": parsed.get("health_narrative", response[:500]),
+            "key_themes": parsed.get("key_themes", []),
+            "suggestions": parsed.get("suggestions", []),
+        }
+
+    def find_cross_project_links(self, project_paths: list[str], max_facts_per: int = 50) -> list[dict]:
+        """Compare facts across projects, generate cross-project insights."""
+        if len(project_paths) < 2:
+            return []
+
+        project_summaries = []
+        for path in project_paths:
+            facts = self.reader.read_facts(path)
+            facts.sort(key=lambda f: int(f.get("relevance_count", 0)), reverse=True)
+            top = facts[:max_facts_per]
+            summary = "\n".join(
+                f"  - [{f.get('category', 'general')}] {f.get('fact', '')[:150]}"
+                for f in top
+            )
+            project_summaries.append(f"### {Path(path).name}\n{summary}")
+
+        prompt = f"""Compare memory facts from these {len(project_paths)} projects:
+
+{"".join(project_summaries)}
+
+Identify cross-project patterns, shared conventions, dependencies, risks, and reuse opportunities.
+
+Return JSON array:
+[
+  {{
+    "type": "pattern|dependency|convention|risk|opportunity|drift",
+    "text": "description of the insight",
+    "projects": ["project_name_1", "project_name_2"],
+    "confidence": 0.0-1.0,
+    "tags": ["tag1"]
+  }}
+]"""
+
+        response = self.bridge.generate(prompt, system=_SYSTEM_PROMPT, max_tokens=2048, num_ctx=16384)
+        if not response:
+            return []
+
+        insights_raw = self._parse_json_array(response)
+        # Map project names back to paths
+        name_to_path = {Path(p).name: p for p in project_paths}
+
+        new_insights = []
+        for raw in insights_raw:
+            source_projects = [
+                name_to_path.get(n, n) for n in raw.get("projects", [])
+            ]
+            if len(source_projects) < 2:
+                continue
+
+            insight = self.cross_memory.add_insight(
+                text=raw.get("text", ""),
+                insight_type=raw.get("type", "pattern"),
+                source_projects=source_projects,
+                confidence=float(raw.get("confidence", 0.7)),
+                tags=raw.get("tags", []),
+            )
+            new_insights.append(insight)
+
+        return new_insights
+
+    def generate_cross_project_insights(
+        self,
+        project_paths: list[str],
+        federated_graph: dict | None = None,
+        max_pairs: int = 40,
+    ) -> list[dict]:
+        """Generate cross-project insights from the federated graph's cross_similar edges.
+
+        Feeds the LLM with concrete pairs of similar facts across projects so it can
+        produce typed insights (shared_convention, duplicated_fact, divergent_decision,
+        shared_bug_pattern). Falls back to find_cross_project_links if no graph given.
+        """
+        if not federated_graph:
+            return self.find_cross_project_links(project_paths)
+
+        nodes_by_id = {n["id"]: n for n in federated_graph.get("nodes", [])}
+        cross_edges = [
+            e for e in federated_graph.get("edges", [])
+            if e.get("scope") == "cross_similar"
+        ]
+        if not cross_edges:
+            return []
+
+        cross_edges.sort(key=lambda e: e.get("weight", 0), reverse=True)
+        pairs = []
+        for e in cross_edges[:max_pairs]:
+            a = nodes_by_id.get(e["src"])
+            b = nodes_by_id.get(e["dst"])
+            if not a or not b:
+                continue
+            pairs.append({
+                "a": {"project": a["project"], "category": a.get("category"),
+                      "text": a.get("text", ""), "local_id": a.get("local_id"),
+                      "project_path": a.get("project_path")},
+                "b": {"project": b["project"], "category": b.get("category"),
+                      "text": b.get("text", ""), "local_id": b.get("local_id"),
+                      "project_path": b.get("project_path")},
+                "similarity": e.get("weight"),
+            })
+        if not pairs:
+            return []
+
+        pair_lines = []
+        for i, p in enumerate(pairs):
+            pair_lines.append(
+                f"{i+1}. [{p['similarity']:.2f}] "
+                f"{p['a']['project']} ({p['a']['category']}): {p['a']['text'][:200]}\n"
+                f"   vs {p['b']['project']} ({p['b']['category']}): {p['b']['text'][:200]}"
+            )
+        prompt = (
+            f"These {len(pairs)} pairs of facts from different C3 projects scored "
+            f"as similar. Classify each into one of: shared_convention, duplicated_fact, "
+            f"divergent_decision, shared_bug_pattern, opportunity, drift. Merge pairs "
+            f"into cross-project insights when they reinforce the same point.\n\n"
+            + "\n".join(pair_lines)
+            + "\n\nReturn JSON array of insights:\n"
+            '[{"type": "<type>", "text": "<insight>", '
+            '"projects": ["<proj_slug>", ...], '
+            '"pair_indices": [<int>, ...], '
+            '"confidence": 0.0-1.0, "tags": ["..."]}]'
+        )
+
+        response = self.bridge.generate(prompt, system=_SYSTEM_PROMPT, max_tokens=2048, num_ctx=16384)
+        if not response:
+            return []
+        raw_list = self._parse_json_array(response)
+
+        slug_to_path = {n["project"]: n["project_path"] for n in federated_graph.get("nodes", [])}
+        new_insights: list[dict] = []
+        for raw in raw_list:
+            proj_slugs = raw.get("projects", [])
+            source_projects = [slug_to_path.get(s, s) for s in proj_slugs]
+            source_projects = [p for p in source_projects if p in project_paths]
+            if len(source_projects) < 2:
+                continue
+            source_fact_ids: dict[str, list[str]] = {}
+            for idx in raw.get("pair_indices", []):
+                try:
+                    p = pairs[int(idx) - 1]
+                except (ValueError, IndexError):
+                    continue
+                for side in ("a", "b"):
+                    path = p[side]["project_path"]
+                    fid = p[side]["local_id"]
+                    if path and fid:
+                        source_fact_ids.setdefault(path, []).append(fid)
+            insight = self.cross_memory.add_insight(
+                text=raw.get("text", ""),
+                insight_type=raw.get("type", "pattern"),
+                source_projects=source_projects,
+                source_fact_ids=source_fact_ids or None,
+                confidence=float(raw.get("confidence", 0.7)),
+                tags=raw.get("tags", []),
+            )
+            new_insights.append(insight)
+        return new_insights
+
+    def suggest_consolidation(self, project_path: str) -> list[dict]:
+        """Analyze a project's facts and suggest merge/archive actions."""
+        facts = self.reader.read_facts(project_path)
+        if len(facts) < 5:
+            return []
+
+        # Focus on potentially stale/duplicate facts
+        fact_texts = "\n".join(
+            f"[{f.get('id', '?')}] ({f.get('category', 'general')}) {f.get('fact', '')[:200]}"
+            for f in facts
+        )
+
+        prompt = f"""Review these {len(facts)} memory facts and suggest consolidation:
+
+{fact_texts}
+
+Find duplicates to merge and stale facts to archive.
+Return JSON array:
+[
+  {{
+    "action": "merge",
+    "fact_ids": ["id1", "id2"],
+    "survivor_id": "id1",
+    "merged_text": "combined fact text",
+    "reason": "why merge"
+  }},
+  {{
+    "action": "archive",
+    "fact_ids": ["id3"],
+    "reason": "why archive"
+  }}
+]"""
+
+        response = self.bridge.generate(prompt, system=_SYSTEM_PROMPT, max_tokens=2048, num_ctx=16384)
+        if not response:
+            return []
+
+        return self._parse_json_array(response)
+
+    def _parse_json(self, text: str) -> dict:
+        """Extract JSON object from LLM response."""
+        # Try direct parse
+        try:
+            return json.loads(text)
+        except Exception:
+            pass
+        # Try extracting from markdown fences
+        match = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
+        if match:
+            try:
+                return json.loads(match.group(1))
+            except Exception:
+                pass
+        # Try finding first { ... }
+        match = re.search(r"\{.*\}", text, re.DOTALL)
+        if match:
+            try:
+                return json.loads(match.group(0))
+            except Exception:
+                pass
+        return {}
+
+    def _parse_json_array(self, text: str) -> list[dict]:
+        """Extract JSON array from LLM response."""
+        try:
+            result = json.loads(text)
+            if isinstance(result, list):
+                return result
+        except Exception:
+            pass
+        match = re.search(r"```(?:json)?\s*\n?(.*?)\n?```", text, re.DOTALL)
+        if match:
+            try:
+                result = json.loads(match.group(1))
+                if isinstance(result, list):
+                    return result
+            except Exception:
+                pass
+        match = re.search(r"\[.*\]", text, re.DOTALL)
+        if match:
+            try:
+                result = json.loads(match.group(0))
+                if isinstance(result, list):
+                    return result
+            except Exception:
+                pass
+        return []

oracle/services/memory_reader.py

@@ -0,0 +1,106 @@
+"""Read-only access to per-project C3 memory stores."""
+
+import json
+import sys
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from services.memory_scorer import MemoryScorer  # noqa: E402
+
+
+class MemoryReader:
+    """Reads project .c3/facts/ files without writing."""
+
+    def __init__(self):
+        self._scorer = MemoryScorer()
+
+    def read_facts(self, project_path: str) -> list[dict]:
+        """Load all facts from a project's facts.json."""
+        facts_file = Path(project_path) / ".c3" / "facts" / "facts.json"
+        if not facts_file.is_file():
+            return []
+        try:
+            with open(facts_file, encoding="utf-8") as f:
+                data = json.load(f)
+            return data if isinstance(data, list) else []
+        except Exception:
+            return []
+
+    def read_graph(self, project_path: str) -> dict:
+        """Load memory graph edges."""
+        graph_file = Path(project_path) / ".c3" / "facts" / "memory_graph.json"
+        if not graph_file.is_file():
+            return {"edges": [], "adjacency": {}}
+        try:
+            with open(graph_file, encoding="utf-8") as f:
+                return json.load(f)
+        except Exception:
+            return {"edges": [], "adjacency": {}}
+
+    def read_fingerprints(self, project_path: str) -> list[dict]:
+        """Load session fingerprints."""
+        fp_file = Path(project_path) / ".c3" / "facts" / "session_fingerprints.json"
+        if not fp_file.is_file():
+            return []
+        try:
+            with open(fp_file, encoding="utf-8") as f:
+                data = json.load(f)
+            return data if isinstance(data, list) else []
+        except Exception:
+            return []
+
+    def get_fact_stats(self, project_path: str) -> dict:
+        """Compute summary statistics for a project's facts."""
+        facts = self.read_facts(project_path)
+        if not facts:
+            return {"total": 0, "by_category": {}, "by_tier": {}, "by_lifecycle": {}}
+
+        by_category: dict[str, int] = {}
+        by_lifecycle: dict[str, int] = {}
+        for f in facts:
+            cat = f.get("category", "general")
+            by_category[cat] = by_category.get(cat, 0) + 1
+            lc = f.get("lifecycle", "active")
+            by_lifecycle[lc] = by_lifecycle.get(lc, 0) + 1
+
+        tiers = self._scorer.tier_partition(facts)
+        by_tier = {tier: len(tier_facts) for tier, tier_facts in tiers.items()}
+
+        return {
+            "total": len(facts),
+            "by_category": by_category,
+            "by_tier": by_tier,
+            "by_lifecycle": by_lifecycle,
+        }
+
+    def get_graph_stats(self, project_path: str) -> dict:
+        """Compute graph statistics."""
+        graph = self.read_graph(project_path)
+        edges = graph.get("edges", [])
+        adjacency = graph.get("adjacency", {})
+
+        nodes = set()
+        edge_types: dict[str, int] = {}
+        for edge in edges:
+            nodes.add(edge.get("src", ""))
+            nodes.add(edge.get("dst", ""))
+            et = edge.get("type", "unknown")
+            edge_types[et] = edge_types.get(et, 0) + 1
+
+        # Count orphaned edges (referencing non-existent facts)
+        facts = self.read_facts(project_path)
+        fact_ids = {f.get("id") for f in facts if f.get("id")}
+        orphaned = sum(
+            1 for e in edges
+            if e.get("src") not in fact_ids or e.get("dst") not in fact_ids
+        )
+
+        return {
+            "total_edges": len(edges),
+            "total_nodes": len(nodes),
+            "edge_types": edge_types,
+            "orphaned_edges": orphaned,
+        }

oracle/services/memory_writer.py

@@ -0,0 +1,182 @@
+"""Handles approved write-backs to project .c3/facts/."""
+
+import json
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from oracle.config import ORACLE_DIR
+
+_SUGGESTIONS_FILE = ORACLE_DIR / "suggestions.json"
+
+
+def _load_suggestions() -> list[dict]:
+    try:
+        if _SUGGESTIONS_FILE.is_file():
+            with open(_SUGGESTIONS_FILE, encoding="utf-8") as f:
+                return json.load(f)
+    except Exception:
+        pass
+    return []
+
+
+def _save_suggestions(suggestions: list[dict]):
+    ORACLE_DIR.mkdir(parents=True, exist_ok=True)
+    with open(_SUGGESTIONS_FILE, "w", encoding="utf-8") as f:
+        json.dump(suggestions, f, indent=2)
+
+
+class MemoryWriter:
+    """Manages suggestion queue and approved write-backs to project memory."""
+
+    def suggest(self, project_path: str, suggestion_type: str, data: dict) -> dict:
+        """Create a pending suggestion for a project.
+
+        suggestion_type: 'merge_facts', 'archive_facts', 'add_fact'
+        data: type-specific payload
+        """
+        suggestions = _load_suggestions()
+        suggestion = {
+            "id": f"sug_{uuid.uuid4().hex[:12]}",
+            "project_path": project_path,
+            "type": suggestion_type,
+            "data": data,
+            "status": "pending",
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "resolved_at": None,
+        }
+        suggestions.append(suggestion)
+        _save_suggestions(suggestions)
+        return suggestion
+
+    def list_pending(self, project_path: str | None = None) -> list[dict]:
+        """Return pending suggestions, optionally filtered by project."""
+        suggestions = _load_suggestions()
+        pending = [s for s in suggestions if s.get("status") == "pending"]
+        if project_path:
+            pending = [s for s in pending if s.get("project_path") == project_path]
+        return pending
+
+    def approve_suggestion(self, suggestion_id: str) -> dict:
+        """Execute an approved suggestion and write to project memory."""
+        suggestions = _load_suggestions()
+        target = None
+        for s in suggestions:
+            if s["id"] == suggestion_id and s["status"] == "pending":
+                target = s
+                break
+        if not target:
+            return {"error": "Suggestion not found or already resolved"}
+
+        result = self._execute(target)
+        target["status"] = "approved"
+        target["resolved_at"] = datetime.now(timezone.utc).isoformat()
+        target["result"] = result
+        _save_suggestions(suggestions)
+        return {"approved": True, "id": suggestion_id, "result": result}
+
+    def dismiss_suggestion(self, suggestion_id: str) -> dict:
+        """Mark a suggestion as dismissed."""
+        suggestions = _load_suggestions()
+        for s in suggestions:
+            if s["id"] == suggestion_id and s["status"] == "pending":
+                s["status"] = "dismissed"
+                s["resolved_at"] = datetime.now(timezone.utc).isoformat()
+                _save_suggestions(suggestions)
+                return {"dismissed": True, "id": suggestion_id}
+        return {"error": "Suggestion not found or already resolved"}
+
+    def _execute(self, suggestion: dict) -> dict:
+        """Execute a suggestion write-back."""
+        stype = suggestion["type"]
+        project_path = suggestion["project_path"]
+        data = suggestion["data"]
+
+        if stype == "archive_facts":
+            return self._archive_facts(project_path, data.get("fact_ids", []))
+        elif stype == "merge_facts":
+            return self._merge_facts(project_path, data)
+        elif stype == "add_fact":
+            return self._add_fact(project_path, data)
+        return {"error": f"Unknown suggestion type: {stype}"}
+
+    def _load_project_facts(self, project_path: str) -> tuple[list[dict], Path]:
+        facts_file = Path(project_path) / ".c3" / "facts" / "facts.json"
+        if not facts_file.is_file():
+            return [], facts_file
+        try:
+            with open(facts_file, encoding="utf-8") as f:
+                return json.load(f), facts_file
+        except Exception:
+            return [], facts_file
+
+    def _save_project_facts(self, facts: list[dict], facts_file: Path):
+        facts_file.parent.mkdir(parents=True, exist_ok=True)
+        with open(facts_file, "w", encoding="utf-8") as f:
+            json.dump(facts, f, indent=2)
+
+    def _archive_facts(self, project_path: str, fact_ids: list[str]) -> dict:
+        """Set lifecycle=archived on specified facts."""
+        facts, fpath = self._load_project_facts(project_path)
+        archived = 0
+        for fact in facts:
+            if fact.get("id") in fact_ids:
+                fact["lifecycle"] = "archived"
+                archived += 1
+        self._save_project_facts(facts, fpath)
+        return {"archived": archived}
+
+    def _merge_facts(self, project_path: str, data: dict) -> dict:
+        """Merge duplicate facts: keep survivor, archive others."""
+        facts, fpath = self._load_project_facts(project_path)
+        survivor_id = data.get("survivor_id")
+        merge_ids = set(data.get("merge_ids", []))
+
+        survivor = None
+        for fact in facts:
+            if fact.get("id") == survivor_id:
+                survivor = fact
+                break
+        if not survivor:
+            return {"error": f"Survivor fact {survivor_id} not found"}
+
+        merged_count = 0
+        for fact in facts:
+            if fact.get("id") in merge_ids and fact.get("id") != survivor_id:
+                fact["lifecycle"] = "archived"
+                survivor["relevance_count"] = (
+                    int(survivor.get("relevance_count", 0))
+                    + int(fact.get("relevance_count", 0))
+                )
+                merged_count += 1
+
+        if data.get("merged_text"):
+            survivor["fact"] = data["merged_text"]
+
+        self._save_project_facts(facts, fpath)
+        return {"merged": merged_count, "survivor_id": survivor_id}
+
+    def _add_fact(self, project_path: str, data: dict) -> dict:
+        """Add a new fact (e.g., cross-project insight) to a project."""
+        facts, fpath = self._load_project_facts(project_path)
+        fact_id = uuid.uuid4().hex[:12]
+        now = datetime.now(timezone.utc).isoformat()
+        entry = {
+            "id": fact_id,
+            "fact": data.get("fact", ""),
+            "category": data.get("category", "oracle"),
+            "source_session": "oracle",
+            "timestamp": now,
+            "last_accessed_at": None,
+            "relevance_count": 0,
+            "confidence": float(data.get("confidence", 0.8)),
+            "source_quality": "oracle",
+            "lifecycle": "active",
+            "vector_id": fact_id,
+            "recall_sessions": [],
+            "confirmation_count": 0,
+            "contradiction_count": 0,
+        }
+        facts.append(entry)
+        self._save_project_facts(facts, fpath)
+        return {"added": True, "id": fact_id}