deja-cli 0.1.0__py3-none-any.whl

deja/config.py

@@ -0,0 +1,127 @@
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Optional
+
+import yaml
+from pydantic import BaseModel, field_validator
+
+
+def _substitute_env(value: str) -> str:
+    """Substitute ${ENV_VAR} patterns with environment variable values."""
+    def replacer(match: re.Match) -> str:
+        var_name = match.group(1)
+        return os.environ.get(var_name, match.group(0))
+
+    return re.sub(r"\$\{([^}]+)\}", replacer, value)
+
+
+def _expand_paths(data: dict) -> dict:
+    """Recursively expand ~ in string values."""
+    result = {}
+    for key, value in data.items():
+        if isinstance(value, dict):
+            result[key] = _expand_paths(value)
+        elif isinstance(value, str):
+            result[key] = _substitute_env(value)
+        else:
+            result[key] = value
+    return result
+
+
+class LLMProviderConfig(BaseModel):
+    provider: str = "ollama"
+    model: str = "qwen2.5:3b"
+    base_url: str = "http://localhost:11434"
+    api_key: Optional[str] = None
+
+    @field_validator("api_key", mode="before")
+    @classmethod
+    def expand_env_vars(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        return _substitute_env(v)
+
+
+class LLMConfig(BaseModel):
+    extraction: LLMProviderConfig = LLMProviderConfig(provider="none")
+    reflection: LLMProviderConfig = LLMProviderConfig(provider="none")
+    fallback: LLMProviderConfig = LLMProviderConfig(
+        provider="anthropic",
+        model="claude-haiku-4-5-20251001",
+        api_key="${ANTHROPIC_API_KEY}",
+    )
+
+
+class StoreConfig(BaseModel):
+    path: str = "~/.deja/store/memories.db"
+    vault_path: str = "~/.deja/store/vault/"
+
+    @property
+    def db_path(self) -> Path:
+        return Path(self.path).expanduser()
+
+    @property
+    def vault_dir(self) -> Path:
+        return Path(self.vault_path).expanduser()
+
+
+class ReflectionConfig(BaseModel):
+    observer_trigger_tokens: int = 30000
+    reflector_trigger_tokens: int = 40000
+    kg_merge_schedule: str = "0 2 * * *"
+    confidence_archive_threshold: float = 0.3
+    # agent memories (gotcha, decision, progress, pattern) decay at this rate.
+    # Operational knowledge goes stale; 0.05/week means a memory hits the 0.3
+    # archive threshold in ~14 weeks without re-confirmation.
+    confidence_decay_per_week: float = 0.05
+    # user memories (preferences, habits) decay ~5x slower. Personal style
+    # preferences don't go stale the way project-specific gotchas do.
+    user_confidence_decay_per_week: float = 0.01
+    min_project_pattern_count: int = 2
+
+
+class WatchersConfig(BaseModel):
+    claude_code: bool = True
+    gemini_cli: bool = False
+    codex_cli: bool = False
+    aider: bool = False
+    debounce_seconds: int = 30
+
+
+class EmbeddingConfig(BaseModel):
+    provider: str = "none"  # none | ollama
+    model: str = "nomic-embed-text"
+    base_url: str = "http://localhost:11434"
+
+
+class Config(BaseModel):
+    llm: LLMConfig = LLMConfig()
+    store: StoreConfig = StoreConfig()
+    reflection: ReflectionConfig = ReflectionConfig()
+    watchers: WatchersConfig = WatchersConfig()
+    embedding: EmbeddingConfig = EmbeddingConfig()
+
+
+def load_config(path: Optional[Path] = None) -> Config:
+    """Load config from path, falling back to ~/.deja/config.yaml,
+    then to the bundled default."""
+    candidates = []
+    if path:
+        candidates.append(Path(path).expanduser())
+    candidates.append(Path("~/.deja/config.yaml").expanduser())
+
+    # Bundled default
+    default_path = Path(__file__).parent.parent / "config" / "default.yaml"
+    candidates.append(default_path)
+
+    for candidate in candidates:
+        if candidate.exists():
+            with open(candidate) as f:
+                raw = yaml.safe_load(f) or {}
+            raw = _expand_paths(raw)
+            return Config.model_validate(raw)
+
+    return Config()

deja/core/extractor.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import sys
+from typing import Optional
+
+from deja.llm.base import LLMAdapter
+
+EXTRACTION_SYSTEM = """You are a memory extraction system for a software engineer.
+Given a coding session transcript or summary, extract ONLY memories
+that would be genuinely useful in a future session.
+
+Be ruthless — most session content is NOT worth remembering.
+Only extract things that are:
+- Non-obvious (not derivable from reading the codebase)
+- Reusable (would apply in future sessions)
+- Important (would cause problems if forgotten)
+
+Memory types:
+- preference: how the user likes to code (style, tools, patterns)
+- pattern: reusable solution / architectural approach that applies across contexts ("knowing what works")
+- decision: non-obvious architectural choice with reasoning
+- gotcha: bug, trap, or non-obvious issue to avoid
+- progress: current state of in-progress work
+- procedure: reusable ordered steps for a recurring class of work ("knowing how to execute"). Keep thin: numbered steps + tool hints + exit criteria only. Do NOT inline gotchas or decisions — save those as separate memory types.
+
+Category:
+- user: personal preferences and habits (applies across all projects)
+- agent: operational knowledge discovered while doing work
+
+Domain (optional, coarse routing tag — use for procedure type mainly):
+- debug | build | test | deploy | research
+
+Output ONLY valid JSON:
+{
+  "memories": [
+    {
+      "type": "preference|pattern|decision|gotcha|progress|procedure",
+      "category": "user|agent",
+      "content": "concise, self-contained fact. 1-2 sentences max for most types. For procedure: one-line description followed by numbered steps, tool hints, and exit criteria.",
+      "scope": "global|project",
+      "project": "project_name or null if global",
+      "confidence": 0.0-1.0,
+      "domain": "debug|build|test|deploy|research|null"
+    }
+  ]
+}
+
+If nothing is worth remembering, return: {"memories": []}"""
+
+EXTRACTION_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "memories": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "type": {
+                        "type": "string",
+                        "enum": ["preference", "pattern", "decision", "gotcha", "progress", "procedure"],
+                    },
+                    "category": {"type": "string", "enum": ["user", "agent"]},
+                    "content": {"type": "string"},
+                    "scope": {"type": "string", "enum": ["global", "project"]},
+                    "project": {"type": ["string", "null"]},
+                    "confidence": {"type": "number"},
+                    "domain": {"type": ["string", "null"]},
+                },
+                "required": ["type", "category", "content", "scope", "confidence"],
+            },
+        }
+    },
+    "required": ["memories"],
+}
+
+
+async def extract_memories(
+    transcript: str,
+    project: str,
+    source: str,
+    adapter: LLMAdapter,
+) -> list[dict]:
+    """Extract memories from a session transcript or summary.
+
+    Returns list of memory dicts ready to pass to store.save().
+    """
+    if not transcript.strip():
+        return []
+
+    user_prompt = f"Session transcript/summary to extract memories from:\n\n{transcript}"
+
+    try:
+        result = await adapter.complete_structured(
+            system=EXTRACTION_SYSTEM,
+            user=user_prompt,
+            schema=EXTRACTION_SCHEMA,
+        )
+    except Exception as e:
+        print(f"[deja] Extraction LLM error: {e}", file=sys.stderr)
+        return []
+
+    memories = result.get("memories", [])
+    if not isinstance(memories, list):
+        return []
+
+    output = []
+    for mem in memories:
+        if not isinstance(mem, dict):
+            continue
+        if not mem.get("content") or not mem.get("type"):
+            continue
+
+        # Normalize scope: if scope is "project" but no project given, use the provided project
+        scope = mem.get("scope", "global")
+        mem_project = mem.get("project") or (project if scope == "project" else None)
+        if scope == "project" and mem_project:
+            scope_value = f"project:{mem_project}"
+        else:
+            scope_value = "global"
+            mem_project = None
+
+        output.append(
+            {
+                "type": mem["type"],
+                "category": mem.get("category", "agent"),
+                "content": mem["content"],
+                "scope": scope_value,
+                "project": mem_project,
+                "source": source,
+                "confidence": float(mem.get("confidence", 0.8)),
+                "domain": mem.get("domain"),
+            }
+        )
+
+    return output

deja/core/reflection.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from typing import Optional
+
+from deja.config import ReflectionConfig
+from deja.core.store import MemoryStore
+from deja.llm.base import LLMAdapter
+
+OBSERVER_SYSTEM = """You are a memory compressor for a software engineer's coding knowledge base.
+Given these recent memories, extract the key observations worth preserving long-term.
+
+Discard:
+- Obvious facts derivable from the codebase
+- Task-specific one-offs that won't recur
+- Superseded progress updates (if newer progress exists)
+
+Keep:
+- Patterns and gotchas
+- Architectural decisions and their reasoning
+- User preferences and working style
+- Ongoing work that is not yet complete
+- Procedures (reusable step sequences) — keep steps thin
+
+For procedure memories:
+- Preserve only: numbered steps, tool hints, exit criteria
+- Strip any inline gotchas or decisions — those belong in separate memories
+- Merge near-duplicate procedures rather than keeping both
+
+Output concise observations in plain text, one per line.
+Each observation should be self-contained and useful without context.
+
+After the observations, if any memories would benefit from metadata improvements, append a
+MEMORY_UPDATES block with a JSON array of suggested changes. Only update what you are
+confident about — omit the block entirely if no improvements are needed.
+
+Rules for memory_updates:
+- Add a trigger only if the memory is a gotcha clearly tied to a specific command or action
+  boundary (e.g. kubectl apply, alembic upgrade, terraform apply, git push --force).
+  Trigger phrases must be commands the agent would literally type, comma-separated.
+- Change type only if clearly wrong (e.g. saved as pattern but describes a specific
+  command failure → gotcha). Be conservative.
+- Never update preferences, decisions, or progress entries.
+- If the memory already has a trigger shown in [trigger:...], skip it.
+
+Format (append at the end, after all observations):
+
+MEMORY_UPDATES:
+[
+  {"id": "01JKB...", "trigger": "kubectl apply, helm upgrade"},
+  {"id": "01JKC...", "type": "gotcha", "trigger": "alembic upgrade"}
+]"""
+
+REFLECTOR_SYSTEM = """Given this observation log, identify which observations are:
+- Superseded by a newer observation about the same topic
+- Redundant (same fact stated multiple ways)
+- Resolved (a gotcha that was fixed, progress that completed)
+
+Remove or merge superseded/redundant/resolved observations.
+For merged observations, keep the most recent and accurate version.
+
+Output the condensed observation log only. Plain text, one observation per line.
+Do not add commentary. Do not add headers. Just the observations."""
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _parse_observer_response(text: str) -> tuple[str, list[dict]]:
+    """Split Observer LLM response into (observations_text, memory_updates).
+
+    The LLM appends a MEMORY_UPDATES: JSON block after the plain-text observations.
+    If the block is absent or malformed, returns the full text as observations and
+    an empty updates list — observations always take priority.
+    """
+    marker = "MEMORY_UPDATES:"
+    idx = text.find(marker)
+    if idx == -1:
+        return text, []
+
+    obs_part = text[:idx].strip()
+    json_part = text[idx + len(marker):].strip()
+
+    # Extract JSON array — tolerate surrounding whitespace or markdown fences
+    json_match = re.search(r"\[.*\]", json_part, re.DOTALL)
+    if not json_match:
+        return obs_part, []
+
+    try:
+        updates = json.loads(json_match.group())
+        if not isinstance(updates, list):
+            return obs_part, []
+        return obs_part, [u for u in updates if isinstance(u, dict) and "id" in u]
+    except json.JSONDecodeError:
+        return obs_part, []
+
+
+def _format_memories_for_prompt(memories: list[dict]) -> str:
+    lines = []
+    for m in memories:
+        label = m.get("project") or "global"
+        trigger_str = f" [trigger:{m['trigger']}]" if m.get("trigger") else ""
+        lines.append(
+            f"[{m['type']}] [conf:{m['confidence']:.2f}] [{label}] [id:{m['id']}]{trigger_str}\n"
+            f"{m['content']}"
+        )
+    return "\n\n".join(lines)
+
+
+def _format_observations_for_prompt(observations: list[dict]) -> str:
+    return "\n".join(o["content"] for o in observations)
+
+
+class ReflectionEngine:
+    """Compresses, deduplicates, decays, and promotes memories over time.
+
+    Two reflection modes
+    --------------------
+    LLM mode  — uses the configured ``reflection`` LLM (Ollama / Anthropic).
+                Triggered automatically by token-count thresholds or manually
+                via ``deja reflect``.
+
+    Agent mode — no extra LLM call. The active coding agent (Claude Code,
+                 Codex, Gemini CLI) reads the output of ``agent_mode_prompt()``
+                 and executes ``deja archive / invalidate / save`` commands
+                 directly. Zero additional API cost — the agent is already
+                 being billed for the session.
+    """
+
+    def __init__(
+        self,
+        store: MemoryStore,
+        config: ReflectionConfig,
+        adapter: Optional[LLMAdapter] = None,
+    ) -> None:
+        self.store = store
+        self.config = config
+        self.adapter = adapter
+
+    # ── LLM-driven compression ─────────────────────────────────────────────
+
+    async def run_observer(self, project: Optional[str] = None) -> int:
+        """Compress memories into observations via LLM. Returns observations created."""
+        if not self.adapter:
+            raise RuntimeError(
+                "No LLM adapter configured for reflection. "
+                "Set reflection.provider in ~/.deja/config.yaml or use --agent-mode."
+            )
+
+        meta = await self.store.get_reflection_meta(project)
+        last_run = meta.get("last_observer_at") if meta else None
+
+        memories = await self.store.list_for_reflection(project, since=last_run)
+        if not memories:
+            return 0
+
+        token_estimate = sum(len(m["content"].split()) * 2 for m in memories)
+        if token_estimate < 100:
+            return 0
+
+        user_prompt = (
+            "Recent memories to compress into observations:\n\n"
+            + _format_memories_for_prompt(memories)
+        )
+
+        try:
+            response = await self.adapter.complete(system=OBSERVER_SYSTEM, user=user_prompt)
+            observations_text = response.content.strip()
+        except Exception as e:
+            print(f"[deja] Observer LLM error: {e}", file=sys.stderr)
+            return 0
+
+        # Split on MEMORY_UPDATES: block — observations come first, JSON updates after
+        obs_part, memory_updates = _parse_observer_response(observations_text)
+
+        new_obs = [line.strip() for line in obs_part.splitlines() if line.strip()]
+        for obs_text in new_obs:
+            await self.store.save_observation(project, obs_text)
+
+        # Apply memory metadata updates suggested by the Observer
+        applied = 0
+        for update in memory_updates:
+            mem_id = update.get("id")
+            if not mem_id:
+                continue
+            fields = {k: v for k, v in update.items() if k in ("trigger", "type")}
+            if fields:
+                ok = await self.store.update_memory(mem_id, fields)
+                if ok:
+                    applied += 1
+        if applied:
+            print(f"[deja] Observer applied {applied} memory metadata update(s).", file=sys.stderr)
+
+        await self.store.set_reflection_meta(project, last_observer_at=_now_iso())
+        return len(new_obs)
+
+    async def run_reflector(self, project: Optional[str] = None) -> int:
+        """Condense the observation log via LLM. Returns reduction in observation count."""
+        if not self.adapter:
+            raise RuntimeError(
+                "No LLM adapter configured for reflection. "
+                "Set reflection.provider in ~/.deja/config.yaml or use --agent-mode."
+            )
+
+        observations = await self.store.list_observations(project)
+        if len(observations) < 3:
+            return 0
+
+        user_prompt = (
+            "Observation log to condense:\n\n"
+            + _format_observations_for_prompt(observations)
+        )
+
+        try:
+            response = await self.adapter.complete(system=REFLECTOR_SYSTEM, user=user_prompt)
+            condensed_text = response.content.strip()
+        except Exception as e:
+            print(f"[deja] Reflector LLM error: {e}", file=sys.stderr)
+            return 0
+
+        condensed = [line.strip() for line in condensed_text.splitlines() if line.strip()]
+        original_count = len(observations)
+        await self.store.replace_observations(project, condensed)
+
+        # Surviving compression is a confirmation signal — increment reuse_count
+        # for all active memories. We can't map observations back to specific
+        # memories, so we treat all active memories as "surviving" this pass.
+        await self.store.increment_reuse_count(project)
+
+        await self.store.set_reflection_meta(project, last_reflector_at=_now_iso())
+        return original_count - len(condensed)
+
+    # ── No-LLM maintenance passes ──────────────────────────────────────────
+
+    async def run_decay(self) -> int:
+        """Reduce confidence on memories not confirmed recently.
+
+        Two rates are applied:
+        - category='agent': config.confidence_decay_per_week (default 0.05/week)
+          Operational knowledge (gotchas, decisions, progress) goes stale.
+        - category='user': config.user_confidence_decay_per_week (default 0.01/week)
+          Preferences and habits are stable; they decay ~5x slower.
+        """
+        count = await self.store.decay_unconfirmed(
+            days_threshold=14,
+            decay_per_week=self.config.confidence_decay_per_week,
+            user_decay_per_week=self.config.user_confidence_decay_per_week,
+        )
+        await self.store.set_reflection_meta(None, last_decay_at=_now_iso())
+        return count
+
+    async def run_promote(self) -> int:
+        """Promote patterns seen in 2+ projects to global scope."""
+        count = await self.store.promote_patterns_to_global(
+            self.config.min_project_pattern_count
+        )
+        await self.store.set_reflection_meta(None, last_promote_at=_now_iso())
+        return count
+
+    async def run_archive(self) -> int:
+        """Archive memories below confidence threshold."""
+        count = await self.store.archive_below_threshold(
+            self.config.confidence_archive_threshold
+        )
+        await self.store.set_reflection_meta(None, last_archive_at=_now_iso())
+        return count
+
+    # ── Agent mode ─────────────────────────────────────────────────────────
+
+    async def agent_mode_prompt(self, project: Optional[str] = None) -> str:
+        """Return a formatted memory dump with instructions for the coding agent to reflect.
+
+        The agent (Claude Code, Codex, Gemini CLI) reads this output and executes
+        deja commands directly — no separate LLM API call needed.
+        """
+        memories = await self.store.list_for_reflection(project)
+        project_label = project or "global"
+        project_flag = f" --project {project}" if project else ""
+
+        if not memories:
+            return f"No active memories found for project '{project_label}'. Nothing to reflect on."
+
+        lines = [
+            f"You are acting as a memory reflector for project '{project_label}'.",
+            "",
+            f"Review the {len(memories)} active memories below and identify any that should be:",
+            f"  1. Archived (stale, no longer relevant):",
+            f"       deja archive <id>",
+            f"  2. Invalidated (contradicted by newer information):",
+            f"       deja invalidate <id>",
+            f"  3. Consolidated (two memories express the same thing):",
+            f"       deja archive <id1>",
+            f"       deja archive <id2>",
+            f'       deja save "<condensed content>" --type <type>{project_flag}',
+            f"  4. Trigger-tagged (gotcha clearly tied to a specific command but has no trigger):",
+            f'       deja update <id> --trigger "cmd1, cmd2"',
+            f"       Use this for gotchas about what to do right before/after a specific command.",
+            f"       Example triggers: 'kubectl apply', 'alembic upgrade', 'terraform apply'.",
+            f"       Only tag gotchas — not preferences, decisions, or progress.",
+            f"  5. Reclassified (saved as the wrong type — e.g. pattern that is really a gotcha):",
+            f"       deja update <id> --type gotcha",
+            "",
+            "Be conservative — only act on memories that clearly need attention.",
+            "For trigger tagging: if a gotcha is already tagged (shown as [trigger:...]), skip it.",
+            "If everything looks good, do nothing.",
+            "",
+            "--- MEMORIES ---",
+            "",
+        ]
+
+        for m in memories:
+            scope_label = f"project:{m['project']}" if m.get("project") else "global"
+            trigger_str = f" [trigger:{m['trigger']}]" if m.get("trigger") else ""
+            lines.append(
+                f"[{m['type']}] [conf:{m['confidence']:.2f}] [scope:{scope_label}]"
+                f" [ID:{m['id']}]{trigger_str}"
+            )
+            lines.append(m["content"])
+            lines.append("")
+
+        lines.append("--- END MEMORIES ---")
+        return "\n".join(lines)
+
+    # ── Full pass + auto-trigger ────────────────────────────────────────────
+
+    async def run_full(self, project: Optional[str] = None) -> dict:
+        """Full reflection pass: observer → reflector → decay → promote → archive."""
+        results: dict = {}
+        if self.adapter:
+            results["observer"] = await self.run_observer(project)
+            results["reflector"] = await self.run_reflector(project)
+        else:
+            results["observer"] = 0
+            results["reflector"] = 0
+        results["decay"] = await self.run_decay()
+        results["promote"] = await self.run_promote()
+        results["archive"] = await self.run_archive()
+        return results
+
+    async def check_and_trigger(self, project: Optional[str] = None) -> None:
+        """Check token thresholds and auto-trigger observer/reflector if exceeded."""
+        if not self.adapter:
+            return
+
+        meta = await self.store.get_reflection_meta(project)
+        last_observer_at = meta.get("last_observer_at") if meta else None
+
+        memories = await self.store.list_for_reflection(project, since=last_observer_at)
+        token_count = sum(len(m["content"].split()) * 2 for m in memories)
+
+        if token_count >= self.config.observer_trigger_tokens:
+            n = await self.run_observer(project)
+            print(f"[deja] Auto-observer triggered: {n} observations created.", file=sys.stderr)
+
+        observations = await self.store.list_observations(project)
+        obs_tokens = sum(len(o["content"].split()) * 2 for o in observations)
+
+        if obs_tokens >= self.config.reflector_trigger_tokens:
+            n = await self.run_reflector(project)
+            print(f"[deja] Auto-reflector triggered: {n} observations reduced.", file=sys.stderr)