oghma 0.0.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

oghma/daemon.py

@@ -0,0 +1,198 @@
+import logging
+import os
+import signal
+import sys
+import time
+from pathlib import Path
+
+from oghma.config import Config
+from oghma.extractor import Extractor
+from oghma.parsers import get_parser_for_file
+from oghma.storage import Storage
+from oghma.watcher import Watcher
+
+logger = logging.getLogger(__name__)
+
+
+class Daemon:
+    """Main daemon for Oghma memory extraction."""
+
+    def __init__(self, config: Config):
+        self.config = config
+        self.storage = Storage(config=config)
+        self.watcher = Watcher(config, self.storage)
+        self.extractor = Extractor(config)
+        self._setup_logging()
+        self._running = False
+
+    def _setup_logging(self) -> None:
+        """Setup logging configuration."""
+        log_file = self.config["daemon"]["log_file"]
+        log_level = self.config["daemon"]["log_level"]
+        Path(log_file).parent.mkdir(parents=True, exist_ok=True)
+
+        logging.basicConfig(
+            level=getattr(logging, log_level),
+            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            handlers=[
+                logging.FileHandler(log_file),
+                logging.StreamHandler(sys.stdout),
+            ],
+        )
+
+    def _signal_handler(self, signum: int, frame) -> None:
+        """Handle shutdown signals gracefully."""
+        logger.info(f"Received signal {signum}, shutting down...")
+        self._running = False
+
+    def start(self) -> None:
+        """Start the daemon main loop."""
+        signal.signal(signal.SIGINT, self._signal_handler)
+        signal.signal(signal.SIGTERM, self._signal_handler)
+
+        pid_file = self.config["daemon"]["pid_file"]
+        self._write_pid_file(pid_file)
+
+        try:
+            self._running = True
+            poll_interval = self.config["daemon"]["poll_interval"]
+
+            logger.info("Oghma daemon started")
+            logger.info(f"Poll interval: {poll_interval} seconds")
+
+            while self._running:
+                try:
+                    self._run_cycle()
+                except Exception as e:
+                    logger.error(f"Error in extraction cycle: {e}", exc_info=True)
+
+                for _ in range(poll_interval):
+                    if not self._running:
+                        break
+                    time.sleep(1)
+
+        finally:
+            self._cleanup(pid_file)
+
+    def _run_cycle(self) -> None:
+        """Run one extraction cycle."""
+        logger.debug("Starting extraction cycle")
+        changed_files = self.watcher.get_changed_files()
+
+        if not changed_files:
+            logger.debug("No changed files found")
+            return
+
+        logger.info(f"Processing {len(changed_files)} changed files")
+
+        for file_path in changed_files:
+            self._process_file(file_path)
+
+    def _process_file(self, file_path: Path) -> None:
+        """Process a single file: parse, extract, and save memories."""
+        logger.info(f"Processing file: {file_path}")
+
+        try:
+            parser = get_parser_for_file(file_path)
+            if not parser:
+                logger.warning(f"No parser found for {file_path}")
+                return
+
+            messages = parser.parse(file_path)
+
+            if not self.watcher.should_process(messages):
+                logger.debug(f"Skipping {file_path}: not enough messages")
+                return
+
+            source_tool = self._get_tool_name(file_path)
+            memories = self.extractor.extract(messages, source_tool)
+
+            mtime = file_path.stat().st_mtime
+            size = file_path.stat().st_size
+            source_session = self._get_session_id(file_path)
+
+            for memory in memories:
+                self.storage.add_memory(
+                    content=memory.content,
+                    category=memory.category,
+                    source_tool=source_tool,
+                    source_file=str(file_path),
+                    source_session=source_session,
+                    confidence=memory.confidence,
+                )
+
+            self.storage.update_extraction_state(str(file_path), mtime, size, len(messages))
+
+            self.storage.log_extraction(
+                source_path=str(file_path),
+                memories_extracted=len(memories),
+            )
+
+            logger.info(f"Extracted {len(memories)} memories from {file_path}")
+
+        except Exception as e:
+            logger.error(f"Failed to process {file_path}: {e}", exc_info=True)
+            self.storage.log_extraction(source_path=str(file_path), error=str(e))
+
+    def _get_tool_name(self, file_path: Path) -> str:
+        """Extract tool name from file path."""
+        path_str = str(file_path)
+
+        if ".claude" in path_str:
+            return "claude_code"
+        elif ".codex" in path_str:
+            return "codex"
+        elif ".openclaw" in path_str:
+            return "openclaw"
+        elif ".local/share/opencode" in path_str or "opencode" in path_str:
+            return "opencode"
+        else:
+            return "unknown"
+
+    def _get_session_id(self, file_path: Path) -> str | None:
+        """Extract session ID from file path if possible."""
+        parts = file_path.parts
+        for part in reversed(parts):
+            if part.startswith("ses_") or part.startswith("rollout-"):
+                return part
+        return None
+
+    def _write_pid_file(self, pid_file: str) -> None:
+        """Write current PID to lock file."""
+        Path(pid_file).parent.mkdir(parents=True, exist_ok=True)
+        with open(pid_file, "w") as f:
+            f.write(str(os.getpid()))
+
+    def _cleanup(self, pid_file: str) -> None:
+        """Cleanup on shutdown."""
+        logger.info("Oghma daemon stopped")
+        pid_path = Path(pid_file)
+        if pid_path.exists():
+            pid_path.unlink()
+
+
+def get_daemon_pid(pid_file: str) -> int | None:
+    """Read PID from lock file, check if process is running."""
+    pid_path = Path(pid_file)
+
+    if not pid_path.exists():
+        return None
+
+    try:
+        with open(pid_path) as f:
+            pid_str = f.read().strip()
+            pid = int(pid_str)
+
+        if pid > 0:
+            try:
+                os.kill(pid, 0)
+                return pid
+            except OSError:
+                pid_path.unlink()
+                return None
+
+    except (OSError, ValueError):
+        if pid_path.exists():
+            pid_path.unlink()
+
+    return None

oghma/embedder.py

@@ -0,0 +1,107 @@
+import os
+import time
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from openai import APIError, OpenAI
+
+
+@dataclass
+class EmbedConfig:
+    provider: str = "openai"
+    model: str = "text-embedding-3-small"
+    dimensions: int = 1536
+    batch_size: int = 100
+    rate_limit_delay: float = 0.1
+    max_retries: int = 3
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: Mapping[str, Any] | None = None,
+        **overrides: Any,
+    ) -> "EmbedConfig":
+        values = {
+            "provider": cls.provider,
+            "model": cls.model,
+            "dimensions": cls.dimensions,
+            "batch_size": cls.batch_size,
+            "rate_limit_delay": cls.rate_limit_delay,
+            "max_retries": cls.max_retries,
+        }
+        if data:
+            for key in values:
+                if key in data:
+                    values[key] = data[key]
+        for key, value in overrides.items():
+            if key in values and value is not None:
+                values[key] = value
+        return cls(**values)
+
+
+class Embedder(ABC):
+    def __init__(self, config: EmbedConfig):
+        self.config = config
+
+    @abstractmethod
+    def embed(self, text: str) -> list[float]:
+        """Embed a single text string."""
+
+    @abstractmethod
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Embed multiple strings."""
+
+
+class OpenAIEmbedder(Embedder):
+    BASE_RETRY_DELAY = 1.0
+
+    def __init__(self, config: EmbedConfig):
+        super().__init__(config)
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError("OPENAI_API_KEY environment variable not set")
+        self.client = OpenAI(api_key=api_key)
+
+    def embed(self, text: str) -> list[float]:
+        return self.embed_batch([text])[0]
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+
+        vectors: list[list[float]] = []
+        for start in range(0, len(texts), self.config.batch_size):
+            chunk = texts[start : start + self.config.batch_size]
+            vectors.extend(self._embed_chunk(chunk))
+            if self.config.rate_limit_delay > 0:
+                time.sleep(self.config.rate_limit_delay)
+        return vectors
+
+    def _embed_chunk(self, texts: list[str]) -> list[list[float]]:
+        last_error: Exception | None = None
+
+        for attempt in range(self.config.max_retries):
+            try:
+                response = self.client.embeddings.create(
+                    model=self.config.model,
+                    input=texts,
+                    dimensions=self.config.dimensions,
+                )
+                return [list(item.embedding) for item in response.data]
+            except APIError as exc:  # pragma: no cover - covered by retry test
+                last_error = exc
+                if attempt < self.config.max_retries - 1:
+                    delay = self.BASE_RETRY_DELAY * (2**attempt)
+                    time.sleep(delay)
+
+        if last_error:
+            raise last_error
+        raise RuntimeError("Embedding request failed")
+
+
+def create_embedder(config: EmbedConfig) -> Embedder:
+    if config.provider == "openai":
+        return OpenAIEmbedder(config)
+    raise ValueError(f"Unsupported embedding provider: {config.provider}")

oghma/exporter.py

@@ -0,0 +1,177 @@
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+from oghma.storage import MemoryRecord, Storage
+
+
+@dataclass
+class ExportOptions:
+    output_dir: Path
+    format: str = "markdown"
+    group_by: str = "category"
+    include_metadata: bool = True
+
+
+class Exporter:
+    def __init__(self, storage: Storage, options: ExportOptions):
+        self.storage = storage
+        self.options = options
+
+    def export(self) -> list[Path]:
+        """Export memories to files, returning list of created file paths."""
+        memories = self.storage.get_all_memories(status="active")
+
+        if not memories:
+            return []
+
+        self.options.output_dir.mkdir(parents=True, exist_ok=True)
+
+        if self.options.group_by == "category":
+            return self._export_by_category(memories)
+        elif self.options.group_by == "date":
+            return self._export_by_date(memories)
+        elif self.options.group_by == "source":
+            return self._export_by_source(memories)
+        else:
+            raise ValueError(f"Unsupported group_by: {self.options.group_by}")
+
+    def export_category(self, category: str) -> Path:
+        """Export memories for a single category."""
+        memories = self.storage.get_all_memories(status="active", category=category)
+
+        if not memories:
+            raise ValueError(f"No memories found for category: {category}")
+
+        self.options.output_dir.mkdir(parents=True, exist_ok=True)
+
+        if self.options.format == "markdown":
+            content = self._format_markdown(memories, category)
+            ext = ".md"
+        elif self.options.format == "json":
+            content = self._format_json(memories)
+            ext = ".json"
+        else:
+            raise ValueError(f"Unsupported format: {self.options.format}")
+
+        safe_category = category.replace("/", "_").replace("\\", "_")
+        filename = f"{safe_category}{ext}"
+        file_path = self.options.output_dir / filename
+
+        file_path.write_text(content, encoding="utf-8")
+        return file_path
+
+    def _export_by_category(self, memories: list[MemoryRecord]) -> list[Path]:
+        categories = {m["category"] for m in memories}
+        files = []
+
+        for category in sorted(categories):
+            category_memories = [m for m in memories if m["category"] == category]
+
+            if self.options.format == "markdown":
+                content = self._format_markdown(category_memories, category)
+                ext = ".md"
+            elif self.options.format == "json":
+                content = self._format_json(category_memories)
+                ext = ".json"
+            else:
+                raise ValueError(f"Unsupported format: {self.options.format}")
+
+            safe_category = category.replace("/", "_").replace("\\", "_")
+            filename = f"{safe_category}{ext}"
+            file_path = self.options.output_dir / filename
+
+            file_path.write_text(content, encoding="utf-8")
+            files.append(file_path)
+
+        return files
+
+    def _export_by_date(self, memories: list[MemoryRecord]) -> list[Path]:
+        dates = {m["created_at"][:10] for m in memories}
+        files = []
+
+        for date_str in sorted(dates):
+            date_memories = [m for m in memories if m["created_at"].startswith(date_str)]
+
+            if self.options.format == "markdown":
+                content = self._format_markdown(date_memories, date_str)
+                ext = ".md"
+            elif self.options.format == "json":
+                content = self._format_json(date_memories)
+                ext = ".json"
+            else:
+                raise ValueError(f"Unsupported format: {self.options.format}")
+
+            filename = f"{date_str}{ext}"
+            file_path = self.options.output_dir / filename
+
+            file_path.write_text(content, encoding="utf-8")
+            files.append(file_path)
+
+        return files
+
+    def _export_by_source(self, memories: list[MemoryRecord]) -> list[Path]:
+        sources = {m["source_tool"] for m in memories}
+        files = []
+
+        for source in sorted(sources):
+            source_memories = [m for m in memories if m["source_tool"] == source]
+
+            if self.options.format == "markdown":
+                content = self._format_markdown(source_memories, source)
+                ext = ".md"
+            elif self.options.format == "json":
+                content = self._format_json(source_memories)
+                ext = ".json"
+            else:
+                raise ValueError(f"Unsupported format: {self.options.format}")
+
+            safe_source = source.replace("/", "_").replace("\\", "_")
+            filename = f"{safe_source}{ext}"
+            file_path = self.options.output_dir / filename
+
+            file_path.write_text(content, encoding="utf-8")
+            files.append(file_path)
+
+        return files
+
+    def _format_markdown(self, memories: list[MemoryRecord], title: str) -> str:
+        """Format memories as markdown with YAML frontmatter."""
+        lines = [
+            "---",
+            f"category: {title}",
+            f"exported_at: {datetime.now().isoformat()}",
+            f"count: {len(memories)}",
+            "---",
+            "",
+            f"# {title.title()}",
+            "",
+        ]
+
+        for memory in memories:
+            content_preview = (
+                memory["content"][:80] + "..." if len(memory["content"]) > 80 else memory["content"]
+            )
+            lines.append(f"## {content_preview}")
+            source_info = (
+                f"*Source: {memory['source_tool']} | {memory['created_at'][:10]} | "
+                f"Confidence: {memory['confidence']:.0%}*"
+            )
+            lines.append(source_info)
+            lines.append("")
+            lines.append(memory["content"])
+            lines.append("")
+            lines.append("---")
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def _format_json(self, memories: list[MemoryRecord]) -> str:
+        """Format memories as JSON."""
+        export_data = {
+            "exported_at": datetime.now().isoformat(),
+            "count": len(memories),
+            "memories": memories,
+        }
+        return json.dumps(export_data, indent=2, ensure_ascii=False, default=str)

oghma/extractor.py

@@ -0,0 +1,180 @@
+import json
+import logging
+import os
+import time
+from dataclasses import dataclass
+
+from openai import OpenAI
+
+from oghma.config import Config
+from oghma.parsers import Message
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Memory:
+    content: str
+    category: str
+    confidence: float = 1.0
+
+
+class Extractor:
+    """Extracts memories from conversations using LLM."""
+
+    MAX_RETRIES = 3
+    BASE_RETRY_DELAY = 1.0
+
+    CATEGORIES = ["learning", "preference", "project_context", "gotcha", "workflow"]
+
+    # Models that require OpenRouter
+    OPENROUTER_PREFIXES = ("google/", "anthropic/", "meta-llama/", "deepseek/", "moonshotai/")
+
+    def __init__(self, config: Config):
+        self.config = config
+        self.model = config.get("extraction", {}).get("model", "gpt-4o-mini")
+        self.max_chars = config.get("extraction", {}).get("max_content_chars", 4000)
+
+        # Determine which API to use based on model name
+        if self.model.startswith(self.OPENROUTER_PREFIXES):
+            api_key = os.environ.get("OPENROUTER_API_KEY")
+            if not api_key:
+                raise ValueError("OPENROUTER_API_KEY environment variable not set")
+            self.client = OpenAI(
+                api_key=api_key,
+                base_url="https://openrouter.ai/api/v1"
+            )
+            self.use_openrouter = True
+        else:
+            api_key = os.environ.get("OPENAI_API_KEY")
+            if not api_key:
+                raise ValueError("OPENAI_API_KEY environment variable not set")
+            self.client = OpenAI(api_key=api_key)
+            self.use_openrouter = False
+
+    def extract(self, messages: list[Message], source_tool: str) -> list[Memory]:
+        """Extract memories from a list of messages."""
+        if not messages:
+            return []
+
+        prompt = self._build_prompt(messages)
+
+        for attempt in range(self.MAX_RETRIES):
+            try:
+                response = self._call_openai(prompt)
+                memories = self._parse_response(response)
+
+                valid_memories = [
+                    m for m in memories if m.category in self.CATEGORIES and m.confidence >= 0.5
+                ]
+
+                logger.info(
+                    f"Extracted {len(valid_memories)} memories from {source_tool} "
+                    f"(attempt {attempt + 1})"
+                )
+                return valid_memories
+
+            except Exception as e:
+                if attempt < self.MAX_RETRIES - 1:
+                    delay = self.BASE_RETRY_DELAY * (2**attempt)
+                    logger.warning(
+                        f"Extraction attempt {attempt + 1} failed: {e}. Retrying in {delay}s..."
+                    )
+                    time.sleep(delay)
+                else:
+                    logger.error(f"Extraction failed after {self.MAX_RETRIES} attempts: {e}")
+
+        return []
+
+    def _call_openai(self, prompt: str) -> str:
+        """Call LLM API and return the response text."""
+        kwargs = {
+            "model": self.model,
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are a memory extraction system. "
+                    "Always respond with valid JSON only, no markdown.",
+                },
+                {"role": "user", "content": prompt},
+            ],
+            "temperature": 0.3,
+            "max_tokens": 1500,
+        }
+
+        # OpenAI models support structured output, OpenRouter models don't
+        if not self.use_openrouter:
+            kwargs["response_format"] = {"type": "json_object"}
+
+        response = self.client.chat.completions.create(**kwargs)
+
+        content = response.choices[0].message.content
+        if not content:
+            raise ValueError("Empty response from LLM API")
+
+        return content
+
+    def _build_prompt(self, messages: list[Message]) -> str:
+        """Build the extraction prompt."""
+        messages_text = ""
+        for msg in messages[:100]:
+            role_label = "User" if msg.role == "user" else "Assistant"
+            messages_text += f"{role_label}: {msg.content}\n\n"
+
+        messages_text = messages_text[: self.max_chars]
+
+        categories_desc = "\n".join(f"- {cat}" for cat in self.CATEGORIES)
+
+        prompt = (
+            "You are a memory extraction system. "
+            "Analyze this conversation and extract key memories.\n\n"
+            f"Categories:\n{categories_desc}\n\n"
+            f"Conversation:\n{messages_text}\n\n"
+            "Extract memories in this JSON format:\n"
+            '[  {"content": "...", "category": "...", "confidence": 0.0-1.0},\n'
+            "  ...\n"
+            "]\n\n"
+            "Only extract clear, specific memories. Skip vague or trivial content.\n"
+            "Return empty array [] if no significant memories found.\n"
+            "Remember: respond with valid JSON only, no markdown formatting."
+        )
+
+        return prompt
+
+    def _parse_response(self, response_text: str) -> list[Memory]:
+        """Parse LLM response into Memory objects."""
+        response_text = response_text.strip()
+
+        if response_text.startswith("```"):
+            lines = response_text.split("\n")
+            response_text = "\n".join(lines[1:-1])
+
+        try:
+            data = json.loads(response_text)
+        except json.JSONDecodeError:
+            logger.warning(f"Failed to parse JSON response: {response_text[:200]}...")
+            return []
+
+        if not isinstance(data, list):
+            return []
+
+        memories: list[Memory] = []
+        for item in data:
+            if not isinstance(item, dict):
+                continue
+
+            content = item.get("content")
+            category = item.get("category")
+            confidence = item.get("confidence", 1.0)
+
+            if not content or not category:
+                continue
+
+            if not isinstance(confidence, (int, float)):
+                confidence = 1.0
+
+            confidence = max(0.0, min(1.0, float(confidence)))
+
+            memories.append(Memory(content=content, category=category, confidence=confidence))
+
+        return memories