openchronicle-mcp 3.0.0.dev0__py3-none-any.whl

openchronicle/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

openchronicle/core/application/config/__init__.py

@@ -0,0 +1 @@
+"""Application configuration loading."""

openchronicle/core/application/config/env_helpers.py

@@ -0,0 +1,117 @@
+"""Consolidated environment variable and config value parsing helpers.
+
+These helpers handle three-layer precedence: dataclass defaults -> JSON file
+values -> env var overrides. They work with both string values (from env vars)
+and native JSON types (bool, int, float) from config files.
+"""
+
+from __future__ import annotations
+
+import os
+
+
+def parse_bool(value: object, *, default: bool) -> bool:
+    """Parse a boolean from string, native bool, or None.
+
+    Truthy strings: "1", "true", "yes", "on" (case-insensitive).
+    Falsy strings: "0", "false", "no", "off" (case-insensitive).
+    Native bools pass through. None returns default.
+    """
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+    return default
+
+
+def parse_int(value: object, *, default: int) -> int:
+    """Parse an integer from string, native int, or None.
+
+    Native ints pass through. Strings are stripped and converted.
+    Invalid values return default. None returns default.
+    """
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        # bool is a subclass of int in Python — reject it
+        return default
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str):
+        raw = value.strip()
+        try:
+            return int(raw)
+        except ValueError:
+            return default
+    return default
+
+
+def parse_float(value: object, *, default: float) -> float:
+    """Parse a float from string, native float/int, or None.
+
+    Native floats/ints pass through. Strings are stripped and converted.
+    Invalid values return default. None returns default.
+    """
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return default
+    if isinstance(value, int | float):
+        return float(value)
+    if isinstance(value, str):
+        raw = value.strip()
+        try:
+            return float(raw)
+        except ValueError:
+            return default
+    return default
+
+
+def parse_str(value: object, *, default: str) -> str:
+    """Parse a string value, returning default for None or empty."""
+    if value is None:
+        return default
+    if isinstance(value, str):
+        return value if value else default
+    return str(value)
+
+
+def parse_str_list(value: object, *, default: list[str]) -> list[str]:
+    """Parse a list of strings from a JSON array or CSV string.
+
+    - list[str]: pass through (filtering empties)
+    - str: split on commas, strip, filter empties
+    - None: return default
+    """
+    if value is None:
+        return list(default)
+    if isinstance(value, list):
+        return [str(item).strip() for item in value if str(item).strip()]
+    if isinstance(value, str):
+        return [item.strip() for item in value.split(",") if item.strip()]
+    return list(default)
+
+
+def env_override(env_name: str, file_value: object) -> object:
+    """Return env var if set, otherwise file_value.
+
+    This implements the precedence: env var > JSON file > (caller's default).
+    Only returns the env var when it is explicitly set in the environment.
+    """
+    env_val = os.getenv(env_name)
+    if env_val is not None:
+        return env_val
+    return file_value
+
+
+def parse_csv_tags(value: str | None) -> list[str] | None:
+    """Parse a comma-separated string into a list of stripped, non-empty tags.
+
+    Returns None if the input is None or empty, which signals "no filter"
+    in search contexts.
+    """
+    if not value:
+        return None
+    return [t.strip() for t in value.split(",") if t.strip()]

openchronicle/core/application/config/paths.py

@@ -0,0 +1,79 @@
+"""Canonical runtime path resolution for the entire project.
+
+All data-directory paths flow through ``RuntimePaths.resolve()``.
+Four-layer precedence: constructor param > per-path env var >
+``OC_DATA_DIR``-derived > hardcoded default.
+
+v3 manages three paths: the SQLite file, the config dir, and an
+output dir. v2's plugin/assets/discord paths are gone with their
+subsystems.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+# ── Default constants ────────────────────────────────────────────────
+
+DEFAULT_DB_PATH = "data/openchronicle.db"
+DEFAULT_CONFIG_DIR = "config"
+DEFAULT_OUTPUT_DIR = "output"
+
+
+def _resolve(
+    explicit: str | Path | None,
+    env_var: str,
+    data_dir: str | None,
+    data_dir_suffix: str,
+    fallback: str,
+) -> Path:
+    """Four-layer path resolution.
+
+    1. Constructor param (``explicit``) — wins unconditionally.
+    2. Per-path env var — checked next.
+    3. ``OC_DATA_DIR + suffix`` — if ``OC_DATA_DIR`` is set.
+    4. Hardcoded fallback — last resort.
+    """
+    if explicit is not None:
+        return Path(explicit)
+    env_val = os.environ.get(env_var)
+    if env_val is not None:
+        return Path(env_val)
+    if data_dir is not None:
+        return Path(data_dir) / data_dir_suffix
+    return Path(fallback)
+
+
+@dataclass(frozen=True)
+class RuntimePaths:
+    """Resolved runtime paths for v3 data artifacts."""
+
+    db_path: Path
+    config_dir: Path
+    output_dir: Path
+
+    @classmethod
+    def resolve(
+        cls,
+        *,
+        db_path: str | Path | None = None,
+        config_dir: str | Path | None = None,
+        output_dir: str | Path | None = None,
+        # Accepted but ignored for backwards-compatible callers (e.g.
+        # CLI flags / kwargs that haven't been pruned in lock step).
+        plugin_dir: str | Path | None = None,  # noqa: ARG003 - accepted for kwarg stability
+    ) -> RuntimePaths:
+        """Build ``RuntimePaths`` with four-layer precedence.
+
+        Constructor params > per-path env vars > ``OC_DATA_DIR``-derived
+        > defaults.
+        """
+        data_dir = os.environ.get("OC_DATA_DIR")
+
+        return cls(
+            db_path=_resolve(db_path, "OC_DB_PATH", data_dir, "openchronicle.db", DEFAULT_DB_PATH),
+            config_dir=_resolve(config_dir, "OC_CONFIG_DIR", data_dir, "config", DEFAULT_CONFIG_DIR),
+            output_dir=_resolve(output_dir, "OC_OUTPUT_DIR", data_dir, "output", DEFAULT_OUTPUT_DIR),
+        )

openchronicle/core/application/config/settings.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from openchronicle.core.application.config.env_helpers import (
+    env_override,
+    parse_int,
+    parse_str,
+)
+
+
+@dataclass(frozen=True)
+class EmbeddingSettings:
+    """Embedding provider configuration."""
+
+    provider: str = "none"
+    model: str = ""
+    dimensions: int | None = None
+    api_key: str = ""
+    timeout: float = 30.0
+
+    def __post_init__(self) -> None:
+        valid = {"none", "stub", "openai", "ollama"}
+        if self.provider not in valid:
+            raise ValueError(f"embedding provider must be one of {valid}, got {self.provider!r}")
+        if self.dimensions is not None and self.dimensions < 1:
+            raise ValueError(f"embedding dimensions must be >= 1, got {self.dimensions}")
+        if self.timeout <= 0:
+            raise ValueError(f"embedding timeout must be > 0, got {self.timeout}")
+
+
+def load_embedding_settings(
+    file_config: dict[str, Any] | None = None,
+) -> EmbeddingSettings:
+    fc = file_config or {}
+    dims_raw = env_override("OC_EMBEDDING_DIMENSIONS", fc.get("dimensions"))
+    dims = parse_int(dims_raw, default=0) if dims_raw is not None else 0
+    timeout_raw = env_override("OC_EMBEDDING_TIMEOUT", fc.get("timeout"))
+    timeout = float(str(timeout_raw)) if timeout_raw is not None else 30.0
+    return EmbeddingSettings(
+        provider=parse_str(
+            env_override("OC_EMBEDDING_PROVIDER", fc.get("provider")),
+            default="none",
+        ).lower(),
+        model=parse_str(
+            env_override("OC_EMBEDDING_MODEL", fc.get("model")),
+            default="",
+        ),
+        dimensions=dims if dims != 0 else None,
+        api_key=parse_str(
+            env_override("OC_EMBEDDING_API_KEY", fc.get("api_key")),
+            default="",
+        ),
+        timeout=timeout,
+    )

openchronicle/core/application/models/__init__.py

@@ -0,0 +1 @@
+"""Application models."""

openchronicle/core/application/models/diagnostics_report.py

@@ -0,0 +1,32 @@
+"""Diagnostics report model.
+
+Returned by ``diagnose_runtime.execute()`` and surfaced by the v3
+health endpoints (``/health``, ``/api/v1/health``, MCP ``health`` tool).
+v2's plugin_dir / model config discovery / OC_LLM_* env summary fields
+are gone with the LLM stack.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class DiagnosticsReport:
+    """Runtime diagnostics report."""
+
+    timestamp_utc: datetime
+    db_path: str
+    db_exists: bool
+    db_size_bytes: int | None
+    db_modified_utc: datetime | None
+    config_dir: str
+    config_dir_exists: bool
+    running_in_container_hint: bool
+    persistence_hint: str
+    # Embedding subsystem status — populated by interfaces with a
+    # CoreContainer in hand (the container injects its own
+    # `embedding_status_dict` here).
+    embedding_status: dict[str, Any] | None = field(default=None)

openchronicle/core/application/services/__init__.py

@@ -0,0 +1 @@
+"""Application services layer."""

openchronicle/core/application/services/embedding_service.py

@@ -0,0 +1,325 @@
+"""Embedding service — generates embeddings and performs hybrid search."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from openchronicle.core.domain.models.memory_item import MemoryItem
+from openchronicle.core.domain.ports.embedding_port import EmbeddingPort
+
+if TYPE_CHECKING:
+    from openchronicle.core.infrastructure.persistence.sqlite_store import SqliteStore
+
+logger = logging.getLogger(__name__)
+
+# RRF constant — standard value from the original RRF paper
+_RRF_K = 60
+
+
+@dataclass(frozen=True)
+class BackfillResult:
+    """Outcome of a backfill run.
+
+    The per-item resilience in ``generate_missing`` keeps a single bad item
+    from blocking the rest, but it must NOT hide total failure from callers.
+    Carrying ``failed`` alongside ``generated`` lets the MCP/API/CLI surfaces
+    return an honest status to clients.
+    """
+
+    generated: int
+    failed: int
+    elapsed_ms: int
+
+
+class EmbeddingService:
+    """Coordinates embedding generation and hybrid (FTS5 + semantic) search."""
+
+    def __init__(self, port: EmbeddingPort, store: SqliteStore) -> None:
+        self._port = port
+        self._store = store
+        # Degraded-search bookkeeping — flips on when the embedding
+        # provider raises during a search, flips off the next time
+        # it succeeds. Surfaced via container.embedding_status_dict
+        # for /api/v1/health.
+        self._search_failure_count: int = 0
+        self._last_failure_at: str | None = None
+
+    @property
+    def port(self) -> EmbeddingPort:
+        return self._port
+
+    @property
+    def search_failure_count(self) -> int:
+        return self._search_failure_count
+
+    @property
+    def last_failure_at(self) -> str | None:
+        return self._last_failure_at
+
+    def generate_for_memory(
+        self,
+        memory_id: str,
+        content: str,
+        *,
+        force: bool = False,
+    ) -> None:
+        """Generate and store embedding for a single memory item.
+
+        Skips generation if an embedding already exists with the same model,
+        unless ``force`` is True (used when content changes).
+        """
+        if not force:
+            existing_model = self._store.get_embedding_model(memory_id)
+            if existing_model == self._port.model_name():
+                return
+
+        vec = self._port.embed(content)
+        self._store.save_embedding(
+            memory_id,
+            vec,
+            model=self._port.model_name(),
+            dimensions=self._port.dimensions(),
+        )
+
+    def generate_missing(self, *, project_id: str | None = None, force: bool = False) -> BackfillResult:
+        """Backfill embeddings for memories that don't have one.
+
+        If *force* is True, regenerate all embeddings (model change scenario).
+        Individual failures are logged and skipped so the backfill always
+        completes — but the failure count is returned so callers can surface
+        a partial/total-failure status instead of falsely reporting "ok".
+        """
+        import time
+
+        items = self._store.list_memory(limit=None, pinned_only=False)
+        if project_id:
+            items = [i for i in items if i.project_id == project_id]
+
+        candidates = []
+        for item in items:
+            if not force:
+                existing_model = self._store.get_embedding_model(item.id)
+                if existing_model == self._port.model_name():
+                    continue
+            candidates.append(item)
+
+        if not candidates:
+            logger.info("Embedding backfill: 0 candidates, nothing to do")
+            return BackfillResult(generated=0, failed=0, elapsed_ms=0)
+
+        logger.info(
+            "Embedding backfill started: %d candidates (model=%s, force=%s)",
+            len(candidates),
+            self._port.model_name(),
+            force,
+        )
+
+        t0 = time.monotonic()
+        count = 0
+        failed = 0
+        for item in candidates:
+            try:
+                vec = self._port.embed(item.content)
+                self._store.save_embedding(
+                    item.id,
+                    vec,
+                    model=self._port.model_name(),
+                    dimensions=self._port.dimensions(),
+                )
+                count += 1
+            except Exception:
+                failed += 1
+                logger.warning("Embedding generation failed for memory %s", item.id, exc_info=True)
+
+        elapsed_ms = int((time.monotonic() - t0) * 1000)
+        logger.info(
+            "Embedding backfill completed: %d generated, %d failed, %dms elapsed",
+            count,
+            failed,
+            elapsed_ms,
+        )
+        return BackfillResult(generated=count, failed=failed, elapsed_ms=elapsed_ms)
+
+    def embedding_status(self) -> dict[str, int]:
+        """Return embedding coverage stats."""
+        total_memories = self._store.count_memory()
+        embedded = self._store.count_embeddings()
+        stale = self._store.count_stale_embeddings(self._port.model_name())
+        return {
+            "total_memories": total_memories,
+            "embedded": embedded,
+            "missing": total_memories - embedded,
+            "stale": stale,
+        }
+
+    def search_hybrid(
+        self,
+        query: str,
+        *,
+        top_k: int = 8,
+        project_id: str | None = None,
+        include_pinned: bool = True,
+        tags: list[str] | None = None,
+        offset: int = 0,
+    ) -> list[MemoryItem]:
+        """Hybrid search: FTS5 keyword + embedding similarity via RRF.
+
+        1. Run keyword search (FTS5) for ranked list A
+        2. Embed query → cosine similarity → ranked list B
+        3. Combine via Reciprocal Rank Fusion
+        4. Return top_k results
+        """
+        effective_top_k = top_k + offset
+
+        # ── Pinned items (always included) ──────────────────────────────
+        pinned_items: list[MemoryItem] = []
+        if include_pinned:
+            pinned_items = self._store.pinned_items(project_id)
+            if tags:
+                pinned_items = [i for i in pinned_items if all(t in i.tags for t in tags)]
+
+        # Pinned items have separate budget — don't reduce search/RRF limit
+        # (prevents pinned items from crowding out query-relevant results)
+
+        pinned_ids = {i.id for i in pinned_items}
+
+        # ── Keyword search (list A) ─────────────────────────────────────
+        keyword_results = self._store.search_memory(
+            query,
+            top_k=effective_top_k * 2,  # over-fetch for RRF merge
+            project_id=project_id,
+            include_pinned=False,
+            tags=tags,
+        )
+        keyword_results = [i for i in keyword_results if i.id not in pinned_ids]
+
+        # ── Semantic search (list B) ─────────────────────────────────────
+        # Embedding-failure degradation: if the provider raises, log it,
+        # mark the service degraded, and return FTS5-only results. The
+        # caller never sees the exception; /api/v1/health surfaces the
+        # degraded state via the failure counters on the service.
+        try:
+            semantic_ranked = self._semantic_search(
+                query,
+                project_id=project_id,
+                tags=tags,
+                exclude_ids=pinned_ids,
+                limit=effective_top_k * 2,
+            )
+            # Successful call clears any prior degraded marker.
+            if self._search_failure_count:
+                logger.info(
+                    "embedding search recovered after %d prior failures",
+                    self._search_failure_count,
+                )
+                self._search_failure_count = 0
+        except Exception as exc:
+            from datetime import UTC, datetime
+
+            self._search_failure_count += 1
+            self._last_failure_at = datetime.now(UTC).isoformat()
+            logger.warning(
+                "embedding search failed (%d total); degrading to FTS5-only: %s",
+                self._search_failure_count,
+                exc,
+            )
+            non_pinned_page = keyword_results[offset : offset + top_k]
+            if offset == 0:
+                return list(pinned_items) + non_pinned_page
+            return non_pinned_page
+
+        # ── RRF merge ──────────────────────────────────────────────────
+        keyword_rank: dict[str, int] = {item.id: rank for rank, item in enumerate(keyword_results, start=1)}
+        semantic_rank: dict[str, int] = {mid: rank for rank, mid in enumerate(semantic_ranked, start=1)}
+
+        all_ids = set(keyword_rank) | set(semantic_rank)
+        # Build lookup for MemoryItem objects
+        item_map: dict[str, MemoryItem] = {i.id: i for i in keyword_results}
+
+        # For semantic-only results, fetch MemoryItem from store
+        for mid in semantic_rank:
+            if mid not in item_map:
+                mem = self._store.get_memory(mid)
+                if mem:
+                    item_map[mid] = mem
+
+        rrf_scores: list[tuple[str, float]] = []
+        for mid in all_ids:
+            if mid not in item_map:
+                continue
+            # Apply tag filter to semantic-only results
+            if tags and not all(t in item_map[mid].tags for t in tags):
+                continue
+            # Apply project filter to semantic-only results
+            if project_id and item_map[mid].project_id != project_id:
+                continue
+
+            kr = keyword_rank.get(mid)
+            sr = semantic_rank.get(mid)
+            score = 0.0
+            if kr is not None:
+                score += 1.0 / (_RRF_K + kr)
+            if sr is not None:
+                score += 1.0 / (_RRF_K + sr)
+            rrf_scores.append((mid, score))
+
+        rrf_scores.sort(key=lambda x: x[1], reverse=True)
+
+        merged = [item_map[mid] for mid, _ in rrf_scores[:effective_top_k]]
+
+        # Pinned items prepended on first page only; offset paginates non-pinned results
+        non_pinned_page = merged[offset : offset + top_k]
+        if offset == 0:
+            return list(pinned_items) + non_pinned_page
+        return non_pinned_page
+
+    def _semantic_search(
+        self,
+        query: str,
+        *,
+        project_id: str | None = None,
+        tags: list[str] | None = None,
+        exclude_ids: set[str] | None = None,
+        limit: int = 16,
+    ) -> list[str]:
+        """Return memory IDs ranked by cosine similarity to query embedding.
+
+        All adapters normalize at output, so dot product = cosine similarity.
+        Numpy single-matmul replaces a per-item Python loop; for ~5k memories
+        at 1536 dims this is ~50-100x faster than the prior pure-Python path.
+        Memory cost is unchanged: list_embeddings still loads the full
+        embedding table — that's the architectural ceiling addressed by a
+        future move to a vector-indexed store (sqlite-vec).
+        """
+        import numpy as np
+
+        query_vec = self._port.embed(query)
+        all_embeddings = self._store.list_embeddings()
+
+        if not all_embeddings:
+            return []
+
+        ids = [mid for mid in all_embeddings if mid not in exclude_ids] if exclude_ids else list(all_embeddings)
+        if not ids:
+            return []
+
+        matrix = np.asarray([all_embeddings[mid] for mid in ids], dtype=np.float32)
+        q = np.asarray(query_vec, dtype=np.float32)
+        scores = matrix @ q  # (N,) cosine similarities
+
+        # argpartition gives top-k unsorted in O(N); sort the slice for ranks.
+        k = min(limit, scores.shape[0])
+        top_unsorted = np.argpartition(-scores, k - 1)[:k] if k < scores.shape[0] else np.arange(scores.shape[0])
+        top_sorted = top_unsorted[np.argsort(-scores[top_unsorted])]
+        return [ids[i] for i in top_sorted]
+
+
+def _cosine_similarity(a: list[float], b: list[float]) -> float:
+    """Dot product of unit vectors = cosine similarity.
+
+    Kept as a small helper for tests + diagnostic callers. The hot search
+    path uses numpy (see _semantic_search).
+    """
+    return sum(x * y for x, y in zip(a, b, strict=False))