know-do-graph 0.1.7__tar.gz → 0.1.8__tar.gz

know_do_graph-0.1.7/README.md → know_do_graph-0.1.8/PKG-INFO

@@ -1,3 +1,29 @@
+Metadata-Version: 2.4
+Name: know-do-graph
+Version: 0.1.8
+Summary: A wiki-native, agent-oriented infrastructure for executable knowledge, operational memory, and capability discovery.
+Requires-Python: >=3.10
+Requires-Dist: duckduckgo-search>=6.0
+Requires-Dist: fastapi>=0.100
+Requires-Dist: httpx>=0.25
+Requires-Dist: networkx>=3.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: openai>=1.0
+Requires-Dist: prompt-toolkit>=3.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: typer>=0.24.1
+Requires-Dist: uvicorn[standard]
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers>=2.7; extra == 'embeddings'
+Provides-Extra: local-embeddings
+Requires-Dist: sentence-transformers>=2.7; extra == 'local-embeddings'
+Description-Content-Type: text/markdown
+
 # Know-Do Graph
 
 A wiki-native, agent-oriented infrastructure for **executable knowledge**, **operational memory**, and **capability discovery**.
@@ -36,6 +62,35 @@ KDG_DB_PATH=./my-data/my-memory.db
 
 Relative `KDG_DB_PATH` values are resolved from the current working directory.
 
+### Optional embeddings
+
+The default install does not include a local embedding model stack, so it will
+not download PyTorch. Search still works through the keyword path; `hybrid` and
+`semantic` retrieval fall back gracefully when no embedding provider is enabled.
+
+To use a local `sentence-transformers` model:
+
+```bash
+pip install "know-do-graph[local-embeddings]"
+export KDG_EMBED_PROVIDER=local
+export KDG_EMBED_MODEL="sentence-transformers/all-MiniLM-L6-v2"  # optional
+```
+
+To use an OpenAI-compatible embeddings API instead:
+
+```bash
+export KDG_EMBED_PROVIDER=openai
+export KDG_EMBED_MODEL="text-embedding-3-small"
+export KDG_EMBED_DIM=384
+export KDG_EMBED_API_KEY="..."                         # or OPENAI_API_KEY
+export KDG_EMBED_BASE_URL="https://example.com/v1"     # optional
+```
+
+`KDG_EMBED_DIM` defaults to `384`, matching the bundled SQLite vector table.
+If your provider does not support choosing dimensions, use a model that returns
+384-dimensional vectors or keyword retrieval will continue to be the reliable
+fallback.
+
 ### Python API
 
 Use the high-level client to embed the graph directly in an agent process:

know_do_graph-0.1.7/PKG-INFO → know_do_graph-0.1.8/README.md

@@ -1,26 +1,3 @@
-Metadata-Version: 2.4
-Name: know-do-graph
-Version: 0.1.7
-Summary: A wiki-native, agent-oriented infrastructure for executable knowledge, operational memory, and capability discovery.
-Requires-Python: >=3.10
-Requires-Dist: duckduckgo-search>=6.0
-Requires-Dist: fastapi>=0.100
-Requires-Dist: httpx>=0.25
-Requires-Dist: networkx>=3.0
-Requires-Dist: numpy>=1.24
-Requires-Dist: openai>=1.0
-Requires-Dist: prompt-toolkit>=3.0
-Requires-Dist: pydantic>=2.0
-Requires-Dist: python-dotenv>=1.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: rich>=13.0
-Requires-Dist: sentence-transformers>=2.7
-Requires-Dist: sqlalchemy>=2.0
-Requires-Dist: sqlite-vec>=0.1.6
-Requires-Dist: typer>=0.24.1
-Requires-Dist: uvicorn[standard]
-Description-Content-Type: text/markdown
-
 # Know-Do Graph
 
 A wiki-native, agent-oriented infrastructure for **executable knowledge**, **operational memory**, and **capability discovery**.
@@ -59,6 +36,35 @@ KDG_DB_PATH=./my-data/my-memory.db
 
 Relative `KDG_DB_PATH` values are resolved from the current working directory.
 
+### Optional embeddings
+
+The default install does not include a local embedding model stack, so it will
+not download PyTorch. Search still works through the keyword path; `hybrid` and
+`semantic` retrieval fall back gracefully when no embedding provider is enabled.
+
+To use a local `sentence-transformers` model:
+
+```bash
+pip install "know-do-graph[local-embeddings]"
+export KDG_EMBED_PROVIDER=local
+export KDG_EMBED_MODEL="sentence-transformers/all-MiniLM-L6-v2"  # optional
+```
+
+To use an OpenAI-compatible embeddings API instead:
+
+```bash
+export KDG_EMBED_PROVIDER=openai
+export KDG_EMBED_MODEL="text-embedding-3-small"
+export KDG_EMBED_DIM=384
+export KDG_EMBED_API_KEY="..."                         # or OPENAI_API_KEY
+export KDG_EMBED_BASE_URL="https://example.com/v1"     # optional
+```
+
+`KDG_EMBED_DIM` defaults to `384`, matching the bundled SQLite vector table.
+If your provider does not support choosing dimensions, use a model that returns
+384-dimensional vectors or keyword retrieval will continue to be the reliable
+fallback.
+
 ### Python API
 
 Use the high-level client to embed the graph directly in an agent process:

{know_do_graph-0.1.7 → know_do_graph-0.1.8}/core/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.7'
-__version_tuple__ = version_tuple = (0, 1, 7)
+__version__ = version = '0.1.8'
+__version_tuple__ = version_tuple = (0, 1, 8)
 
 __commit_id__ = commit_id = None

know_do_graph-0.1.8/core/retrieval/embedder.py

@@ -0,0 +1,233 @@
+"""Embedding service for hybrid retrieval.
+
+Pluggable via the ``Embedder`` protocol. Embeddings are disabled by default
+to keep the base install lightweight. Set ``KDG_EMBED_PROVIDER`` to ``local``
+or ``openai`` to opt in.
+
+Failures (missing dependency, model download error, etc.) are logged once
+and the embedder reports ``available=False`` so retrieval falls back to
+keyword search rather than raising.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from typing import Optional, Protocol
+
+logger = logging.getLogger(__name__)
+
+
+def build_embedding_text(
+    title: str,
+    aliases: list[str],
+    tags: list[str],
+    content: str,
+    content_chars: int = 2000,
+) -> str:
+    """Canonical text representation used to embed an entry."""
+    parts = [title.strip()]
+    if aliases:
+        parts.append(" | ".join(a.strip() for a in aliases if a.strip()))
+    if tags:
+        parts.append("tags: " + ", ".join(t.strip() for t in tags if t.strip()))
+    body = (content or "").strip()
+    if body:
+        parts.append("")
+        parts.append(body[:content_chars])
+    return "\n".join(parts)
+
+
+def text_hash(text: str) -> str:
+    return hashlib.sha1(text.encode("utf-8")).hexdigest()
+
+
+class Embedder(Protocol):
+    dim: int
+    available: bool
+
+    def embed(self, texts: list[str]) -> list[list[float]]: ...
+
+
+class _NullEmbedder:
+    """Stand-in used when vector embeddings are disabled or unavailable."""
+
+    dim = 0
+    available = False
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        return [[] for _ in texts]
+
+
+class SentenceTransformerEmbedder:
+    """Local CPU embedder using sentence-transformers."""
+
+    def __init__(self, model_name: Optional[str] = None) -> None:
+        self.model_name = model_name or os.environ.get(
+            "KDG_EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2"
+        )
+        self._model = None
+        self._dim: Optional[int] = None
+        self.available = True  # set to False on first failed load
+
+    def _load(self) -> None:
+        if self._model is not None or not self.available:
+            return
+        try:
+            from sentence_transformers import SentenceTransformer  # type: ignore
+        except ImportError:
+            logger.warning(
+                "sentence-transformers not installed; hybrid retrieval disabled. "
+                "Install with: pip install 'know-do-graph[local-embeddings]'"
+            )
+            self.available = False
+            return
+        try:
+            self._model = SentenceTransformer(self.model_name)
+            self._dim = int(self._model.get_sentence_embedding_dimension())
+        except Exception as exc:
+            logger.warning("Failed to load embedding model %s: %s", self.model_name, exc)
+            self.available = False
+
+    @property
+    def dim(self) -> int:
+        self._load()
+        return self._dim or 0
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        self._load()
+        if not self.available or self._model is None:
+            return [[] for _ in texts]
+        vecs = self._model.encode(
+            texts,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+            show_progress_bar=False,
+        )
+        return [v.tolist() for v in vecs]
+
+
+class OpenAIEmbedder:
+    """OpenAI-compatible embeddings API backend."""
+
+    def __init__(
+        self,
+        model_name: Optional[str] = None,
+        dimensions: Optional[int] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+    ) -> None:
+        self.model_name = model_name or os.environ.get(
+            "KDG_EMBED_MODEL", "text-embedding-3-small"
+        )
+        self.dimensions = dimensions or _optional_int(os.environ.get("KDG_EMBED_DIM")) or 384
+        self.api_key = api_key or os.environ.get("KDG_EMBED_API_KEY") or os.environ.get(
+            "OPENAI_API_KEY"
+        )
+        self.base_url = base_url or os.environ.get("KDG_EMBED_BASE_URL") or os.environ.get(
+            "OPENAI_API_BASE"
+        )
+        self._client = None
+        self.available = True
+
+    def _load(self) -> None:
+        if self._client is not None or not self.available:
+            return
+        if not self.api_key:
+            logger.warning(
+                "KDG_EMBED_PROVIDER=openai but no API key is configured; "
+                "set KDG_EMBED_API_KEY or OPENAI_API_KEY."
+            )
+            self.available = False
+            return
+        try:
+            from openai import OpenAI
+        except ImportError:
+            logger.warning("openai package not installed; API embeddings disabled.")
+            self.available = False
+            return
+        try:
+            kwargs = {"api_key": self.api_key}
+            if self.base_url:
+                kwargs["base_url"] = self.base_url
+            self._client = OpenAI(**kwargs)
+        except Exception as exc:
+            logger.warning("Failed to initialize embeddings API client: %s", exc)
+            self.available = False
+
+    @property
+    def dim(self) -> int:
+        self._load()
+        return self.dimensions if self.available else 0
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        self._load()
+        if not self.available or self._client is None:
+            return [[] for _ in texts]
+        try:
+            response = self._client.embeddings.create(
+                model=self.model_name,
+                input=texts,
+                dimensions=self.dimensions,
+            )
+            data = sorted(response.data, key=lambda item: item.index)
+            return [list(item.embedding) for item in data]
+        except TypeError:
+            # Some OpenAI-compatible providers do not accept the dimensions argument.
+            try:
+                response = self._client.embeddings.create(
+                    model=self.model_name,
+                    input=texts,
+                )
+                data = sorted(response.data, key=lambda item: item.index)
+                return [list(item.embedding) for item in data]
+            except Exception as exc:
+                logger.warning("Embedding API request failed: %s", exc)
+                self.available = False
+                return [[] for _ in texts]
+        except Exception as exc:
+            logger.warning("Embedding API request failed: %s", exc)
+            self.available = False
+            return [[] for _ in texts]
+
+
+def _optional_int(value: Optional[str]) -> Optional[int]:
+    if value is None or value.strip() == "":
+        return None
+    try:
+        return int(value)
+    except ValueError:
+        logger.warning("Invalid KDG_EMBED_DIM=%r; using the default.", value)
+        return None
+
+
+def _provider_name() -> str:
+    return os.environ.get("KDG_EMBED_PROVIDER", "none").strip().lower()
+
+
+_default: Optional[Embedder] = None
+
+
+def get_default_embedder() -> Embedder:
+    """Process-wide singleton embedder."""
+    global _default
+    if _default is None:
+        provider = _provider_name()
+        if provider in {"", "0", "false", "no", "none", "off", "disabled"}:
+            _default = _NullEmbedder()
+            return _default
+        if provider in {"local", "sentence-transformers", "sentence_transformers"}:
+            candidate: Embedder = SentenceTransformerEmbedder()
+        elif provider in {"openai", "api", "remote"}:
+            candidate = OpenAIEmbedder()
+        else:
+            logger.warning(
+                "Unknown KDG_EMBED_PROVIDER=%r; hybrid retrieval disabled.", provider
+            )
+            _default = _NullEmbedder()
+            return _default
+        # Trigger load eagerly so we know whether it works; downgrade to null if not.
+        _ = candidate.dim
+        _default = candidate if candidate.available else _NullEmbedder()
+    return _default

{know_do_graph-0.1.7 → know_do_graph-0.1.8}/pyproject.toml

@@ -22,11 +22,14 @@ dependencies = [
     "duckduckgo-search>=6.0",
     "python-dotenv>=1.0",
     "prompt_toolkit>=3.0",
-    "sentence-transformers>=2.7",
     "sqlite-vec>=0.1.6",
     "numpy>=1.24",
 ]
 
+[project.optional-dependencies]
+embeddings = ["sentence-transformers>=2.7"]
+local-embeddings = ["sentence-transformers>=2.7"]
+
 [project.scripts]
 know-do-graph = "main:app"
 

know_do_graph-0.1.8/tests/test_embedder_config.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+from types import ModuleType, SimpleNamespace
+from unittest.mock import patch
+
+from core.retrieval import embedder
+
+
+class EmbedderConfigTests(unittest.TestCase):
+    def tearDown(self) -> None:
+        embedder._default = None
+
+    def test_default_provider_is_disabled(self) -> None:
+        with patch.dict(os.environ, {}, clear=True):
+            embedder._default = None
+
+            selected = embedder.get_default_embedder()
+
+        self.assertFalse(selected.available)
+        self.assertEqual(selected.dim, 0)
+        self.assertEqual(selected.embed(["hello"]), [[]])
+
+    def test_openai_provider_requires_api_key(self) -> None:
+        with patch.dict(os.environ, {"KDG_EMBED_PROVIDER": "openai"}, clear=True):
+            embedder._default = None
+
+            selected = embedder.get_default_embedder()
+
+        self.assertFalse(selected.available)
+        self.assertEqual(selected.dim, 0)
+
+    def test_openai_provider_uses_compatible_embeddings_client(self) -> None:
+        fake_openai = ModuleType("openai")
+
+        class FakeEmbeddings:
+            def create(self, **kwargs):
+                self.kwargs = kwargs
+                return SimpleNamespace(
+                    data=[
+                        SimpleNamespace(index=1, embedding=[0.3, 0.4]),
+                        SimpleNamespace(index=0, embedding=[0.1, 0.2]),
+                    ]
+                )
+
+        class FakeOpenAI:
+            def __init__(self, **kwargs):
+                self.kwargs = kwargs
+                self.embeddings = FakeEmbeddings()
+
+        fake_openai.OpenAI = FakeOpenAI
+
+        env = {
+            "KDG_EMBED_PROVIDER": "openai",
+            "KDG_EMBED_API_KEY": "test-key",
+            "KDG_EMBED_MODEL": "embedding-model",
+            "KDG_EMBED_DIM": "2",
+        }
+        with patch.dict(os.environ, env, clear=True), patch.dict(
+            sys.modules, {"openai": fake_openai}
+        ):
+            embedder._default = None
+
+            selected = embedder.get_default_embedder()
+
+            self.assertTrue(selected.available)
+            self.assertEqual(selected.dim, 2)
+            self.assertEqual(selected.embed(["a", "b"]), [[0.1, 0.2], [0.3, 0.4]])
+
+
+if __name__ == "__main__":
+    unittest.main()

know_do_graph-0.1.7/core/retrieval/embedder.py

@@ -1,122 +0,0 @@
-"""Embedding service for hybrid retrieval.
-
-Pluggable via the ``Embedder`` protocol. Default backend is a local
-sentence-transformers model (MiniLM, 384 dim, CPU). Override the model via
-the ``KDG_EMBED_MODEL`` env var.
-
-Failures (missing dependency, model download error, etc.) are logged once
-and the embedder reports ``available=False`` so retrieval falls back to
-keyword search rather than raising.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-import os
-from typing import Optional, Protocol
-
-logger = logging.getLogger(__name__)
-
-
-def build_embedding_text(
-    title: str,
-    aliases: list[str],
-    tags: list[str],
-    content: str,
-    content_chars: int = 2000,
-) -> str:
-    """Canonical text representation used to embed an entry."""
-    parts = [title.strip()]
-    if aliases:
-        parts.append(" | ".join(a.strip() for a in aliases if a.strip()))
-    if tags:
-        parts.append("tags: " + ", ".join(t.strip() for t in tags if t.strip()))
-    body = (content or "").strip()
-    if body:
-        parts.append("")
-        parts.append(body[:content_chars])
-    return "\n".join(parts)
-
-
-def text_hash(text: str) -> str:
-    return hashlib.sha1(text.encode("utf-8")).hexdigest()
-
-
-class Embedder(Protocol):
-    dim: int
-    available: bool
-
-    def embed(self, texts: list[str]) -> list[list[float]]: ...
-
-
-class _NullEmbedder:
-    """Stand-in used when sentence-transformers is not installed."""
-
-    dim = 0
-    available = False
-
-    def embed(self, texts: list[str]) -> list[list[float]]:
-        return [[] for _ in texts]
-
-
-class SentenceTransformerEmbedder:
-    """Local CPU embedder using sentence-transformers."""
-
-    def __init__(self, model_name: Optional[str] = None) -> None:
-        self.model_name = model_name or os.environ.get(
-            "KDG_EMBED_MODEL", "sentence-transformers/all-MiniLM-L6-v2"
-        )
-        self._model = None
-        self._dim: Optional[int] = None
-        self.available = True  # set to False on first failed load
-
-    def _load(self) -> None:
-        if self._model is not None or not self.available:
-            return
-        try:
-            from sentence_transformers import SentenceTransformer  # type: ignore
-        except ImportError:
-            logger.warning(
-                "sentence-transformers not installed; hybrid retrieval disabled. "
-                "Install with: pip install 'know-do-graph[embeddings]'"
-            )
-            self.available = False
-            return
-        try:
-            self._model = SentenceTransformer(self.model_name)
-            self._dim = int(self._model.get_sentence_embedding_dimension())
-        except Exception as exc:
-            logger.warning("Failed to load embedding model %s: %s", self.model_name, exc)
-            self.available = False
-
-    @property
-    def dim(self) -> int:
-        self._load()
-        return self._dim or 0
-
-    def embed(self, texts: list[str]) -> list[list[float]]:
-        self._load()
-        if not self.available or self._model is None:
-            return [[] for _ in texts]
-        vecs = self._model.encode(
-            texts,
-            normalize_embeddings=True,
-            convert_to_numpy=True,
-            show_progress_bar=False,
-        )
-        return [v.tolist() for v in vecs]
-
-
-_default: Optional[Embedder] = None
-
-
-def get_default_embedder() -> Embedder:
-    """Process-wide singleton embedder."""
-    global _default
-    if _default is None:
-        candidate = SentenceTransformerEmbedder()
-        # Trigger load eagerly so we know whether it works; downgrade to null if not.
-        _ = candidate.dim
-        _default = candidate if candidate.available else _NullEmbedder()
-    return _default