andnp-searchkernel 0.1.0__tar.gz

andnp_searchkernel-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Sync dependencies
+        run: uv sync
+
+      - name: Lint with ruff
+        run: uv run ruff check .
+
+      - name: Type check
+        run: uv run pyright
+
+      - name: Check imports with import-linter
+        run: uv run lint-imports
+
+      - name: Run unit tests
+        run: uv run pytest -m unit -v

andnp_searchkernel-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

andnp_searchkernel-0.1.0/.gitignore

@@ -0,0 +1,54 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+.env
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.hypothesis/
+
+# Tools
+.ruff_cache/
+.mypy_cache/
+.pytype/
+
+# Lock files (generated by uv)
+uv.lock

andnp_searchkernel-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 andnp-searchkernel contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

andnp_searchkernel-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: andnp-searchkernel
+Version: 0.1.0
+Summary: Source-agnostic search/indexing kernel: domain-agnostic content ingestion, hybrid vector+keyword+graph search, pluggable embedding/LLM/reranker providers.
+Project-URL: Homepage, https://github.com/andnp/andnp-searchkernel
+Project-URL: Documentation, https://github.com/andnp/andnp-searchkernel/tree/main/docs
+Project-URL: Repository, https://github.com/andnp/andnp-searchkernel.git
+Project-URL: Issues, https://github.com/andnp/andnp-searchkernel/issues
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: faiss-cpu>=1.9.0
+Requires-Dist: llama-index-core>=0.12.0
+Requires-Dist: llama-index-embeddings-huggingface>=0.4.0
+Requires-Dist: llama-index-vector-stores-faiss>=0.3.0
+Requires-Dist: psycopg2-binary>=2.9.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: tenacity>=9.0.0
+Requires-Dist: tomlkit>=0.13.0
+Requires-Dist: tree-sitter-markdown>=0.3.2
+Requires-Dist: tree-sitter>=0.23.0
+Description-Content-Type: text/markdown
+
+# `andnp-searchkernel`
+
+A domain-agnostic search/indexing kernel for building hybrid vector + keyword + graph search systems with pluggable embedding, LLM, and reranker providers.
+
+## Status
+
+**Pre-alpha, extraction in progress.** This library is being extracted from [`mcp-markdown-ragdocs`](https://github.com/andnp/mcp-markdown-ragdocs) to enable reuse across arbitrary content sources and search backends.
+
+## License
+
+MIT License. See [LICENSE](LICENSE) for details.

andnp_searchkernel-0.1.0/README.md

@@ -0,0 +1,11 @@
+# `andnp-searchkernel`
+
+A domain-agnostic search/indexing kernel for building hybrid vector + keyword + graph search systems with pluggable embedding, LLM, and reranker providers.
+
+## Status
+
+**Pre-alpha, extraction in progress.** This library is being extracted from [`mcp-markdown-ragdocs`](https://github.com/andnp/mcp-markdown-ragdocs) to enable reuse across arbitrary content sources and search backends.
+
+## License
+
+MIT License. See [LICENSE](LICENSE) for details.

andnp_searchkernel-0.1.0/pyproject.toml

@@ -0,0 +1,79 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "andnp-searchkernel"
+version = "0.1.0"
+description = "Source-agnostic search/indexing kernel: domain-agnostic content ingestion, hybrid vector+keyword+graph search, pluggable embedding/LLM/reranker providers."
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "llama-index-core>=0.12.0",
+    "llama-index-embeddings-huggingface>=0.4.0",
+    "llama-index-vector-stores-faiss>=0.3.0",
+    "faiss-cpu>=1.9.0",
+    "sentence-transformers>=3.0.0",
+    "tenacity>=9.0.0",
+    "psycopg2-binary>=2.9.0",
+    "pyyaml>=6.0.2",
+    "tomlkit>=0.13.0",
+    "tree-sitter>=0.23.0",
+    "tree-sitter-markdown>=0.3.2",
+]
+
+[project.urls]
+Homepage = "https://github.com/andnp/andnp-searchkernel"
+Documentation = "https://github.com/andnp/andnp-searchkernel/tree/main/docs"
+Repository = "https://github.com/andnp/andnp-searchkernel.git"
+Issues = "https://github.com/andnp/andnp-searchkernel/issues"
+
+[dependency-groups]
+dev = [
+    "pyright>=1.1.407",
+    "pytest>=9.0.2",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=6.0.0",
+    "pytest-xdist>=3.5.0",
+    "ruff>=0.14.10",
+    "ty>=0.0.12",
+    "import-linter>=2.0",
+]
+
+[tool.uv]
+package = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["searchkernel"]
+
+[tool.pytest.ini_options]
+pythonpath = "."
+addopts = "-n 4 --dist worksteal"
+markers = [
+    "serial: marks tests that must run serially (not in parallel)",
+    "unit: marks unit tests (fast, isolated)",
+    "integration: marks integration tests (involve multiple components)",
+    "e2e: marks end-to-end tests (full system tests)",
+    "slow: marks slow tests (>5s, load models or process large datasets)",
+    "real_embeddings: marks tests that require real embedding models (not fake deterministic embeddings)",
+]
+
+[tool.importlinter]
+root_packages = ["searchkernel"]
+default_mapping = "searchkernel"
+
+[[tool.importlinter.contracts]]
+name = "domain and ports are inward-only"
+type = "forbidden"
+source_modules = ["searchkernel.domain", "searchkernel.ports"]
+forbidden_modules = [
+    "searchkernel.adapters",
+    "searchkernel.pipeline",
+    "searchkernel.search",
+    "searchkernel.indices",
+    "searchkernel.storage",
+    "searchkernel.runtime",
+    "searchkernel.eval",
+    "searchkernel.compression",
+]
+ignore_imports = []

andnp_searchkernel-0.1.0/searchkernel/__init__.py

@@ -0,0 +1,19 @@
+"""
+Source-agnostic search/indexing kernel.
+
+A domain-agnostic library for building hybrid vector+keyword+graph search systems
+with pluggable embedding/LLM/reranker providers.
+"""
+
+from searchkernel.search.orchestrator import SearchOrchestrator
+from searchkernel.search.pipeline import SearchPipelineConfig
+from searchkernel.search.utils import classify_query_type, truncate_content
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "SearchOrchestrator",
+    "SearchPipelineConfig",
+    "classify_query_type",
+    "truncate_content",
+]

andnp_searchkernel-0.1.0/searchkernel/adapters/cache/__init__.py

@@ -0,0 +1,6 @@
+"""Cache store adapters implementing the CacheStore port."""
+
+from searchkernel.adapters.cache.memory_lru import MemoryLRUCacheStore
+from searchkernel.adapters.cache.sqlite import SQLiteCacheStore
+
+__all__ = ["MemoryLRUCacheStore", "SQLiteCacheStore"]

andnp_searchkernel-0.1.0/searchkernel/adapters/cache/memory_lru.py

@@ -0,0 +1,71 @@
+"""In-memory LRU cache store with epoch-based invalidation."""
+
+from collections import OrderedDict
+from typing import Any
+
+
+class MemoryLRUCacheStore:
+    """LRU in-memory cache store implementing the CacheStore port.
+
+    Stores key-value pairs in memory with a bounded size. When capacity is
+    exceeded, the least recently used entry is evicted. Entries are tagged
+    with an epoch for invalidation support.
+    """
+
+    def __init__(self, max_entries: int = 128):
+        """Initialize memory LRU cache store.
+
+        Args:
+            max_entries: Maximum number of entries to store (default 128).
+                        Must be at least 1.
+        """
+        self._max_entries = max(1, max_entries)
+        self._entries: OrderedDict[str, tuple[Any, int]] = OrderedDict()
+
+    def get(self, key: str) -> Any | None:
+        """Retrieve a cached value.
+
+        Args:
+            key: Cache key.
+
+        Returns:
+            Cached value, or None if not found.
+        """
+        if key not in self._entries:
+            return None
+
+        value, _epoch = self._entries[key]
+        # Move to end to mark as recently used (LRU)
+        self._entries.move_to_end(key)
+        return value
+
+    def set(self, key: str, value: Any, epoch: int) -> None:
+        """Store a value with an associated epoch.
+
+        Args:
+            key: Cache key.
+            value: Value to cache.
+            epoch: Index epoch at cache time. Used for invalidation.
+        """
+        if key in self._entries:
+            self._entries.pop(key)
+
+        self._entries[key] = (value, epoch)
+        self._entries.move_to_end(key)
+
+        # Evict LRU entry if over capacity
+        while len(self._entries) > self._max_entries:
+            self._entries.popitem(last=False)
+
+    def invalidate_epoch(self, epoch: int) -> None:
+        """Invalidate all entries from an epoch or earlier.
+
+        Args:
+            epoch: Entries with epoch <= this are discarded.
+        """
+        keys_to_delete = [
+            key for key, (_, entry_epoch) in self._entries.items()
+            if entry_epoch <= epoch
+        ]
+        for key in keys_to_delete:
+            del self._entries[key]

andnp_searchkernel-0.1.0/searchkernel/adapters/cache/sqlite.py

@@ -0,0 +1,113 @@
+"""SQLite-backed cache store with epoch-based invalidation."""
+
+import json
+import logging
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class SQLiteCacheStore:
+    """SQLite-backed cache store implementing the CacheStore port.
+
+    Persists key-value pairs to a SQLite database. Entries are tagged with
+    an epoch for invalidation support. Designed for durability across restarts.
+    """
+
+    def __init__(self, db_path: Path | str):
+        """Initialize SQLite cache store.
+
+        Args:
+            db_path: Path to SQLite database file.
+        """
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        """Create the cache table if it doesn't exist."""
+        with sqlite3.connect(self.db_path) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS cache_store (
+                    key TEXT PRIMARY KEY,
+                    value TEXT NOT NULL,
+                    epoch INTEGER NOT NULL,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                );
+            """)
+            conn.execute("""
+                CREATE INDEX IF NOT EXISTS idx_cache_epoch
+                ON cache_store (epoch);
+            """)
+            conn.commit()
+
+    def get(self, key: str) -> Any | None:
+        """Retrieve a cached value.
+
+        Args:
+            key: Cache key.
+
+        Returns:
+            Cached value, or None if not found.
+        """
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                cursor = conn.execute(
+                    "SELECT value FROM cache_store WHERE key = ?;",
+                    (key,),
+                )
+                row = cursor.fetchone()
+                if row:
+                    value_json = row[0]
+                    return json.loads(value_json)
+                return None
+        except (sqlite3.DatabaseError, json.JSONDecodeError) as e:
+            logger.warning(f"Error retrieving cache key {key}: {e}", exc_info=True)
+            return None
+
+    def set(self, key: str, value: Any, epoch: int) -> None:
+        """Store a value with an associated epoch.
+
+        Args:
+            key: Cache key.
+            value: Value to cache.
+            epoch: Index epoch at cache time. Used for invalidation.
+        """
+        try:
+            value_json = json.dumps(value)
+            with sqlite3.connect(self.db_path) as conn:
+                conn.execute(
+                    """
+                    INSERT INTO cache_store (key, value, epoch)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT (key) DO UPDATE SET
+                        value = EXCLUDED.value,
+                        epoch = EXCLUDED.epoch;
+                    """,
+                    (key, value_json, epoch),
+                )
+                conn.commit()
+        except (sqlite3.DatabaseError, TypeError):
+            logger.exception(f"Error storing cache key {key}")
+
+    def invalidate_epoch(self, epoch: int) -> None:
+        """Invalidate all entries from an epoch or earlier.
+
+        Args:
+            epoch: Entries with epoch <= this are discarded.
+        """
+        try:
+            with sqlite3.connect(self.db_path) as conn:
+                cursor = conn.execute(
+                    "DELETE FROM cache_store WHERE epoch <= ?;",
+                    (epoch,),
+                )
+                conn.commit()
+                logger.debug(
+                    f"Invalidated cache entries for epochs <= {epoch} "
+                    f"({cursor.rowcount} entries deleted)"
+                )
+        except sqlite3.DatabaseError:
+            logger.exception(f"Error invalidating cache epoch {epoch}")

andnp_searchkernel-0.1.0/searchkernel/adapters/embedding/__init__.py

@@ -0,0 +1,5 @@
+"""Embedding provider adapters implementing the EmbeddingProvider port."""
+
+from searchkernel.adapters.embedding.huggingface import HuggingFaceEmbeddingProvider
+
+__all__ = ["HuggingFaceEmbeddingProvider"]

andnp_searchkernel-0.1.0/searchkernel/adapters/embedding/huggingface.py

@@ -0,0 +1,89 @@
+"""HuggingFace / sentence-transformers EmbeddingProvider adapter.
+
+In-process embedding via ``sentence_transformers.SentenceTransformer``.
+Defaults to Qwen3-Embedding-0.6B. No Ollama, no external service.
+
+This is an ADDITIVE port implementation. The live embedding path
+(``searchkernel/indices/vector.py``, BAAI/bge-small-en-v1.5) is untouched.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from searchkernel.domain import Vector
+
+if TYPE_CHECKING:
+    from sentence_transformers import SentenceTransformer
+
+# Documented Qwen3-Embedding query instruction, used only if the loaded model
+# does not expose a "query" prompt via sentence-transformers.
+_QWEN3_QUERY_INSTRUCTION = (
+    "Instruct: Given a web search query, retrieve relevant passages\nQuery: "
+)
+
+
+class HuggingFaceEmbeddingProvider:
+    """EmbeddingProvider backed by a sentence-transformers model.
+
+    Qwen3-Embedding is asymmetric: queries take an instruction prompt,
+    documents do not. ``embed`` embeds DOCUMENTS (no prompt); ``embed_query``
+    applies the query instruction. The EmbeddingProvider port itself has no
+    query variant yet -- that asymmetry lives here until W4a lifts it into
+    the port contract.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "Qwen/Qwen3-Embedding-0.6B",
+        *,
+        truncate_dim: int | None = None,
+        device: str | None = None,
+    ):
+        from sentence_transformers import SentenceTransformer
+
+        self.model_name = model_name
+        self._truncate_dim = truncate_dim
+        # Matryoshka (MRL): passing truncate_dim makes the model emit
+        # truncated + re-normalized vectors directly.
+        self._model: SentenceTransformer = SentenceTransformer(
+            model_name, truncate_dim=truncate_dim, device=device
+        )
+        native_dim = self._model.get_sentence_embedding_dimension()
+        self.dim: int = truncate_dim if truncate_dim is not None else int(native_dim)
+        # Whether the model ships a named "query" prompt we can reference.
+        self._has_query_prompt = "query" in getattr(self._model, "prompts", {})
+
+    def embed(self, texts: list[str]) -> list[Vector]:
+        """Embed DOCUMENTS (no instruction prompt), L2-normalized."""
+        embeddings = self._model.encode(
+            texts,
+            batch_size=32,
+            normalize_embeddings=True,
+            convert_to_numpy=True,
+        )
+        return embeddings.tolist()
+
+    def embed_query(self, text: str) -> Vector:
+        """Embed a single QUERY with the Qwen3 instruction prompt applied."""
+        return self.embed_queries([text])[0]
+
+    def embed_queries(self, texts: list[str]) -> list[Vector]:
+        """Embed QUERIES with the Qwen3 instruction prompt applied."""
+        if self._has_query_prompt:
+            embeddings = self._model.encode(
+                texts,
+                prompt_name="query",
+                batch_size=32,
+                normalize_embeddings=True,
+                convert_to_numpy=True,
+            )
+        else:
+            embeddings = self._model.encode(
+                texts,
+                prompt=_QWEN3_QUERY_INSTRUCTION,
+                batch_size=32,
+                normalize_embeddings=True,
+                convert_to_numpy=True,
+            )
+        return embeddings.tolist()

andnp_searchkernel-0.1.0/searchkernel/adapters/llm/__init__.py

@@ -0,0 +1,5 @@
+"""LLM provider adapters implementing the LLMProvider port."""
+
+from searchkernel.adapters.llm.copilot import CopilotLLMProvider
+
+__all__ = ["CopilotLLMProvider"]