dbs-vector 0.5.1__tar.gz

dbs_vector-0.5.1/.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    name: Lint, Typecheck, Tests
+    runs-on: macos-14
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Run quality checks
+        run: uv run poe github-release-test

dbs_vector-0.5.1/.github/workflows/release.yml

@@ -0,0 +1,94 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  verify_main:
+    name: Verify tag points to main
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Ensure tagged commit is on main
+        run: |
+          git fetch origin main
+          if ! git branch -r --contains "$GITHUB_SHA" | grep -q "origin/main"; then
+            echo "Tag commit is not on main. Aborting release."
+            exit 1
+          fi
+
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    needs: verify_main
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tooling
+        run: python -m pip install --upgrade pip build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Validate distributions
+        run: twine check dist/*
+
+      - name: Upload dist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+  github_release:
+    name: Create GitHub Release
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: write
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Publish GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+          files: dist/*
+
+  publish_pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: build
+    if: vars.PYPI_PUBLISH == 'true'
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-dist
+          path: dist/
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

dbs_vector-0.5.1/.gitignore

@@ -0,0 +1,16 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.DS_Store
+
+# Virtual environments
+.venv
+.ayder
+.coverage
+
+# test db
+lancedb_dbs_vector/

dbs_vector-0.5.1/.python-version

@@ -0,0 +1 @@
+3.12

dbs_vector-0.5.1/CLAUDE.md

@@ -0,0 +1,95 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+```bash
+# Install dependencies
+uv sync
+
+# Run full validation suite (format, lint, typecheck, test)
+uv run poe check
+
+# Run tests only
+uv run poe test
+
+# Run tests with coverage
+uv run poe test-cov
+
+# Run a single test file
+uv run pytest tests/unit/test_chunker.py -v
+
+# Run a single test by name
+uv run pytest tests/unit/test_chunker.py::test_function_name -v
+
+# Lint and format
+uv run poe lint
+uv run poe format
+
+# Type checking
+uv run poe typecheck
+
+# CLI commands
+uv run dbs-vector ingest "docs/" --type md
+uv run dbs-vector ingest "queries.json" --type sql
+uv run dbs-vector search "query text" --type md
+uv run dbs-vector serve
+uv run dbs-vector mcp
+```
+
+## Architecture
+
+This is a Clean Architecture, configuration-driven RAG search engine for Apple Silicon (MLX). The dependency flow is: **CLI/API → Services → Core Protocols → Infrastructure**.
+
+### Layers
+
+**`core/`** — Pure domain layer with no external dependencies.
+- `models.py`: Pydantic domain models (`Document`, `Chunk`, `SqlChunk`, `SearchResult`, `SqlSearchResult`).
+- `ports.py`: Protocol interfaces (`IEmbedder`, `IChunker`, `IVectorStore`, `IStoreMapper`) that decouple infrastructure from services.
+- `registry.py`: `ComponentRegistry` maps string names from `config.yaml` to concrete mapper/chunker classes.
+
+**`infrastructure/`** — Concrete implementations of the core protocols.
+- `embeddings/mlx_engine.py`: `MLXEmbedder` — runs models on Apple GPU via MLX, casts tensors to NumPy via Unified Memory. Includes a process-level `_MODEL_CACHE` dict to avoid reloading models.
+- `storage/lancedb_engine.py`: `LanceDBStore` — Arrow-native storage; uses `IVF_PQ` vector index + Tantivy FTS. Schema mismatch on startup means `--rebuild --force` is needed.
+- `storage/mappers.py`: `DocumentMapper` and `SqlMapper` convert domain chunks ↔ PyArrow `RecordBatch` for zero-copy ingestion and back to domain models on retrieval.
+- `chunking/document.py`: `DocumentChunker` — uses `markdown-it-py` to parse `.md` semantically (code fences are kept atomic); falls back to naive splitting for `.txt`.
+- `chunking/sql.py`: `SqlChunker` — parses JSON slow query log format.
+
+**`services/`** — Orchestration, depend only on protocols.
+- `ingestion.py`: `IngestionService` — reads files, chunks, deduplicates via SHA-256 content hashes, batches, embeds, and streams to `IVectorStore`.
+- `search.py`: `SearchService` — embeds query and delegates hybrid search; also formats results for CLI output.
+
+**`api/`** — FastAPI HTTP server + MCP stdio server.
+- `main.py`: FastAPI app with lifespan startup (loads all engines from config), `/health`, `/search/md`, `/search/sql` endpoints. MCP is mounted at `/mcp` (SSE).
+- `mcp_server.py`: `FastMCP` server exposing search as MCP tools.
+- `state.py`: Shared `_services` dict (engine name → `SearchService`) used by both the FastAPI lifespan and the MCP command.
+
+**`config.py`** — `Settings` (pydantic-settings) + `EngineConfig` per engine. Loaded from `config.yaml` at startup. Env prefix: `DBS_`. The path can be overridden with `--config-file` or `DBS_CONFIG_FILE` env var.
+
+### Configuration-Driven Registry Pattern
+
+Adding a new engine type requires:
+1. Implement `IChunker` and `IStoreMapper` concrete classes.
+2. Register them in `ComponentRegistry._chunkers` / `ComponentRegistry._mappers`.
+3. Add the engine block to `config.yaml` with appropriate `mapper_type`, `chunker_type`, `model_name`, etc.
+
+No changes to services, CLI, or API are needed.
+
+### Key Design Details
+
+- **Deduplication**: Content hashes (SHA-256 truncated to 16 chars) are computed at the file level and stored per chunk. Ingestion skips chunks whose hash already exists in the store.
+- **Schema evolution**: If `LanceDBStore` detects a schema mismatch on startup, it raises a descriptive `ValueError` that the CLI surfaces with a `--rebuild --force` hint.
+- **Asymmetric embeddings**: `MLXEmbedder` prepends different prefixes for passages (`passage_prefix`) vs queries (`query_prefix`), supporting instruction-tuned models like `embeddinggemma`.
+- **Thread safety**: `MLXEmbedder` uses a per-model `threading.Lock`; FastAPI offloads synchronous search to `asyncio.to_thread`.
+- **IVF_PQ indexing**: Only created when `total_rows > 256`; partitions scale as `sqrt(total_rows)` capped at 256.
+
+### Test Structure
+
+```
+tests/
+  unit/         # Mock-based, no I/O — fast
+  integration/  # Uses tmpdir LanceDB + real chunkers/mappers
+```
+
+Mypy ignores `lancedb`, `pyarrow`, and `mlx_embeddings` (no stubs). Ruff enforces pycodestyle, pyflakes, bugbear, pyupgrade, and isort at line length 100.

dbs_vector-0.5.1/LICENSE.md

@@ -0,0 +1,10 @@
+# GNU General Public License v3.0 or later
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This project is licensed under the **GNU General Public License, version 3 or (at your option) any later version**.
+
+- Official GPL-3.0 license text: https://www.gnu.org/licenses/gpl-3.0.txt
+- GPL license overview: https://www.gnu.org/licenses/gpl-3.0.en.html
+
+By contributing to or distributing this project, you agree that it is provided under the terms of GPL-3.0-or-later.

dbs_vector-0.5.1/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: dbs-vector
+Version: 0.5.1
+Summary: High-performance, local RAG search engine and MCP/API server for Apple Silicon
+License-File: LICENSE.md
+Requires-Python: >=3.12
+Requires-Dist: fastapi>=0.133.0
+Requires-Dist: lance==1.2.1
+Requires-Dist: lancedb==0.29.2
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: markdown-it-py>=4.0.0
+Requires-Dist: mcp>=1.26.0
+Requires-Dist: mlx-embeddings>=0.0.5
+Requires-Dist: numpy>=2.2.3
+Requires-Dist: polars==1.38.1
+Requires-Dist: pyarrow==19.0.1
+Requires-Dist: pydantic-settings==2.13.1
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: tantivy==0.25.1
+Requires-Dist: typer>=0.15.1
+Requires-Dist: types-pyyaml>=6.0.12.20250915
+Requires-Dist: uvicorn>=0.41.0
+Provides-Extra: api
+Requires-Dist: httpx>=0.27.0; extra == 'api'
+Provides-Extra: sql
+Requires-Dist: duckdb>=1.2.0; extra == 'sql'
+Description-Content-Type: text/markdown
+
+# ⚡️ dbs-vector
+
+**A High-Performance, Arrow-Native Local Codebase Search Engine for Apple Silicon.**
+
+`dbs-vector` is a optimized Retrieval-Augmented Generation (RAG) search engine designed specifically for macOS (M-Series chips). It bypasses traditional Python serialization bottlenecks by utilizing Apple's Unified Memory Architecture (UMA) and pure Apache Arrow data pipelines.
+
+It enables lightning-fast, hybrid (Vector + Full-Text) search across your local codebase, entirely offline.
+
+---
+
+## ✨ Features
+
+*   **Zero-Copy Memory Pipelines**: Uses **MLX** to compute embeddings on the Mac GPU, casting the resulting tensors instantly into NumPy arrays via Unified Memory without costly `float` object instantiation.
+*   **Arrow-Native Storage**: Uses **LanceDB** to stream ingestion batches directly to disk via PyArrow, avoiding the massive memory overhead of JSON and dictionary comprehensions.
+*   **Hybrid Retrieval**: Simultaneously executes Approximate Nearest Neighbor (ANN) cosine vector search and native **Tantivy** Full-Text Search (FTS).
+*   **Code-Aware Chunking**: Intelligently splits documentation and code, respecting markdown fences so that code blocks are indexed as atomic units.
+*   **Production Robustness**: Features dynamic `IVF_PQ` indexing, Rust-level predicate pushdown (metadata filtering), and dataset compaction for delta-updates.
+*   **Remote SQL API Ingestion**: `ApiChunker` pulls pre-aggregated slow-query records from any networked backend over HTTP, replacing local files with a paginated REST API — no changes to the embedding or storage layers.
+
+## 🚀 Installation
+
+This project is built using `uv`, an extremely fast Python package manager.
+
+1. **Clone the repository:**
+   ```bash
+   git clone https://github.com/dbsmedya/dbs-vector.git
+   cd dbs-vector
+   ```
+
+2. **Install the CLI package:**
+   ```bash
+   uv sync
+   ```
+   *This automatically sets up the environment and creates the `dbs-vector` executable in your path.*
+
+   Optional extras unlock additional ingestion sources:
+
+   ```bash
+   uv sync --extra sql  # DuckDB ingestion
+   uv sync --extra api  # Remote HTTP API ingestion
+   ```
+
+## 💻 Usage
+
+The application is entirely configuration-driven via `config.yaml`. It supports multiple data types (Engines) such as Markdown and SQL.
+
+### Global Options
+*   `--config-file` / `-c`: Path to your custom `config.yaml` (Defaults to `./config.yaml`).
+
+### Ingesting Documents
+Index markdown files, JSON SQL logs, DuckDB analytical files, or a remote HTTP slow-query API into the local vector store.
+
+```bash
+# Ingest all markdown files (default)
+uv run dbs-vector ingest "docs/"
+
+# Ingest SQL slow query logs (JSON format)
+uv run dbs-vector ingest "slow_queries.json" --type sql
+
+# Ingest SQL slow queries from DuckDB (High-Performance Columnar)
+uv run dbs-vector ingest "slow_queries.duckdb" --type sql --rebuild
+
+# Ingest from a remote HTTP API (paginated GET)
+uv run dbs-vector ingest "https://slow-log-api.internal/api/v1" --type sql-api
+
+# Ingest via a custom SELECT sent to the remote API
+uv run dbs-vector ingest "https://slow-log-api.internal/api/v1" --type sql-api \
+  --query "SELECT fingerprint_id AS id, sanitized_sql AS text, db AS source, ..."
+```
+
+### Searching the Codebase
+Execute queries against your chosen engine.
+
+```bash
+# Semantic hybrid search across markdown
+uv run dbs-vector search "What is MLX?"
+
+# Find similar slow queries (SQL clustering)
+uv run dbs-vector search "SELECT * FROM users" --type sql --min-time 1000
+```
+
+> **Indexes are built automatically at the end of every `ingest` run.** Two indexes are created:
+> - **IVF_PQ** vector index (only when the table has > 256 rows)
+> - **Tantivy FTS** inverted index (required for hybrid search)
+>
+> If you see a *"Cannot perform full text search unless an INVERTED index has been created"* error, it means the FTS index was never built for your table. Fix it by re-running ingestion — use `--rebuild` to wipe and re-index from scratch:
+> ```bash
+> uv run dbs-vector ingest "docs/" --rebuild
+> uv run dbs-vector ingest "slow_queries.json" --type sql --rebuild
+> ```
+
+For detailed specifications on each ingestion source, see:
+👉 **[SQL Engine Documentation](docs/README_SQL.md)**
+👉 **[DuckDB Ingestion Documentation](docs/README_duckdb.md)**
+👉 **[Remote SQL API Ingestion](docs/README_REMOTE_SQL_API.md)**
+
+### Async API Server
+The application includes a high-performance FastAPI server to expose the search engine over HTTP.
+
+```bash
+# Start the API server (loads all engines defined in config.yaml)
+uv run dbs-vector serve
+```
+
+For full API specifications and swagger documentation, see:
+👉 **[API Usage & Documentation](docs/README_API.md)**
+
+### Model Context Protocol (MCP) Server
+`dbs-vector` includes a built-in MCP server compatible with Claude Desktop, Claude Code (CLI), and Cursor. Supports both **stdio** (no server required) and **Streamable HTTP** (shared instance, saves VRAM).
+
+```bash
+# stdio — each client spawns its own process
+uv run dbs-vector mcp
+
+# HTTP — one shared server for all clients
+uv run dbs-vector serve   # MCP endpoint: http://127.0.0.1:8000/mcp
+```
+
+For setup instructions for all clients and transport types, see:
+👉 **[MCP Server Documentation](docs/README_MCP.md)**
+
+## 🏗 Architecture & Roadmap
+
+`dbs-vector` is built upon strict **Clean Architecture** and **SOLID** principles. It utilizes a **Configuration-Driven Registry Pattern**, allowing new data engines (e.g., LibCST, Logs) to be added by simply updating `config.yaml` and registering new mappers/chunkers without modifying core orchestration logic.
+
+### Specialized Gemma Workflows
+The project is optimized for instruction-tuned models like `embeddinggemma`. It supports asymmetric task-based workflows defined in `config.yaml`:
+*   **Markdown (Search Result)**: Uses the `task: search result` prefix for queries and `title: none | text: ` for documents, maximizing retrieval accuracy for RAG.
+*   **SQL (Clustering)**: Uses the `task: clustering` prefix for both ingestion and search, enabling high-precision semantic grouping of logically similar slow queries.
+
+### Future Hardware Support (CUDA/TPU)
+Because the core RAG orchestration relies exclusively on the `IEmbedder` Protocol, the application is strictly hardware-agnostic at its core. While currently optimized for Apple Silicon via `MLXEmbedder`, future deployment to cloud GPUs or Linux environments simply requires implementing a new `CudaEmbedder` (using PyTorch/Transformers) that returns standard NumPy arrays. No changes to the ingestion, storage, or API layers are necessary to support new hardware accelerators. No access to a CUDA hardware at the moment.
+
+For a deep dive into the engineering, the Apache Arrow ingestion lifecycle, and the blueprint for AST/LibCST integration, see the official documentation:
+
+👉 **[Architecture & Engineering Documentation](docs/README.md)**
+
+## 🛠 Development
+
+To contribute to `dbs-vector`, the project utilizes `poethepoet` as a task runner and implements strict quality gates (Ruff & Mypy).
+
+```bash
+# Run the entire validation suite (Format, Lint, Typecheck, Pytest)
+uv run poe check
+
+# Run tests with coverage
+uv run poe test-cov
+```
+

dbs_vector-0.5.1/README.md

@@ -0,0 +1,149 @@
+# ⚡️ dbs-vector
+
+**A High-Performance, Arrow-Native Local Codebase Search Engine for Apple Silicon.**
+
+`dbs-vector` is a optimized Retrieval-Augmented Generation (RAG) search engine designed specifically for macOS (M-Series chips). It bypasses traditional Python serialization bottlenecks by utilizing Apple's Unified Memory Architecture (UMA) and pure Apache Arrow data pipelines.
+
+It enables lightning-fast, hybrid (Vector + Full-Text) search across your local codebase, entirely offline.
+
+---
+
+## ✨ Features
+
+*   **Zero-Copy Memory Pipelines**: Uses **MLX** to compute embeddings on the Mac GPU, casting the resulting tensors instantly into NumPy arrays via Unified Memory without costly `float` object instantiation.
+*   **Arrow-Native Storage**: Uses **LanceDB** to stream ingestion batches directly to disk via PyArrow, avoiding the massive memory overhead of JSON and dictionary comprehensions.
+*   **Hybrid Retrieval**: Simultaneously executes Approximate Nearest Neighbor (ANN) cosine vector search and native **Tantivy** Full-Text Search (FTS).
+*   **Code-Aware Chunking**: Intelligently splits documentation and code, respecting markdown fences so that code blocks are indexed as atomic units.
+*   **Production Robustness**: Features dynamic `IVF_PQ` indexing, Rust-level predicate pushdown (metadata filtering), and dataset compaction for delta-updates.
+*   **Remote SQL API Ingestion**: `ApiChunker` pulls pre-aggregated slow-query records from any networked backend over HTTP, replacing local files with a paginated REST API — no changes to the embedding or storage layers.
+
+## 🚀 Installation
+
+This project is built using `uv`, an extremely fast Python package manager.
+
+1. **Clone the repository:**
+   ```bash
+   git clone https://github.com/dbsmedya/dbs-vector.git
+   cd dbs-vector
+   ```
+
+2. **Install the CLI package:**
+   ```bash
+   uv sync
+   ```
+   *This automatically sets up the environment and creates the `dbs-vector` executable in your path.*
+
+   Optional extras unlock additional ingestion sources:
+
+   ```bash
+   uv sync --extra sql  # DuckDB ingestion
+   uv sync --extra api  # Remote HTTP API ingestion
+   ```
+
+## 💻 Usage
+
+The application is entirely configuration-driven via `config.yaml`. It supports multiple data types (Engines) such as Markdown and SQL.
+
+### Global Options
+*   `--config-file` / `-c`: Path to your custom `config.yaml` (Defaults to `./config.yaml`).
+
+### Ingesting Documents
+Index markdown files, JSON SQL logs, DuckDB analytical files, or a remote HTTP slow-query API into the local vector store.
+
+```bash
+# Ingest all markdown files (default)
+uv run dbs-vector ingest "docs/"
+
+# Ingest SQL slow query logs (JSON format)
+uv run dbs-vector ingest "slow_queries.json" --type sql
+
+# Ingest SQL slow queries from DuckDB (High-Performance Columnar)
+uv run dbs-vector ingest "slow_queries.duckdb" --type sql --rebuild
+
+# Ingest from a remote HTTP API (paginated GET)
+uv run dbs-vector ingest "https://slow-log-api.internal/api/v1" --type sql-api
+
+# Ingest via a custom SELECT sent to the remote API
+uv run dbs-vector ingest "https://slow-log-api.internal/api/v1" --type sql-api \
+  --query "SELECT fingerprint_id AS id, sanitized_sql AS text, db AS source, ..."
+```
+
+### Searching the Codebase
+Execute queries against your chosen engine.
+
+```bash
+# Semantic hybrid search across markdown
+uv run dbs-vector search "What is MLX?"
+
+# Find similar slow queries (SQL clustering)
+uv run dbs-vector search "SELECT * FROM users" --type sql --min-time 1000
+```
+
+> **Indexes are built automatically at the end of every `ingest` run.** Two indexes are created:
+> - **IVF_PQ** vector index (only when the table has > 256 rows)
+> - **Tantivy FTS** inverted index (required for hybrid search)
+>
+> If you see a *"Cannot perform full text search unless an INVERTED index has been created"* error, it means the FTS index was never built for your table. Fix it by re-running ingestion — use `--rebuild` to wipe and re-index from scratch:
+> ```bash
+> uv run dbs-vector ingest "docs/" --rebuild
+> uv run dbs-vector ingest "slow_queries.json" --type sql --rebuild
+> ```
+
+For detailed specifications on each ingestion source, see:
+👉 **[SQL Engine Documentation](docs/README_SQL.md)**
+👉 **[DuckDB Ingestion Documentation](docs/README_duckdb.md)**
+👉 **[Remote SQL API Ingestion](docs/README_REMOTE_SQL_API.md)**
+
+### Async API Server
+The application includes a high-performance FastAPI server to expose the search engine over HTTP.
+
+```bash
+# Start the API server (loads all engines defined in config.yaml)
+uv run dbs-vector serve
+```
+
+For full API specifications and swagger documentation, see:
+👉 **[API Usage & Documentation](docs/README_API.md)**
+
+### Model Context Protocol (MCP) Server
+`dbs-vector` includes a built-in MCP server compatible with Claude Desktop, Claude Code (CLI), and Cursor. Supports both **stdio** (no server required) and **Streamable HTTP** (shared instance, saves VRAM).
+
+```bash
+# stdio — each client spawns its own process
+uv run dbs-vector mcp
+
+# HTTP — one shared server for all clients
+uv run dbs-vector serve   # MCP endpoint: http://127.0.0.1:8000/mcp
+```
+
+For setup instructions for all clients and transport types, see:
+👉 **[MCP Server Documentation](docs/README_MCP.md)**
+
+## 🏗 Architecture & Roadmap
+
+`dbs-vector` is built upon strict **Clean Architecture** and **SOLID** principles. It utilizes a **Configuration-Driven Registry Pattern**, allowing new data engines (e.g., LibCST, Logs) to be added by simply updating `config.yaml` and registering new mappers/chunkers without modifying core orchestration logic.
+
+### Specialized Gemma Workflows
+The project is optimized for instruction-tuned models like `embeddinggemma`. It supports asymmetric task-based workflows defined in `config.yaml`:
+*   **Markdown (Search Result)**: Uses the `task: search result` prefix for queries and `title: none | text: ` for documents, maximizing retrieval accuracy for RAG.
+*   **SQL (Clustering)**: Uses the `task: clustering` prefix for both ingestion and search, enabling high-precision semantic grouping of logically similar slow queries.
+
+### Future Hardware Support (CUDA/TPU)
+Because the core RAG orchestration relies exclusively on the `IEmbedder` Protocol, the application is strictly hardware-agnostic at its core. While currently optimized for Apple Silicon via `MLXEmbedder`, future deployment to cloud GPUs or Linux environments simply requires implementing a new `CudaEmbedder` (using PyTorch/Transformers) that returns standard NumPy arrays. No changes to the ingestion, storage, or API layers are necessary to support new hardware accelerators. No access to a CUDA hardware at the moment.
+
+For a deep dive into the engineering, the Apache Arrow ingestion lifecycle, and the blueprint for AST/LibCST integration, see the official documentation:
+
+👉 **[Architecture & Engineering Documentation](docs/README.md)**
+
+## 🛠 Development
+
+To contribute to `dbs-vector`, the project utilizes `poethepoet` as a task runner and implements strict quality gates (Ruff & Mypy).
+
+```bash
+# Run the entire validation suite (Format, Lint, Typecheck, Pytest)
+uv run poe check
+
+# Run tests with coverage
+uv run poe test-cov
+```
+

dbs_vector-0.5.1/config.yaml

@@ -0,0 +1,52 @@
+system:
+  db_path: "./lancedb_dbs_vector"
+  batch_size: 64
+  nprobes: 20
+
+engines:
+  md:
+    description: "Markdown & Prose Document Engine (Gemma Search)"
+    model_name: "mlx-community/embeddinggemma-300m-bf16"
+    vector_dimension: 768
+    max_token_length: 2048
+    table_name: "knowledge_vault"
+    mapper_type: "document"
+    chunker_type: "document"
+    chunk_max_chars: 1000
+    passage_prefix: "title: none | text: "
+    query_prefix: "task: search result | query: "
+    workflow: "md_search"
+
+  sql:
+    description: "SQL Slow Query Log Engine (Gemma Clustering)"
+    model_name: "mlx-community/embeddinggemma-300m-bf16"
+    vector_dimension: 768
+    max_token_length: 2048
+    table_name: "query_vault"
+    mapper_type: "sql"
+    chunker_type: "duckdb"
+    chunk_max_chars: 0 
+    passage_prefix: "task: clustering | query: "
+    query_prefix: "task: clustering | query: "
+    workflow: "sql_clustering"
+
+  sql-api:
+    description: "Remote slow query log via HTTP API"
+    model_name: "mlx-community/embeddinggemma-300m-bf16"
+    vector_dimension: 768
+    max_token_length: 2048
+    table_name: "query_vault"
+    mapper_type: "sql"
+    chunker_type: "api"
+    chunk_max_chars: 0
+    passage_prefix: "task: clustering | query: "
+    query_prefix: "task: clustering | query: "
+    workflow: "sql_clustering"
+    # --- ApiChunker-specific fields ---
+    api_base_url: "http://localhost:8080/api/v1"
+    api_key: "0Byuf9P9e5UxUNIsngNjr6b9u8sldoHd1ek_ImBbxiI"          # set via DBS_API_KEY env var in production
+    api_page_size: 200         # records per GET request (max 1000)
+    api_since_days: 60         # lower bound on latest_ts  (default: 15)
+    api_timeout_sec: 30        # HTTP request timeout in seconds
+    api_min_execution_ms: 0    # filter: skip queries below this threshold
+    api_database: ""           # leave empty to fetch all databases