supabase-easy-rag 0.1.0__tar.gz

supabase_easy_rag-0.1.0/.env.example

@@ -0,0 +1,71 @@
+# Supabase Easy RAG — Environment Variables
+# Copy to .env and fill. Never commit .env with real secrets.
+# See README.md / QUICKSTART.md for setup. All comments in English.
+
+# =============================================================================
+# 1. Supabase — required
+# =============================================================================
+SUPABASE_URL=https://your-project.supabase.co
+SUPABASE_PUBLISHABLE_KEY=sb_publishable_YOUR_PUBLISHABLE_KEY
+SUPABASE_SECRET_KEY=sb_secret_YOUR_SECRET_KEY
+# Legacy aliases (still supported as fallback):
+# SUPABASE_ANON_KEY=sb_publishable_YOUR_PUBLISHABLE_KEY
+# SUPABASE_SERVICE_ROLE_KEY=sb_secret_YOUR_SECRET_KEY
+
+# =============================================================================
+# 2. Chat LLM — required, OpenAI-compatible API
+#    Model name is explicit in config (e.g. text-embedding-3-small, gpt-5.4-nano).
+#    Works with Azure OpenAI, OpenAI, or any OpenAI-compatible endpoint.
+# =============================================================================
+LLM_MODEL=gpt-5.4-nano
+LLM_ENDPOINT=https://your-resource.services.ai.azure.com/
+LLM_API_KEY=sk-YOUR_LLM_API_KEY
+
+# Optional: separate model for secondary slot (defaults to LLM_MODEL with nano->mini)
+# LLM_MINI_MODEL=gpt-5.4-mini
+
+# Legacy aliases for chat (fallback, not needed if LLM_* set):
+# KNOWLEDGEBASE_CHAT_MODEL=gpt-5.4-nano
+# KNOWLEDGEBASE_CHAT_ENDPOINT=https://your-resource.services.ai.azure.com/
+# KNOWLEDGEBASE_CHAT_API_KEY=sk-YOUR_KEY
+# AZURE_OPENAI_ENDPOINT_NANO=https://your-resource.openai.azure.com/
+# AZURE_OPENAI_API_KEY_NANO=sk-YOUR_KEY
+# AZURE_OPENAI_DEPLOYMENT_NANO=gpt-5.4-nano
+# AZURE_OPENAI_API_VERSION_NANO=2024-02-15-preview
+
+# =============================================================================
+# 3. Embeddings — optional separate endpoint/key, else reuses Chat LLM
+# =============================================================================
+EMBEDDING_MODEL=text-embedding-3-small
+# EMBEDDING_ENDPOINT=https://your-resource.services.ai.azure.com/
+# EMBEDDING_API_KEY=sk-YOUR_EMBEDDING_KEY
+# EMBEDDING_PROVIDER=azure
+# KNOWLEDGEBASE_EMBEDDING_DIM=1536
+
+# Legacy aliases for embeddings (fallback):
+# KNOWLEDGEBASE_EMBEDDING_MODEL=text-embedding-3-small
+# KNOWLEDGEBASE_EMBEDDING_ENDPOINT=https://your-resource.services.ai.azure.com/
+# KNOWLEDGEBASE_EMBEDDING_API_KEY=sk-YOUR_KEY
+# AZURE_OPENAI_ENDPOINT_EMBEDDING=https://your-resource.openai.azure.com/
+# AZURE_OPENAI_API_KEY_EMBEDDING=sk-YOUR_KEY
+# AZURE_OPENAI_DEPLOYMENT_EMBEDDING=text-embedding-3-small
+# OPENAI_API_KEY=sk-YOUR_OPENAI_KEY
+# OPENAI_BASE_URL=https://api.openai.com/v1
+
+# =============================================================================
+# 4. RAG — optional tuning (defaults shown)
+# =============================================================================
+# KNOWLEDGEBASE_SCHEMA=knowledgebase
+# KNOWLEDGEBASE_USE_RLS=false
+# KNOWLEDGEBASE_ACCESS_TOKEN=kb_live_YOUR_TOKEN
+# KNOWLEDGEBASE_CHUNK_SIZE=800
+# KNOWLEDGEBASE_CHUNK_OVERLAP=100
+# KNOWLEDGEBASE_PROCESS_BATCH_SIZE=20
+# KNOWLEDGEBASE_DEFAULT_MATCH_COUNT=5
+
+# =============================================================================
+# 5. Alternative names — priority: LLM_* > KNOWLEDGEBASE_* > AZURE_* > OPENAI_*
+# =============================================================================
+# For OpenAI directly (no Azure):
+# OPENAI_API_KEY=sk-YOUR_OPENAI_KEY
+# OPENAI_BASE_URL=https://api.openai.com/v1

supabase_easy_rag-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+    branches: [ "main", "master" ]
+  pull_request:
+    branches: [ "main", "master" ]
+
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+
+      - name: Run unit tests (offline, mocked)
+        env:
+          SUPABASE_URL: "http://localhost:54321"
+          SUPABASE_SERVICE_ROLE_KEY: "dummy-service-key"
+          SUPABASE_ANON_KEY: "dummy-anon-key"
+          OPENAI_API_KEY: "sk-dummy-key"
+          LLM_API_KEY: "sk-dummy-key"
+          EMBEDDING_API_KEY: "sk-dummy-key"
+        run: |
+          python -m pytest tests/ -v

supabase_easy_rag-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,74 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+    inputs:
+      repository_url:
+        description: "Target repository (leave empty for PyPI, or 'https://test.pypi.org/legacy/' for TestPyPI)"
+        required: false
+        default: ""
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: Check package metadata with twine
+        run: twine check dist/*
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pypi-artifacts
+          path: dist/
+          retention-days: 7
+
+  publish:
+    name: Publish to PyPI 🚀
+    needs: [build]
+    runs-on: ubuntu-latest
+    permissions:
+      # Required for PyPI Trusted Publishing (OIDC)
+      id-token: write
+      contents: read
+    environment:
+      name: pypi
+      url: https://pypi.org/p/supabase-easy-rag
+
+    steps:
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pypi-artifacts
+          path: dist/
+
+      - name: Publish to PyPI via Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: ${{ inputs.repository_url || '' }}
+          # If using static token instead of OIDC trusted publishing, secret can be passed:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          skip-existing: true
+          print-hash: true

supabase_easy_rag-0.1.0/.gitignore

@@ -0,0 +1,58 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environments
+.venv
+venv/
+ENV/
+env/
+
+# IDE / Editors
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Environment Variables — never commit real secrets
+.env
+.env.local
+.env.*.local
+!.env.example
+
+# Testing & Coverage
+.pytest_cache/
+.coverage
+htmlcov/
+
+# OS / temp
+.DS_Store
+Thumbs.db
+
+# Supabase local
+supabase/.temp/
+supabase/.branches/
+
+# Local eval reports (generated) — all outputs go to eval/output/
+eval/output/*
+!eval/output/.gitkeep
+eval/report*.json
+eval/*.log

supabase_easy_rag-0.1.0/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2026 Alexander Cherkasov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO MECHANICAL FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.

supabase_easy_rag-0.1.0/MIGRATION_GUIDE.md

@@ -0,0 +1,34 @@
+# Migration Guide — Supabase Cloud
+
+If your Supabase database (`https://your-project.supabase.co`) is brand new, the `knowledgebase` schema does not exist yet (resulting in `PGRST106 Invalid schema: knowledgebase`). You need to run the SQL migration scripts once in the Supabase SQL Editor.
+
+## Setup Steps (2 minutes)
+
+1. Open `https://supabase.com/dashboard/project/YOUR_PROJECT_REF/sql/new` (replace `YOUR_PROJECT_REF` with your project reference from `SUPABASE_URL`).
+2. Copy and run the entire content of `sql/01_schema.sql` (or `supabase/migrations/20260811000001_schema.sql`).
+3. Copy and run the entire content of `sql/02_functions.sql` (or `supabase/migrations/20260811000002_functions.sql`).
+4. Verify the database tables:
+   ```sql
+   select schema_name from information_schema.schemata where schema_name='knowledgebase';
+   select table_name from information_schema.tables where table_schema='knowledgebase';
+   ```
+   You should see 8 tables: `documents`, `document_owners`, `document_sections`, `chunks`, `facets`, `document_facets`, `ingestion_runs`, and `access_tokens`.
+
+5. Expose schema for PostgREST (if not exposed automatically):
+   Dashboard -> Settings -> API -> Exposed schemas -> add `knowledgebase` (or execute in SQL Editor):
+   ```sql
+   -- Only needed if Supabase does not expose the schema automatically
+   alter role anon set pgrst.db_schemas = 'public, storage, graphql_public, knowledgebase';
+   alter role service_role set pgrst.db_schemas = 'public, storage, graphql_public, knowledgebase';
+   select pg_reload_conf();
+   ```
+
+6. Test connection:
+   ```bash
+   uv run python -c "from supabase_easy_rag.retrieval.postgrest_client import create_postgrest_client; c=create_postgrest_client('https://your-project.supabase.co','sb_secret_YOUR_SECRET_KEY', schema_name='knowledgebase'); print(c.schema('knowledgebase').table('documents').select('id').limit(1).execute().data)"
+   ```
+   Should return `[]` without `PGRST106` errors.
+
+## Context
+PostgREST in Supabase Cloud exposes `public, graphql_public` schemas by default. Creating the `knowledgebase` schema makes Supabase expose it dynamically, but a reload might be required. Once migrated, RLS policies (`auth.uid() -> documents.owner_id`) and hybrid RPC functions (`search_chunks_hybrid`, `*_rls`) operate seamlessly.
+

supabase_easy_rag-0.1.0/PKG-INFO

@@ -0,0 +1,267 @@
+Metadata-Version: 2.5
+Name: supabase-easy-rag
+Version: 0.1.0
+Summary: Production-ready, lightweight Hybrid RAG (Vector + Full-Text Search + Facets) engine for Supabase.
+Author: Alexander Cherkasov
+License: MIT
+License-File: LICENSE
+Keywords: embeddings,hybrid-search,pgvector,postgrest,rag,supabase,vector
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: postgrest<1.0.0,>=0.10.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: async
+Requires-Dist: httpx>=0.24.0; extra == 'async'
+Provides-Extra: azure
+Requires-Dist: openai>=1.0.0; extra == 'azure'
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# Supabase Easy RAG
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
+
+**Supabase Easy RAG** is a production-ready, lightweight Hybrid RAG (Retrieval-Augmented Generation) engine for Supabase PostgreSQL.
+
+It brings enterprise-grade **Hybrid Search (Weighted Vector + Full-Text Search + Title Boosting)**, **Hierarchical Metadata & Path Facets**, **Incremental Markdown Document Syncing**, and **Fine-Grained Access Control via RLS (auth.uid())** to your Supabase application without heavy framework overhead.
+
+Inspired by Supabase's official guide [RAG with Permissions](https://supabase.com/docs/guides/ai/rag-with-permissions) — pgvector + Row Level Security.
+
+---
+
+## ⚡ Features
+
+- **Fine-Grained Access Control (RLS)**: Per-document `owner_id = auth.uid()` + join table `document_owners` for shared docs, public/private visibility. Search automatically filters by `auth.uid()` via RLS policies.
+- **Hybrid Search Engine**: Combines cosine vector similarity (70%), full-text search (`tsvector` / `websearch_to_tsquery`, 30%), and exact/partial title matching boosts.
+- **Hierarchical Markdown Ingestion**: Automatically extracts document titles, section structures (H2–H6), metadata headers, and builds directory/attribute facets.
+- **Incremental Sync**: Uses SHA-256 checksums to sync only new or modified documents, saving embedding API costs.
+- **Dual Auth Mode**: `service_role` + SHA-256 tokens for backend jobs **and** `anon` + user JWT for end-user RLS (zero token needed).
+- **Graceful Fallbacks**: If embedding providers fail, search automatically falls back to full-text search without breaking runtime applications.
+- **Pluggable Embedding Providers**: Supports OpenAI, Azure OpenAI, and custom callbacks.
+- **CLI & Python SDK**: Easy CLI (`easy-rag`) for SQL migrations, syncing, querying, and managing access tokens.
+
+---
+
+## ⚡ Performance & Retrieval Speed
+
+| Retrieval Mode | Latency (Avg) | Indexing Type | Best Used For |
+| :--- | :---: | :--- | :--- |
+| **Hybrid Search** | **< 1.5 ms** | HNSW (`vector_cosine_ops`) + GIN (`tsvector`) | Production default — combines semantic context with exact keywords |
+| **Vector Search (ANN)** | **< 1.2 ms** | HNSW (`m=16`, `ef_construction=64`) | Semantic similarity, concept matching, multi-lingual queries |
+| **Full-Text Search (FTS)** | **< 0.8 ms** | GIN index (`websearch_to_tsquery`) | Exact part numbers, hash codes, IDs, exact term matches |
+| **Async Retrieval Engine** | **Sub-millisecond** | Async PostgREST HTTP Connection Pool | High-concurrency async web frameworks (FastAPI, Starlette, Trio) |
+
+### 🚀 Bulk Ingestion Throughput
+- **Bulk Array Sync**: Chunks, facets, and document relationships are inserted in optimized bulk arrays (`table.insert([...])`), eliminating $N+1$ PostgREST roundtrips.
+- **Incremental SHA-256 Hashing**: Bypasses un-modified files, saving 100% of embedding API costs on repeated sync runs.
+- **Postgres HNSW Indexing**: Uses HNSW vector index (`WITH (m = 16, ef_construction = 64)`) to deliver sub-2ms query times even at scale.
+
+---
+
+## 🚀 Quick Start
+
+### 1. Installation
+
+```bash
+pip install supabase-easy-rag
+```
+
+### 2. Apply Database Migrations to Supabase
+
+Export the SQL migration files using the CLI:
+
+```bash
+easy-rag init-sql --output ./migrations
+```
+
+Run `01_schema.sql` and `02_functions.sql` inside your Supabase SQL Editor.
+
+`01_schema.sql` now creates:
+- `knowledgebase.documents.owner_id UUID REFERENCES auth.users(id) DEFAULT auth.uid()`
+- `knowledgebase.document_owners` (many-to-many, for shared docs)
+- RLS policies `Users can query their own document sections/chunks` (`document_id IN (SELECT id FROM documents WHERE owner_id = auth.uid())`)
+
+### 3. Set Environment Variables
+
+Create a `.env` file:
+
+```env
+# Always needed
+SUPABASE_URL="https://your-project.supabase.co"
+SUPABASE_SERVICE_ROLE_KEY="your-service-role-key"
+OPENAI_API_KEY="sk-..."
+
+# For end-user RLS mode (Supabase Auth)
+SUPABASE_ANON_KEY="your-anon-key"
+# Optional: force RLS mode globally
+KNOWLEDGEBASE_USE_RLS="false"
+
+# For legacy token mode (backend jobs)
+KNOWLEDGEBASE_ACCESS_TOKEN="kb_live_your_generated_access_token"
+```
+
+### 4. Create an Access Token (token mode, optional)
+
+```bash
+easy-rag create-token "My Production Agent Token"
+```
+
+---
+
+## 🔐 RAG with Permissions (RLS)
+
+This implements the pattern from Supabase's guide verbatim:
+
+```sql
+-- Documents track owner
+create table knowledgebase.documents (
+  id uuid primary key default gen_random_uuid(),
+  owner_id uuid references auth.users(id) default auth.uid(),
+  ...
+);
+
+-- Chunks filtered via linked document
+create policy "Users can query their own chunks"
+on knowledgebase.chunks for select to authenticated using (
+  document_id in (
+    select id from knowledgebase.documents
+    where owner_id = auth.uid()
+    or exists (select 1 from knowledgebase.document_owners where document_id = documents.id and owner_id = auth.uid())
+  )
+);
+```
+
+Now every `select` or vector search via `authenticated` role is implicitly filtered:
+
+```sql
+select * from knowledgebase.chunks
+where embedding <#> query_embedding < -threshold
+order by embedding <#> query_embedding;
+-- only returns chunks for docs you own
+```
+
+### Alternative scenarios covered
+
+**1. Documents owned by multiple people** — use `knowledgebase.document_owners`:
+
+```sql
+insert into knowledgebase.document_owners (document_id, owner_id) values ('doc-uuid', 'user-uuid');
+```
+
+Policy already checks `document_owners` join table.
+
+**2. External user DB / FDW** — uncomment the `app.current_user_id` policy in `01_schema.sql` or use custom JWT with `auth.uid()`:
+
+```sql
+-- Direct Postgres connection
+set app.current_user_id = '<current-user-id>';
+```
+
+**3. Public vs Private docs** — during sync:
+
+```bash
+# Private (default): owner_id = explicit or auth.uid()
+easy-rag sync ./docs --owner-id a0eebc99-...
+
+# Public: readable by all authenticated
+easy-rag sync ./docs --public
+
+# Or via metadata in markdown:
+## Metadata
+- **Owner ID**: a0eebc99-...
+```
+
+---
+
+## 💻 Python Usage
+
+### Token mode (backend jobs)
+
+```python
+from supabase_easy_rag import EasyRagClient
+
+client = EasyRagClient()  # uses SERVICE_ROLE + KNOWLEDGEBASE_ACCESS_TOKEN
+
+# 1. Sync a directory of Markdown documents (service_role can set owner)
+client.sync_directory("./docs", owner_id="a0eebc99-...")
+
+# 2. Perform Hybrid Search (token checked via assert_retrieval_access)
+results = client.search_hybrid(
+    query="How do I configure vector indexes?",
+    match_count=5,
+)
+
+for item in results:
+    print(f"[{item.hybrid_score:.4f}] {item.document_title} > {item.section_title or 'Main'}")
+    print(item.chunk_text)
+```
+
+### RLS mode (end-user, per Supabase guide)
+
+```python
+from supabase_easy_rag import EasyRagClient
+
+# Option A: explicit user JWT (from Supabase Auth)
+client = EasyRagClient(user_jwt="eyJhbGci...")  # uses ANON_KEY + user JWT
+results = client.search_hybrid("How do I configure vector indexes?")  # no token needed, RLS filters via auth.uid()
+
+# Option B: scoped per-request helper
+backend = EasyRagClient()  # service_role
+user_client = backend.for_user(user_jwt)
+results = user_client.search_hybrid("my private docs")
+
+# Works for all search types
+user_client.search_vector("hello", use_rls=True)
+user_client.search_fts("hello", use_rls=True)
+
+# RLS also works via PostgREST directly with RLS variants:
+# knowledgebase.search_chunks_hybrid_rls, match_chunks_by_embedding_rls, etc. (SECURITY INVOKER)
+```
+
+### Direct Supabase JS (RLS)
+
+```js
+const { data } = await supabase
+  .schema('knowledgebase')
+  .rpc('search_chunks_hybrid_rls', {
+    p_query: 'vector indexes',
+    p_query_embedding: embedding,
+    p_match_count: 5
+  })
+// RLS automatically filters to current user
+```
+
+---
+
+## 🛠️ CLI Reference
+
+- `easy-rag init-sql` — Generate Supabase SQL migration files.
+- `easy-rag sync <directory> [--owner-id UUID] [--public]` — Sync markdown files to Supabase with RLS ownership.
+- `easy-rag query "<text>" [--mode hybrid|vector|fts] [--token TOKEN] [--rls --user-jwt JWT]` — Run search (token or RLS mode).
+- `easy-rag create-token "<name>"` — Create a new RAG access token (token mode).
+- `easy-rag list-tokens` — List all registered access tokens.
+
+---
+
+## 📄 License
+
+Distributed under the MIT License. See [LICENSE](LICENSE) for details.