rsc-universal 0.1.1

package/skills/unit-economics/scripts/verify.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+#
+# verify.sh - re-derive a unit-economics worksheet and fail if the arithmetic lies.
+#
+# Usage:
+#   cd <dir containing unit-economics.yaml|.csv|.md>
+#   ./verify.sh [path-to-worksheet]
+#
+# Read-only: never mutates anything. Parses named inputs/outputs and checks:
+#   - cac                 == period_sm_spend / new_customers
+#   - contribution_margin == monthly_arpa * gross_margin_pct
+#   - payback_months      == cac / (monthly_arpa * gross_margin_pct)
+#   - ltv  uses gross-margin form (ARPA*GM)/churn, NOT revenue form ARPA/churn
+#   - ltv_cac             == ltv / cac          (within 0.05)
+#   - implied lifetime (1/monthly_churn) <= 48 months unless lifetime_cap_override set
+#
+# Tolerance: +/-0.5% on money figures, +/-0.05 on ratios.
+# Exits 0 on a clean worksheet AND on no worksheet found (no false failure).
+# Exits 1 with the specific failing check on any violation.
+#
+# Portable: stock macOS bash 3.2 + awk. No associative arrays, no mapfile.
+
+set -euo pipefail
+
+if [ -t 1 ]; then
+  RED=$'\033[31m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m'; RESET=$'\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; RESET=''
+fi
+
+fail() { printf '%s[fail]%s %s\n' "$RED" "$RESET" "$*"; failed=1; }
+ok()   { printf '%s[ ok ]%s %s\n' "$GREEN" "$RESET" "$*"; }
+
+# --- locate the worksheet ----------------------------------------------------
+file="${1:-}"
+if [ -z "$file" ]; then
+  for cand in unit-economics.yaml unit-economics.yml unit-economics.csv unit-economics.md; do
+    if [ -f "$cand" ]; then file="$cand"; break; fi
+  done
+fi
+
+if [ -z "$file" ] || [ ! -f "$file" ]; then
+  printf '%s[skip]%s no unit-economics.{yaml,csv,md} found - nothing to verify.\n' "$YELLOW" "$RESET"
+  exit 0
+fi
+
+# --- extract a numeric field by key (yaml-ish, csv-ish, or "key: value") -----
+# Matches lines like "  cac: 1500", "cac,1500", "cac = 1500", "| cac | 1500 |".
+# Strips $ , % and surrounding noise; takes the first number after the key.
+getnum() {
+  key="$1"
+  awk -v k="$key" '
+    { line=$0
+      # normalise separators to spaces, drop currency/percent/comma noise
+      gsub(/[,$%|{}\[\]]/, " ", line)
+      gsub(/[:=]/, " ", line)
+    }
+    line ~ ("(^|[^a-zA-Z_])" k "([^a-zA-Z_]|$)") {
+      n=split(line, a, " ")
+      seen=0
+      for (i=1;i<=n;i++){
+        if (a[i]==k){ seen=1; continue }
+        if (seen && a[i] ~ /^-?[0-9]+(\.[0-9]+)?$/){ print a[i]; exit }
+      }
+    }
+  ' "$file"
+}
+
+spend=$(getnum period_sm_spend)
+newc=$(getnum new_customers)
+arpa=$(getnum monthly_arpa)
+gm=$(getnum gross_margin_pct)
+churn=$(getnum monthly_churn)
+override=$(getnum lifetime_cap_override)
+
+cac=$(getnum cac)
+cm=$(getnum contribution_margin)
+ltv=$(getnum ltv)
+payback=$(getnum payback_months)
+ltvcac=$(getnum ltv_cac)
+
+failed=0
+
+require() { # name value
+  if [ -z "$2" ]; then fail "missing required input '$1' in $file"; fi
+}
+require period_sm_spend "$spend"
+require new_customers "$newc"
+require monthly_arpa "$arpa"
+require gross_margin_pct "$gm"
+require monthly_churn "$churn"
+if [ "$failed" -ne 0 ]; then
+  printf '\n%sFAIL:%s worksheet is missing inputs needed to verify.\n' "$RED" "$RESET"
+  exit 1
+fi
+
+# gross_margin_pct may be given as 0.75 or 75; normalise to a fraction.
+gm=$(awk -v g="$gm" 'BEGIN{ if (g>1.0001) g=g/100; printf "%.6f", g }')
+
+printf -- '----- verifying %s\n' "$file"
+
+# helper: are two numbers within tolerance? tol is a fraction of |expected|.
+close() { # actual expected tol
+  awk -v a="$1" -v e="$2" -v t="$3" 'BEGIN{
+    d=a-e; if(d<0)d=-d;
+    lim=e; if(lim<0)lim=-lim;
+    lim=lim*t; if(lim<0.0001)lim=0.0001;   # floor so exact zeros still compare
+    exit (d<=lim)?0:1
+  }'
+}
+
+exp_cac=$(awk -v s="$spend" -v n="$newc" 'BEGIN{ if(n==0){print "NaN"} else printf "%.6f", s/n }')
+exp_cm=$(awk  -v a="$arpa" -v g="$gm" 'BEGIN{ printf "%.6f", a*g }')
+exp_pay=$(awk -v c="$exp_cac" -v a="$arpa" -v g="$gm" 'BEGIN{ d=a*g; if(d==0){print "NaN"} else printf "%.6f", c/d }')
+exp_ltv=$(awk -v a="$arpa" -v g="$gm" -v ch="$churn" 'BEGIN{ if(ch==0){print "NaN"} else printf "%.6f", (a*g)/ch }')
+rev_ltv=$(awk -v a="$arpa" -v ch="$churn" 'BEGIN{ if(ch==0){print "NaN"} else printf "%.6f", a/ch }')
+
+# 1. CAC
+if [ -n "$cac" ]; then
+  if close "$cac" "$exp_cac" 0.005; then ok "CAC $cac == spend/new ($exp_cac)";
+  else fail "CAC $cac != spend/new_customers ($exp_cac)"; fi
+else
+  fail "stated output 'cac' missing"
+fi
+
+# 2. contribution margin
+if [ -n "$cm" ]; then
+  if close "$cm" "$exp_cm" 0.005; then ok "contribution_margin $cm == ARPA*GM ($exp_cm)";
+  else fail "contribution_margin $cm != ARPA*GM% ($exp_cm)"; fi
+fi
+
+# 3. payback
+if [ -n "$payback" ]; then
+  if close "$payback" "$exp_pay" 0.005; then ok "payback_months $payback == CAC/(ARPA*GM) ($exp_pay)";
+  else fail "payback_months $payback != CAC/(ARPA*GM%) ($exp_pay)"; fi
+fi
+
+# 4. LTV must be gross-margin form, not revenue form
+if [ -n "$ltv" ]; then
+  if close "$ltv" "$exp_ltv" 0.005; then
+    ok "LTV $ltv == (ARPA*GM)/churn ($exp_ltv)"
+  elif close "$ltv" "$rev_ltv" 0.005; then
+    fail "LTV $ltv uses REVENUE form ARPA/churn ($rev_ltv); use gross-margin (ARPA*GM)/churn ($exp_ltv)"
+  else
+    fail "LTV $ltv != (ARPA*GM)/churn ($exp_ltv)"
+  fi
+fi
+
+# 5. LTV:CAC
+if [ -n "$ltvcac" ] && [ -n "$cac" ] && [ -n "$ltv" ]; then
+  exp_ratio=$(awk -v l="$ltv" -v c="$cac" 'BEGIN{ if(c==0){print "NaN"} else printf "%.6f", l/c }')
+  # absolute 0.05 tolerance on the ratio
+  if awk -v a="$ltvcac" -v e="$exp_ratio" 'BEGIN{ d=a-e; if(d<0)d=-d; exit (d<=0.05)?0:1 }'; then
+    ok "ltv_cac $ltvcac == LTV/CAC ($exp_ratio)"
+  else
+    fail "ltv_cac $ltvcac != LTV/CAC ($exp_ratio)"
+  fi
+fi
+
+# 6. lifetime cap
+life=$(awk -v ch="$churn" 'BEGIN{ if(ch==0){print "999999"} else printf "%.2f", 1/ch }')
+cap=48
+if [ -n "$override" ]; then cap="$override"; fi
+if awk -v l="$life" -v c="$cap" 'BEGIN{ exit (l<=c)?0:1 }'; then
+  ok "implied lifetime ${life}mo <= cap ${cap}mo"
+else
+  if [ -n "$override" ]; then
+    fail "implied lifetime ${life}mo exceeds override cap ${cap}mo"
+  else
+    fail "implied lifetime ${life}mo exceeds 48mo cap; set lifetime_cap_override to justify"
+  fi
+fi
+
+echo
+if [ "$failed" -ne 0 ]; then
+  printf '%sFAIL:%s worksheet arithmetic is inconsistent.\n' "$RED" "$RESET"
+  exit 1
+fi
+printf '%sPASS:%s worksheet is self-consistent.\n' "$GREEN" "$RESET"

package/skills/vector-db/SKILL.md

@@ -0,0 +1,189 @@
+---
+name: vector-db
+description: "Use when operating a vector store as a data layer — choosing or migrating between Pinecone, Qdrant, Weaviate, or pgvector; designing a collection/index (distance metric, dimensions, HNSW m/ef_construction, named vectors); filtering on metadata/payload; running hybrid dense+sparse search; or picking quantization to cut RAM and cost. Symptoms: search returns garbage, filters get silently ignored, recall is low, queries are slow, the bill is too high, a filtered query returns fewer than k rows. Triggers: 'set up a vector database', 'my similarity search ignores the metadata filter', 'ef_search vs recall', 'move off Pinecone to pgvector to cut cost', 'hybrid search with RRF', 'why does my filtered query return fewer than k results', 'crear una colección en Qdrant con cuantización binaria', 'la cerca vectorial retorna resultats dolents'. NOT how to produce, chunk, or judge embeddings (that is embeddings-search)."
+tags: [vector-database, pinecone, qdrant, weaviate, pgvector, hnsw, hybrid-search, metadata-filter, quantization]
+recommends: [embeddings-search, rag, postgresdb, redis, supabase]
+origin: risco
+---
+
+# vector-db — operate the store, not the embeddings
+
+You own the **store**: the collection schema, the index, the filter path, recall-vs-latency
+tuning, hybrid fusion, quantization, and the production knobs (upserts, namespaces, deletes,
+backups). You operate it across the four engines a Claude agent actually meets: **pgvector**
+(Postgres extension), **Qdrant**, **Weaviate**, **Pinecone** (serverless).
+
+Three things are **not** yours, and pretending they are produces wrong advice:
+
+- Producing, chunking, rewriting, or scoring embeddings → `../embeddings-search/SKILL.md`. You
+  store vectors; you do not make or judge them.
+- Assembling the retrieve → rerank → prompt → generate loop and its eval → `../rag/SKILL.md`.
+- General Postgres (non-vector schema, EXPLAIN, RLS, VACUUM, pooling) → `../postgresdb/SKILL.md`.
+  You own **only** the pgvector surface: the `vector`/`halfvec` column, its index, its operators,
+  its recall. "My Postgres is slow in general" is theirs; "my `<=>` query has low recall" is yours.
+
+## Pick the engine
+
+Match the engine to where the data already lives and how much ops you want to run. All four
+implement HNSW with comparable recall at a matched `ef`, so the differentiator is operations and
+hybrid, not raw quality.
+
+| Engine | Best when | Hybrid built-in | Ops cost | Scale sweet spot |
+|---|---|---|---|---|
+| **pgvector** | Data already in Postgres; one less system to run | No — DIY (`vector` + `tsvector`, combine yourself) | You already run Postgres | ≤ a few M vectors |
+| **Qdrant** | You want best filtered-search latency and self-host control | Yes — `query` API, `prefetch` + RRF/DBSF, server-side IDF | Self-host or cloud | 10M+ |
+| **Weaviate** | You want hybrid + modules out of the box | Yes — `alpha` + `fusionType`, BlockMax WAND BM25 | Self-host or cloud | 10M+ |
+| **Pinecone** | You refuse to operate anything | Yes — sparse-dense, integrated inference | Zero (serverless) | Any (pay per use) |
+
+Rule: **don't add a new system to host vectors if the rows already live in Postgres and you are
+under a few million.** pgvector is one extension, not a second database to back up and monitor.
+
+## Design the collection & index
+
+1. **Distance metric MUST match the embedding model.** A model trained for cosine, indexed with
+   L2, ranks *silently wrong* — no error, just bad results. OpenAI `text-embedding-3-*`, Cohere,
+   most sentence-transformers → cosine. pgvector operator cheatsheet:
+
+   ```text
+   <-> L2 / Euclidean   (vector_l2_ops)
+   <=> cosine distance  (vector_cosine_ops)   <- the common one
+   <#> negative inner product (vector_ip_ops) <- for normalized vectors
+   ```
+
+2. **Dimensions are fixed by the model**, not a choice. `text-embedding-3-small` = 1536,
+   `-3-large` = 3072. pgvector's `vector` caps at 2000 dims for an index; for more, use `halfvec`.
+
+3. **HNSW defaults and the one invariant.** Build-time `m` (default 16) and `ef_construction`
+   (default 64); query-time `ef_search` (pgvector default 40). Keep `ef_construction >= 2*m`
+   (so ≥32 at the default `m`) — too low starves graph quality and recall never recovers without
+   a rebuild. Raise `m` to 32–48 only for high-dim or high-recall needs (more RAM, slower build).
+
+4. **IVFFlat only when build speed beats recall.** It is cheaper to build but lower recall and
+   needs `lists`/`probes` tuning; on a selective filter it is the wrong default (see next section).
+   Prefer HNSW unless you have a measured reason.
+
+5. **Named vectors when one object has multiple spaces** (e.g. a dense semantic vector + a sparse
+   BM25 vector, or title-vector + body-vector). Qdrant and Weaviate support this natively; it is
+   how you do hybrid in one collection instead of two.
+
+## Metadata / payload filtering
+
+The #1 "search is broken" bug: the filter is applied **after** top-k, so a selective filter
+returns fewer than `k` rows (or zero). Fix it by filtering *inside* the search and indexing the
+filter field.
+
+```text
+Bad:  ANN top-k=10, THEN drop rows where tenant_id != 'acme'  -> often < 10, sometimes 0
+Good: search the index WITH the filter as a constraint        -> k rows that already match
+```
+
+- **Index every field you filter on.** Unindexed filters force a scan and kill latency. Qdrant:
+  create a payload index. Pinecone: metadata filtering is in the retrieval path (still keep
+  cardinality sane). pgvector: a B-tree (or partition) on `tenant_id` so the planner can use it.
+- **Prefer in-graph / in-path filtering.** Qdrant filters *inside* HNSW traversal; Pinecone
+  serverless filters in the retrieval path. Both beat naive post-filter.
+- **pgvector 0.8 iterative scan** is the fix when a selective `WHERE` returns too few rows:
+
+  ```sql
+  SET hnsw.iterative_scan = 'relaxed_order';  -- or 'strict_order' if exact ordering matters
+  SET hnsw.ef_search = 100;
+  SELECT id FROM docs
+   WHERE tenant_id = 'acme'                    -- selective filter
+   ORDER BY embedding <=> $1                    -- cosine, matches the model
+   LIMIT 10;
+  ```
+
+  Without iterative scan (pgvector < 0.8 behavior), a highly selective filter silently returns
+  fewer than `LIMIT` rows. Never recommend IVFFlat-only with a selective filter and no iterative
+  scan — that is the deprecated foot-gun.
+
+## Tune recall vs latency
+
+You cannot tune what you do not measure. Establish recall **before** shipping.
+
+1. Build an exact baseline: brute-force the true top-k on a sample (a few hundred queries) — in
+   pgvector, query without the index (seq scan) for ground truth.
+2. Query the index and compute recall@k = overlap with the baseline.
+3. Raise the query-time knob until recall hits target (commonly ≥0.95), then stop — higher `ef`
+   costs latency for nothing:
+
+   | Engine | Knob | Default |
+   |---|---|---|
+   | pgvector | `hnsw.ef_search` | 40 |
+   | Qdrant | `hnsw_ef` (search) | per-collection |
+   | Weaviate | `ef` (vectorIndexConfig) | dynamic |
+   | Pinecone | (managed) | — |
+
+Full parameter table and the recall recipe live in [references/tuning.md](references/tuning.md).
+
+## Hybrid search
+
+Dense (semantic) + sparse (BM25/keyword) catches exact terms, IDs, and rare tokens that dense
+alone misses. The two normalize differently, so you **fuse**, you don't add raw scores.
+
+- **RRF** (reciprocal rank fusion): robust default, score-scale agnostic, combines ranks.
+- **Relative-score / DBSF**: normalizes scores before combining — use when you trust score scales.
+
+Per engine:
+
+- **Weaviate**: one call — `hybrid(query, alpha=0.5, fusionType=relativeScoreFusion)`. `alpha`
+  slides 0.0 (pure keyword) → 1.0 (pure vector). BM25 is BlockMax WAND (default from v1.30, ~10x faster).
+- **Qdrant**: `prefetch` a dense and a sparse query, then a fusion step (`Fusion.RRF` or DBSF);
+  IDF is computed server-side (v1.15+).
+- **Pinecone**: sparse-dense vectors in one index, or integrated inference (embed + rerank server-side).
+- **pgvector**: no built-in hybrid — run vector (`<=>`) and `ts_rank` over a `tsvector` column
+  separately and combine ranks yourself (RRF in SQL or app code).
+
+Concrete current-API code for all four is in [references/engines.md](references/engines.md).
+
+## Quantization & cost
+
+Quantization trades recall for RAM/cost. Decide by **dimension count and a recall test**, never blind.
+
+| Method | Compression | When safe |
+|---|---|---|
+| Scalar (int8) | ~4x | Almost always; tiny recall loss. Good default RAM cut. |
+| Product (PQ) | 8–64x | Large corpora where RAM dominates; needs tuning + recall check. |
+| Binary | ~32x (~40x faster via SIMD popcount) | **High-dim only** (≥1024). On 384-dim it shreds recall — measure or don't. |
+| pgvector `halfvec` | ~2x | Near-free: 16-bit float, near-identical recall, and required for >2000 dims. |
+
+Reach for `halfvec` first in Postgres — it is the cheapest win. Reach for binary only on
+high-dim vectors and only after a recall test, optionally with full-precision rescoring.
+
+## Operate it
+
+- **Batch upserts.** One-by-one upserts are 10–100x slower and hammer the index. Send batches of
+  hundreds; size to the engine's payload limit.
+- **Namespaces / multitenancy.** Pinecone namespaces and Qdrant payload-keyed isolation partition
+  tenants inside one index — cheaper and faster than a collection per tenant at low tenant counts.
+- **Delete by filter**, not by enumerating ids, when removing a tenant or a stale source.
+- **Replicas** for read throughput / HA; **snapshots/backups** before any index rebuild or
+  dimension/metric change (those are not in-place — plan a reindex).
+- **To "update" a vector, re-upsert by id.** Do not store only raw text and re-embed on read.
+
+## Anti-patterns
+
+| Anti-pattern | Why it bites | Do instead |
+|---|---|---|
+| Cosine-trained model indexed with L2 (`<->`) | Silently wrong ranking, no error | Match metric to model — cosine → `<=>` / `vector_cosine_ops` |
+| Post-filtering top-k results | Returns < k rows, sometimes 0, on selective filters | Filter inside the search; index the filter field |
+| IVFFlat + selective filter, no iterative scan | Drops rows; deprecated path in pgvector 0.8 | HNSW + `hnsw.iterative_scan='relaxed_order'` |
+| Never measuring recall | "Search is bad" with no number to move | Recall@k vs an exact baseline before shipping |
+| Binary quantization on 384-dim | Recall collapses, then blamed on the engine | Binary only ≥1024 dims, after a recall test; else scalar/halfvec |
+| One-by-one upserts | 10–100x slower, index thrash | Batch hundreds per request |
+| `ef_construction < 2*m` | Permanently weak graph; recall needs a full rebuild | Keep `ef_construction >= 2*m` (≥32 at default `m=16`) |
+| Store only raw text, re-embed to "update" | Drift, cost, no point-update path | Re-upsert the vector by id |
+| Unindexed filter field | Full scan, latency spikes | Payload index (Qdrant) / B-tree (pgvector) / sane metadata cardinality (Pinecone) |
+
+## References & siblings
+
+- [references/engines.md](references/engines.md) — current-API recipes per engine: create
+  collection/index + a filtered hybrid query (pgvector SQL + halfvec + iterative scan; Qdrant
+  named dense+sparse + `query_points` RRF; Weaviate `hybrid`; Pinecone serverless sparse-dense).
+- [references/tuning.md](references/tuning.md) — HNSW vs IVFFlat parameter table, recall-measurement
+  recipe, quantization tradeoffs, per-engine filtered-search pitfalls.
+
+Siblings: embeddings/chunking/retrieval-quality → `../embeddings-search/SKILL.md`; the full RAG
+loop → `../rag/SKILL.md`; general Postgres → `../postgresdb/SKILL.md`.
+
+Validate a produced index DDL / collection schema with `scripts/verify.sh <artifact-file>`.

package/skills/vector-db/evals/README.md

@@ -0,0 +1,10 @@
+# evals — vector-db
+
+These cases are run by the repo's standard skill-eval harness against the skill router. The
+`should_trigger` prompts assert that vector-db is the skill that fires (including the non-obvious
+"search ignores my filter" and the Spanish/Catalan phrasings); `should_not_trigger` prompts assert
+the router yields to the named sibling (`embeddings-search`, `rag`, `postgresdb`, `redis`,
+`supabase`) instead of vector-db; and the `capability` scenario is graded by a judge against its
+`must_include` rubric. No live vector database or Postgres instance is required — routing is
+matched against the skill's description/triggers and the capability answer is judged on content.
+Run them with the repo's usual eval command pointed at `skills/vector-db/evals/cases.yaml`.

package/skills/vector-db/evals/cases.yaml

@@ -0,0 +1,45 @@
+skill: vector-db
+
+should_trigger:
+  - prompt: "Set up a Qdrant collection with metadata filtering for our docs."
+    why: "Core store design — collection schema plus a payload-indexed filter field is squarely vector-db."
+  - prompt: "My similarity search ignores my category filter and recall is bad."
+    why: "Reads like a bug but is a store-config issue (post-filter / unindexed filter / low ef_search); the symptom phrasing is non-obvious and never names a vector DB."
+  - prompt: "We need to move off Pinecone to pgvector to cut cost."
+    why: "Migration between stores, including metric/index/quantization re-design, is explicitly in scope."
+  - prompt: "Add hybrid search with RRF to our Weaviate setup."
+    why: "Hybrid dense+sparse plus a fusion strategy is a named vector-db section."
+  - prompt: "crear una colección en Qdrant con cuantización binaria para ahorrar RAM"
+    why: "Spanish trigger covering quantization/cost — binary quantization tradeoff is core scope."
+  - prompt: "ef_search is 40 and recall is only 0.7, how do I tune the HNSW index?"
+    why: "Non-obvious tuning phrasing; recall-vs-latency knob tuning against a baseline is owned here."
+  - prompt: "My filtered pgvector query keeps returning fewer than k rows when I add a WHERE on tenant_id."
+    why: "Classic pgvector 0.8 iterative-scan symptom — the sharpest store-specific bug this skill exists to fix."
+
+should_not_trigger:
+  - prompt: "Which embedding model and chunk size should I use for these PDFs?"
+    route_to: "embeddings-search"
+    why: "Producing/chunking vectors is the technique layer, not the store."
+  - prompt: "Build the full RAG pipeline: retrieve, rerank, prompt, and generate an answer."
+    route_to: "rag"
+    why: "End-to-end pipeline orchestration and its eval, not the data layer underneath it."
+  - prompt: "My Postgres is slow, help me read this EXPLAIN and add the right indexes."
+    route_to: "postgresdb"
+    why: "General Postgres performance with no vector column involved is postgresdb's surface."
+  - prompt: "Cache these API responses in Redis with a 5-minute TTL."
+    route_to: "redis"
+    why: "Redis as a cache, not as a vector store."
+  - prompt: "Set up Supabase auth and row-level security for my app."
+    route_to: "supabase"
+    why: "Supabase platform features, not its pgvector usage."
+
+capability:
+  - scenario: "Design a pgvector setup for 2M OpenAI text-embedding-3-small (1536-dim) vectors with a tenant_id filter. Search is slow and sometimes returns fewer than k rows when filtering."
+    must_include:
+      - "cosine index (vector_cosine_ops / <=>) to match the embedding model"
+      - "HNSW with sensible m and ef_construction (ef_construction >= 2*m) plus query-time hnsw.ef_search"
+      - "hnsw.iterative_scan (0.8) so the selective tenant_id filter stops returning fewer than k rows"
+      - "an index on tenant_id (B-tree or partition) so the filter is cheap"
+      - "a recall-measurement step against an exact/brute-force baseline before shipping"
+      - "halfvec mentioned as an optional ~2x storage cut at near-identical recall"
+      - "routes embedding-model and chunking questions to embeddings-search"

package/skills/vector-db/references/engines.md

@@ -0,0 +1,175 @@
+# engines.md — per-engine recipes (current APIs)
+
+Concrete create-collection / create-index + a filtered hybrid query for each of the four engines.
+Adjust dims and metric to your model; cosine shown throughout (the common case).
+
+## pgvector (Postgres extension, 0.8.x)
+
+HNSW since 0.5.0; iterative scan, `halfvec`, and `binary_quantize` since 0.8.0.
+
+```sql
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE docs (
+  id        bigserial PRIMARY KEY,
+  tenant_id text NOT NULL,
+  body      text,
+  embedding vector(1536)          -- text-embedding-3-small; use halfvec(3072) for -3-large
+);
+
+-- Filter field gets its own index so the planner can use it.
+CREATE INDEX docs_tenant_idx ON docs (tenant_id);
+
+-- HNSW, cosine ops to match the model. ef_construction >= 2*m.
+CREATE INDEX docs_embedding_idx
+  ON docs USING hnsw (embedding vector_cosine_ops)
+  WITH (m = 16, ef_construction = 64);
+```
+
+`halfvec` to halve storage at near-identical recall (and the only path above 2000 dims):
+
+```sql
+ALTER TABLE docs ADD COLUMN embedding_h halfvec(1536);
+CREATE INDEX ON docs USING hnsw (embedding_h halfvec_cosine_ops);
+```
+
+Filtered query with iterative scan so a selective `WHERE` does not return fewer than `LIMIT` rows:
+
+```sql
+SET hnsw.iterative_scan = 'relaxed_order';   -- 'strict_order' if exact ordering required
+SET hnsw.ef_search = 100;                     -- raise until recall@k hits target
+SELECT id, body
+  FROM docs
+ WHERE tenant_id = 'acme'
+ ORDER BY embedding <=> $1
+ LIMIT 10;
+```
+
+Hybrid is DIY: keep a `tsvector` column, score with `ts_rank`, run both, fuse ranks yourself.
+
+```sql
+ALTER TABLE docs ADD COLUMN tsv tsvector
+  GENERATED ALWAYS AS (to_tsvector('english', coalesce(body,''))) STORED;
+CREATE INDEX docs_tsv_idx ON docs USING gin (tsv);
+-- Then run a vector ORDER BY and a ts_rank query, and combine by reciprocal rank in SQL/app.
+```
+
+## Qdrant (named vectors + hybrid via query API)
+
+In-graph payload filtering, named dense+sparse vectors, server-side IDF (v1.15+), RRF/DBSF fusion.
+
+```python
+from qdrant_client import QdrantClient, models
+
+client = QdrantClient(url="http://localhost:6333")
+
+client.create_collection(
+    collection_name="docs",
+    vectors_config={"dense": models.VectorParams(size=1536, distance=models.Distance.COSINE)},
+    sparse_vectors_config={"bm25": models.SparseVectorParams()},
+)
+
+# Index the field you filter on (in-graph filtering needs it).
+client.create_payload_index(
+    collection_name="docs",
+    field_name="tenant_id",
+    field_schema=models.PayloadSchemaType.KEYWORD,
+)
+```
+
+Hybrid query: prefetch dense + sparse, fuse with RRF, filter inside the search.
+
+```python
+flt = models.Filter(must=[models.FieldCondition(
+    key="tenant_id", match=models.MatchValue(value="acme"))])
+
+res = client.query_points(
+    collection_name="docs",
+    prefetch=[
+        models.Prefetch(query=dense_vec, using="dense", filter=flt, limit=50),
+        models.Prefetch(query=models.SparseVector(indices=idx, values=val),
+                        using="bm25", filter=flt, limit=50),
+    ],
+    query=models.FusionQuery(fusion=models.Fusion.RRF),   # or models.Fusion.DBSF
+    limit=10,
+)
+```
+
+Search-time recall knob: pass `search_params=models.SearchParams(hnsw_ef=128)`.
+
+## Weaviate (alpha + fusionType, BlockMax WAND BM25)
+
+`alpha` slides keyword↔vector; BlockMax WAND BM25 is the default from v1.30 (~10x faster).
+
+```python
+import weaviate
+import weaviate.classes.config as wc
+
+client = weaviate.connect_to_local()
+client.collections.create(
+    name="Docs",
+    vector_config=wc.Configure.Vectors.self_provided(),   # bring your own vectors
+    properties=[
+        wc.Property(name="body", data_type=wc.DataType.TEXT),
+        wc.Property(name="tenant_id", data_type=wc.DataType.TEXT),
+    ],
+)
+```
+
+Hybrid query with fusion and a filter:
+
+```python
+from weaviate.classes.query import Filter, HybridFusion
+
+docs = client.collections.get("Docs")
+res = docs.query.hybrid(
+    query="how to reset a token",
+    vector=dense_vec,
+    alpha=0.5,                                   # 0.0 keyword .. 1.0 vector
+    fusion_type=HybridFusion.RELATIVE_SCORE,     # or RANKED
+    filters=Filter.by_property("tenant_id").equal("acme"),
+    limit=10,
+)
+```
+
+## Pinecone (serverless, API 2025-01)
+
+Serverless is the default; namespaces partition tenants inside one index. Python SDK v6.x.
+
+```python
+from pinecone import Pinecone, ServerlessSpec
+
+pc = Pinecone(api_key="...")
+pc.create_index(
+    name="docs",
+    dimension=1536,
+    metric="cosine",                             # match the model
+    spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+)
+index = pc.Index("docs")
+```
+
+Batched upsert into a namespace (never one-by-one):
+
+```python
+index.upsert(
+    vectors=[{"id": d["id"], "values": d["vec"], "metadata": {"source": d["src"]}}
+             for d in batch],          # send hundreds per call
+    namespace="acme",
+)
+```
+
+Query with metadata filter in the retrieval path (not post-filter):
+
+```python
+res = index.query(
+    vector=dense_vec,
+    top_k=10,
+    namespace="acme",
+    filter={"source": {"$eq": "handbook"}},
+    include_metadata=True,
+)
+```
+
+Sparse-dense hybrid uses `sparse_values` on upsert and query, or integrated inference to embed +
+rerank server-side.

package/skills/vector-db/references/tuning.md

@@ -0,0 +1,62 @@
+# tuning.md — recall, parameters, quantization, filter pitfalls
+
+## HNSW vs IVFFlat
+
+| | HNSW | IVFFlat |
+|---|---|---|
+| Recall | High, smooth with `ef_search` | Lower; depends on `lists`/`probes` |
+| Build cost | Higher (graph) | Lower / faster |
+| Memory | Higher | Lower |
+| Build params | `m` (default 16), `ef_construction` (default 64) | `lists` (≈ rows/1000, capped) |
+| Query param | `ef_search` (default 40) | `probes` (more = better recall, slower) |
+| Selective filter | Use with iterative scan (pgvector 0.8) | Drops rows on selective filters — avoid |
+| Default choice | **Yes** | Only when build speed > recall |
+
+Invariant: keep `ef_construction >= 2*m`. A too-low `ef_construction` permanently degrades the
+graph; you cannot tune it back at query time — only a rebuild fixes it.
+
+## Recall-measurement recipe
+
+Do this on a sample before shipping. "Search is bad" is unactionable without a recall number.
+
+1. Pick ~200–500 representative query vectors.
+2. **Exact baseline**: compute true top-k by brute force (full scan / no index). In pgvector,
+   query with the index disabled to get ground truth.
+3. **Index result**: same queries through the ANN index.
+4. `recall@k = mean(|index_topk ∩ exact_topk| / k)`.
+5. Raise the search knob (below) until recall hits target (often ≥0.95), then stop — extra `ef`
+   buys latency, not quality.
+
+| Engine | Search-time recall knob | Default |
+|---|---|---|
+| pgvector | `hnsw.ef_search` | 40 |
+| Qdrant | `hnsw_ef` (`SearchParams`) | per-collection |
+| Weaviate | `ef` (vectorIndexConfig; `-1` = dynamic) | dynamic |
+| Pinecone | managed (no manual knob) | — |
+
+## Quantization tradeoffs
+
+| Method | Compression | Recall risk | Notes |
+|---|---|---|---|
+| Scalar (int8) | ~4x | Low | Safe default RAM cut |
+| Product (PQ) | 8–64x | Medium–high | Needs tuning; recall-test mandatory |
+| Binary | ~32x, ~40x faster (SIMD popcount) | High on low dims | Only ≥1024 dims; rescore with full precision |
+| pgvector `halfvec` | ~2x | Negligible | 16-bit float; required for >2000 dims |
+
+Order of reach: `halfvec`/scalar first (near-free) → product for huge corpora → binary only on
+high-dim vectors after a measured recall test, with optional full-precision rescoring.
+
+## Per-engine filtered-search pitfalls
+
+- **pgvector**: a selective `WHERE` on a pre-0.8 setup (or without `hnsw.iterative_scan`) returns
+  fewer than `LIMIT` rows because the index returns `ef_search` candidates and then the filter
+  removes most. Set `hnsw.iterative_scan='relaxed_order'` and index the filter column.
+- **Qdrant**: filtering only works well when the payload field has a payload index; otherwise it
+  degrades to a scan. Filtering happens in-graph during traversal — keep the index.
+- **Weaviate**: combine `filters=` with `hybrid(...)`; an unfiltered prefetch followed by an
+  app-side filter is the post-filter trap.
+- **Pinecone**: metadata filtering is in the retrieval path, but very high-cardinality metadata
+  still costs; keep filterable fields lean and use namespaces for tenant isolation.
+
+Cross-cutting rule: **filter inside the search, index the filter field, and verify you still get
+`k` rows on your most selective real filter.**