baked-vector 0.1.0__py3-none-any.whl

baked/__init__.py

@@ -0,0 +1,25 @@
+from .store import VectorStore
+from .embed import (
+    EmbeddingStore,
+    sentence_transformer_embedder,
+    openai_embedder,
+    azure_openai_embedder,
+    bedrock_embedder,
+    google_embedder,
+)
+from .chunking import chunk_fixed, chunk_sentences, chunk_paragraphs, chunk_recursive
+
+__all__ = [
+    "VectorStore",
+    "EmbeddingStore",
+    "sentence_transformer_embedder",
+    "openai_embedder",
+    "azure_openai_embedder",
+    "bedrock_embedder",
+    "google_embedder",
+    "chunk_fixed",
+    "chunk_sentences",
+    "chunk_paragraphs",
+    "chunk_recursive",
+]
+__version__ = "0.1.0"

baked/chunking.py

@@ -0,0 +1,281 @@
+"""Text chunking utilities — stdlib only (re, math). No external dependencies."""
+
+from __future__ import annotations
+
+import re
+
+
+def chunk_fixed(text: str, size: int = 512, overlap: int = 64) -> list[str]:
+    """Split text into fixed-size character chunks with overlap.
+
+    Slides a window of *size* chars over *text* with a step of ``size - overlap``.
+    Each chunk is stripped; empty chunks are dropped.
+
+    Args:
+        text: Input text to split.
+        size: Maximum characters per chunk.
+        overlap: Number of characters repeated at the start of the next chunk.
+
+    Returns:
+        List of non-empty stripped chunks.
+    """
+    if not text or not text.strip():
+        return []
+
+    step = size - overlap
+    if step <= 0:
+        raise ValueError("overlap must be less than size")
+
+    chunks: list[str] = []
+    start = 0
+    while start < len(text):
+        chunk = text[start : start + size].strip()
+        if chunk:
+            chunks.append(chunk)
+        start += step
+
+    return chunks
+
+
+# Sentence-ending punctuation followed by whitespace or end-of-string.
+_SENT_RE = re.compile(r"(?<=[.!?])(?:\s+|$)")
+
+
+def chunk_sentences(text: str, max_size: int = 512, overlap: int = 1) -> list[str]:
+    """Split on sentence boundaries (.!?).
+
+    Sentences are detected by ``.``, ``!``, or ``?`` followed by whitespace or
+    end-of-string. ``overlap`` is the number of sentences from the end of the
+    previous chunk to repeat at the start of the next chunk (not characters).
+
+    Short sentences are merged into a single chunk until *max_size* would be
+    exceeded, then the current chunk is flushed.
+
+    Args:
+        text: Input text to split.
+        max_size: Soft maximum characters per chunk. A single sentence longer
+            than *max_size* is kept as its own chunk.
+        overlap: Number of trailing sentences carried into the next chunk.
+
+    Returns:
+        List of non-empty stripped chunks.
+    """
+    if not text or not text.strip():
+        return []
+
+    # Split and keep the punctuation attached to the preceding sentence.
+    raw = _SENT_RE.split(text)
+    sentences = [s.strip() for s in raw if s.strip()]
+
+    if not sentences:
+        return []
+
+    chunks: list[str] = []
+    current: list[str] = []
+    current_len = 0
+
+    def _flush() -> None:
+        if current:
+            joined = " ".join(current)
+            if joined.strip():
+                chunks.append(joined.strip())
+
+    for sent in sentences:
+        # +1 for the space separator
+        added_len = len(sent) + (1 if current else 0)
+        if current and current_len + added_len > max_size:
+            _flush()
+            # carry overlap sentences into next chunk
+            tail = current[-overlap:] if overlap else []
+            current = list(tail)
+            current_len = sum(len(s) for s in current) + max(0, len(current) - 1)
+
+        current.append(sent)
+        current_len += len(sent) + (1 if len(current) > 1 else 0)
+
+    _flush()
+    return chunks
+
+
+def chunk_paragraphs(text: str, max_size: int = 1024, overlap: int = 0) -> list[str]:
+    """Split on blank lines, merging short paragraphs up to *max_size*.
+
+    Paragraphs are separated by one or more blank lines (``\\n\\n+``). Consecutive
+    short paragraphs are merged (joined with ``\\n\\n``) until adding the next one
+    would exceed *max_size*, then the current buffer is flushed. ``overlap=0``
+    means no paragraph is repeated; ``overlap=N`` repeats the last N paragraphs
+    of each chunk at the start of the next.
+
+    Args:
+        text: Input text to split.
+        max_size: Soft maximum characters per chunk.
+        overlap: Number of trailing paragraphs carried into the next chunk.
+
+    Returns:
+        List of non-empty stripped chunks.
+    """
+    if not text or not text.strip():
+        return []
+
+    raw = re.split(r"\n\n+", text)
+    paragraphs = [p.strip() for p in raw if p.strip()]
+
+    if not paragraphs:
+        return []
+
+    chunks: list[str] = []
+    current: list[str] = []
+    current_len = 0
+    sep = "\n\n"
+    sep_len = len(sep)
+
+    def _flush() -> None:
+        if current:
+            joined = sep.join(current)
+            if joined.strip():
+                chunks.append(joined.strip())
+
+    for para in paragraphs:
+        added_len = len(para) + (sep_len if current else 0)
+        if current and current_len + added_len > max_size:
+            _flush()
+            tail = current[-overlap:] if overlap else []
+            current = list(tail)
+            current_len = (
+                sum(len(p) for p in current) + sep_len * max(0, len(current) - 1)
+            )
+
+        current.append(para)
+        current_len += len(para) + (sep_len if len(current) > 1 else 0)
+
+    _flush()
+    return chunks
+
+
+# Ordered list of separators tried by chunk_recursive.
+_RECURSIVE_SEPS = ["\n\n", "\n", ". ", ""]
+
+
+def chunk_recursive(text: str, size: int = 512, overlap: int = 64) -> list[str]:
+    """LangChain-style recursive text splitter.
+
+    Tries splitting on ``\\n\\n``, then ``\\n``, then ``". "``, then falls back
+    to fixed character splits. Each piece that is still larger than *size* is
+    recursively split with the remaining separators.
+
+    Args:
+        text: Input text to split.
+        size: Maximum characters per chunk.
+        overlap: Characters of overlap between consecutive chunks on hard cuts.
+
+    Returns:
+        List of non-empty stripped chunks.
+    """
+    if not text or not text.strip():
+        return []
+    return _recursive_split(text, size, overlap, list(_RECURSIVE_SEPS))
+
+
+def _recursive_split(
+    text: str, size: int, overlap: int, seps: list[str]
+) -> list[str]:
+    text = text.strip()
+    if not text:
+        return []
+    if len(text) <= size:
+        return [text]
+
+    # Try each separator in order until one produces more than one piece.
+    chosen_sep: str | None = None
+    splits: list[str] = []
+    remaining_seps = list(seps)
+    while remaining_seps:
+        sep = remaining_seps.pop(0)
+        if sep == "":
+            # Hard character split — handled after the loop.
+            break
+        parts = text.split(sep)
+        if len(parts) > 1:
+            chosen_sep = sep
+            splits = [p for p in parts if p.strip()]
+            break
+    else:
+        chosen_sep = None
+
+    if chosen_sep is None:
+        # Fall back to fixed character split with overlap.
+        return _fixed_split(text, size, overlap)
+
+    # Recursively handle pieces that are still too long.
+    next_seps = [s for s in _RECURSIVE_SEPS if s != chosen_sep]
+    good: list[str] = []
+    for piece in splits:
+        piece = piece.strip()
+        if not piece:
+            continue
+        if len(piece) <= size:
+            good.append(piece)
+        else:
+            good.extend(_recursive_split(piece, size, overlap, list(next_seps)))
+
+    # Merge good pieces back up to size.
+    return _merge(good, chosen_sep, size, overlap)
+
+
+def _merge(
+    splits: list[str], sep: str, size: int, overlap: int
+) -> list[str]:
+    """Merge split pieces into chunks up to *size* with *overlap* char carry-over."""
+    chunks: list[str] = []
+    current: list[str] = []
+    current_len = 0
+    sep_len = len(sep)
+
+    for piece in splits:
+        piece = piece.strip()
+        if not piece:
+            continue
+        piece_len = len(piece)
+        join_len = current_len + sep_len + piece_len if current else piece_len
+
+        if join_len > size and current:
+            chunk_text = sep.join(current).strip()
+            if chunk_text:
+                chunks.append(chunk_text)
+            # Build overlap from the tail of current.
+            ov: list[str] = []
+            ov_len = 0
+            for part in reversed(current):
+                part_len = len(part)
+                candidate = ov_len + (sep_len if ov else 0) + part_len
+                if candidate > overlap and ov:
+                    break
+                ov.insert(0, part)
+                ov_len = candidate
+            current = ov
+            current_len = ov_len
+
+        current.append(piece)
+        current_len = (
+            sum(len(p) for p in current) + sep_len * max(0, len(current) - 1)
+        )
+
+    if current:
+        chunk_text = sep.join(current).strip()
+        if chunk_text:
+            chunks.append(chunk_text)
+
+    return chunks
+
+
+def _fixed_split(text: str, size: int, overlap: int) -> list[str]:
+    """Hard character split with overlap — last resort."""
+    step = max(1, size - overlap)
+    chunks: list[str] = []
+    start = 0
+    while start < len(text):
+        chunk = text[start : start + size].strip()
+        if chunk:
+            chunks.append(chunk)
+        start += step
+    return chunks

baked/embed.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from .store import VectorStore
+
+
+Embedder = Callable[[str], list[float]]
+
+
+class EmbeddingStore:
+    """VectorStore wrapper that auto-embeds text via a provided embedder callable."""
+
+    def __init__(self, path: str, embedder: Embedder):
+        self._store = VectorStore(path)
+        self._embed = embedder
+
+    def add(self, text: str, metadata: dict[str, Any] | None = None) -> int:
+        vec = self._embed(text)
+        meta = {"text": text, **(metadata or {})}
+        return self._store.add(vec, meta)
+
+    def add_many(self, texts: list[str], metadatas: list[dict[str, Any]] | None = None) -> list[int]:
+        """Embed and add multiple texts. Returns list of assigned ids."""
+        if metadatas is None:
+            metadatas = [{} for _ in texts]
+        if len(texts) != len(metadatas):
+            raise ValueError(
+                f"Length mismatch: {len(texts)} texts but {len(metadatas)} metadatas."
+            )
+        return [self.add(text, meta) for text, meta in zip(texts, metadatas)]
+
+    def search(
+        self,
+        query: str,
+        top_k: int = 5,
+        radius: float | None = None,
+        filter: dict[str, Any] | None = None,
+    ) -> list[tuple[dict[str, Any], float]]:
+        vec = self._embed(query)
+        return self._store.search(vec, top_k=top_k, radius=radius, filter=filter)
+
+    def remove(self, id: int) -> None:
+        self._store.remove(id)
+
+    def update(
+        self,
+        id: int,
+        text: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Re-embed and/or update metadata for an existing entry."""
+        if text is not None:
+            vec = self._embed(text)
+            merged_meta = {"text": text, **(metadata or {})}
+            self._store.update(id, embedding=vec, metadata=merged_meta)
+        else:
+            self._store.update(id, metadata=metadata)
+
+    def save(self) -> None:
+        self._store.save()
+
+    def __len__(self) -> int:
+        return len(self._store)
+
+    def __repr__(self) -> str:
+        return f"EmbeddingStore(store={self._store!r})"
+
+
+# ---------------------------------------------------------------------------
+# Provider adapters
+# ---------------------------------------------------------------------------
+
+def sentence_transformer_embedder(model_name: str = "all-MiniLM-L6-v2") -> Embedder:
+    """Returns an embedder using a sentence-transformers model (install baked[st])."""
+    try:
+        from sentence_transformers import SentenceTransformer
+    except ImportError:
+        raise ImportError("Install sentence-transformers: pip install baked-vector[st]")
+
+    model = SentenceTransformer(model_name)
+
+    def embed(text: str) -> list[float]:
+        return model.encode(text, normalize_embeddings=False).tolist()
+
+    return embed
+
+
+def openai_embedder(model: str = "text-embedding-3-small", api_key: str | None = None) -> Embedder:
+    """Returns an embedder using OpenAI's API (install baked[openai])."""
+    try:
+        from openai import OpenAI
+    except ImportError:
+        raise ImportError("Install openai: pip install baked-vector[openai]")
+
+    client = OpenAI(api_key=api_key)
+
+    def embed(text: str) -> list[float]:
+        response = client.embeddings.create(input=text, model=model)
+        return response.data[0].embedding
+
+    return embed
+
+
+def azure_openai_embedder(
+    deployment: str,
+    endpoint: str,
+    api_key: str,
+    api_version: str = "2024-02-01",
+) -> Embedder:
+    """Returns an embedder using Azure OpenAI (install baked[openai])."""
+    try:
+        from openai import AzureOpenAI
+    except ImportError:
+        raise ImportError("Install openai: pip install baked-vector[openai]")
+
+    client = AzureOpenAI(azure_endpoint=endpoint, api_key=api_key, api_version=api_version)
+
+    def embed(text: str) -> list[float]:
+        response = client.embeddings.create(input=text, model=deployment)
+        return response.data[0].embedding
+
+    return embed
+
+
+def bedrock_embedder(
+    model_id: str = "amazon.titan-embed-text-v2:0",
+    region_name: str = "us-east-1",
+) -> Embedder:
+    """Returns an embedder using AWS Bedrock (install baked[bedrock])."""
+    try:
+        import boto3
+    except ImportError:
+        raise ImportError("Install boto3: pip install baked-vector[bedrock]")
+
+    import json
+
+    client = boto3.client("bedrock-runtime", region_name=region_name)
+
+    def embed(text: str) -> list[float]:
+        body = json.dumps({"inputText": text})
+        response = client.invoke_model(modelId=model_id, body=body)
+        result = json.loads(response["body"].read())
+        return result["embedding"]
+
+    return embed
+
+
+def google_embedder(
+    model: str = "models/text-embedding-004",
+    api_key: str | None = None,
+) -> Embedder:
+    """Returns an embedder using Google Generative AI (install baked[google])."""
+    try:
+        import google.generativeai as genai
+    except ImportError:
+        raise ImportError("Install google-generativeai: pip install baked-vector[google]")
+
+    if api_key:
+        genai.configure(api_key=api_key)
+
+    def embed(text: str) -> list[float]:
+        result = genai.embed_content(model=model, content=text)
+        return result["embedding"]
+
+    return embed

baked/evals/__init__.py

@@ -0,0 +1,42 @@
+"""
+baked.evals — retrieval evaluation toolkit
+
+Typical workflow:
+
+    1. Generate labeled pairs (query → expected doc) once and save them:
+
+        from baked.evals.synthetic import generate_pairs, openai_llm
+        from baked.evals import save_pairs
+
+        pairs = generate_pairs(docs, llm=openai_llm(), questions_per_doc=5)
+        save_pairs(pairs, "evals/my_pairs.json")
+
+    2. Run eval against your store whenever embeddings or search params change:
+
+        from baked.evals import load_pairs, run_eval, print_report
+
+        pairs  = load_pairs("evals/my_pairs.json")
+        result = run_eval(store, pairs, top_k=5)
+        print_report(result, verbose=True)
+
+Metrics:
+    Recall@k  — fraction of queries where the correct doc appears in top-k results.
+    MRR       — mean reciprocal rank; rewards finding the correct doc earlier.
+"""
+
+from .harness import EvalPair, EvalResult, QueryResult, run_eval, print_report, save_pairs, load_pairs
+from .synthetic import generate_pairs, groq_llm, openai_llm, anthropic_llm
+
+__all__ = [
+    "EvalPair",
+    "EvalResult",
+    "QueryResult",
+    "run_eval",
+    "print_report",
+    "save_pairs",
+    "load_pairs",
+    "generate_pairs",
+    "groq_llm",
+    "openai_llm",
+    "anthropic_llm",
+]

baked/evals/harness.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from typing import Any
+
+
+@dataclass
+class EvalPair:
+    """A single labeled evaluation example.
+
+    query:    the question or search string to issue against the store.
+    expected: a metadata subset that the correct result must contain.
+              Partial match — {"title": "X"} matches any result whose metadata
+              has at least title="X", regardless of other keys.
+    label:    optional human-readable name shown in print_report output.
+    """
+    query: str
+    expected: dict[str, Any]
+    label: str = ""
+
+
+@dataclass
+class QueryResult:
+    """Outcome of a single query during an eval run."""
+    pair: EvalPair
+    rank: int | None  # 1-indexed rank of first match; None = not found in top_k
+    hits: list[dict[str, Any]]  # metadata of each top_k result
+    scores: list[float]  # cosine scores of each top_k result
+
+
+@dataclass
+class EvalResult:
+    """Aggregate metrics from run_eval().
+
+    recall_at_k: fraction of queries where the correct doc appeared in top-k.
+    mrr:         mean reciprocal rank across all queries.
+    top_k:       the k used during this eval run.
+    n:           total number of queries evaluated.
+    query_results: per-query breakdown for verbose inspection.
+    """
+    recall_at_k: float
+    mrr: float
+    top_k: int
+    n: int
+    query_results: list[QueryResult]
+
+    def __str__(self) -> str:
+        return (
+            f"Recall@{self.top_k}: {self.recall_at_k:.3f}  "
+            f"MRR: {self.mrr:.3f}  "
+            f"(n={self.n})"
+        )
+
+
+def _matches(metadata: dict[str, Any], expected: dict[str, Any]) -> bool:
+    return all(metadata.get(k) == v for k, v in expected.items())
+
+
+def run_eval(
+    store: Any,
+    pairs: list[EvalPair],
+    top_k: int = 5,
+) -> EvalResult:
+    """Evaluate retrieval quality against labeled query-document pairs.
+
+    Works with EmbeddingStore (pass str queries in EvalPair.query) and
+    VectorStore (pass vector queries). Any object with a compatible
+    .search(query, top_k) interface works.
+
+    Args:
+        store:  the store to evaluate. Must implement .search(query, top_k).
+        pairs:  labeled examples — see EvalPair.
+        top_k:  number of results to retrieve per query.
+
+    Returns:
+        EvalResult with Recall@k, MRR, and per-query breakdown.
+    """
+    query_results: list[QueryResult] = []
+
+    for pair in pairs:
+        results = store.search(pair.query, top_k=top_k)
+        hits = [meta for meta, _ in results]
+        scores = [score for _, score in results]
+        rank = next(
+            (i for i, meta in enumerate(hits, 1) if _matches(meta, pair.expected)),
+            None,
+        )
+        query_results.append(
+            QueryResult(pair=pair, rank=rank, hits=hits, scores=scores)
+        )
+
+    n = len(pairs)
+    recall = sum(1 for r in query_results if r.rank is not None) / n if n else 0.0
+    mrr = sum(1.0 / r.rank for r in query_results if r.rank is not None) / n if n else 0.0
+
+    return EvalResult(
+        recall_at_k=recall,
+        mrr=mrr,
+        top_k=top_k,
+        n=n,
+        query_results=query_results,
+    )
+
+
+def print_report(result: EvalResult, verbose: bool = False) -> None:
+    """Print a human-readable summary of an EvalResult.
+
+    Args:
+        result:  output from run_eval().
+        verbose: if True, prints rank and label for every query.
+                 Missed queries also show the top-scoring result for debugging.
+    """
+    print(str(result))
+    if verbose:
+        for qr in result.query_results:
+            status = f"rank {qr.rank}" if qr.rank is not None else "MISS"
+            label = qr.pair.label or qr.pair.query[:70]
+            print(f"  [{status:>6}] {label!r}")
+            if qr.rank is None and qr.hits:
+                print(f"           top hit: {qr.hits[0]}")
+
+
+def save_pairs(pairs: list[EvalPair], path: str) -> None:
+    """Save eval pairs to a JSON file so they can be reused across runs.
+
+    Generate pairs once (with generate_pairs or by hand), save them,
+    then load and re-run whenever you change embeddings or search params.
+    """
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump([asdict(p) for p in pairs], f, indent=2, ensure_ascii=False)
+
+
+def load_pairs(path: str) -> list[EvalPair]:
+    """Load eval pairs previously saved with save_pairs()."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    return [EvalPair(**d) for d in data]

baked/evals/synthetic.py

@@ -0,0 +1,148 @@
+"""
+baked.evals.synthetic — LLM-powered eval pair generation
+
+Use this when you have documents but no labeled query-answer pairs.
+An LLM reads each document and generates realistic questions whose answers
+are in that document, producing EvalPairs ready for run_eval().
+
+Typical use:
+
+    from baked.evals.synthetic import generate_pairs, openai_llm
+    from baked.evals import save_pairs
+
+    docs = [
+        ("Wealth is not money...", {"title": "How to Make Wealth", "author": "PG"}),
+        ("A startup is a company designed to grow fast...", {"title": "Startup = Growth"}),
+    ]
+
+    pairs = generate_pairs(docs, llm=openai_llm(), questions_per_doc=5)
+    save_pairs(pairs, "evals/pairs.json")
+
+The LLM is only called during generation — not during eval. Save the pairs
+and reuse them to compare embedding models or search configurations cheaply.
+"""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable
+
+from .harness import EvalPair
+
+# Any callable that takes a prompt string and returns the LLM's reply.
+LLMFn = Callable[[str], str]
+
+_PROMPT = """\
+Read the following text and generate {n} specific, self-contained questions whose \
+answers are found within it. Each question should test a different aspect of the content. \
+Do not ask vague or generic questions.
+
+Return ONLY a JSON array of strings. No explanation, no markdown fences, no extra text.
+
+Text:
+{text}"""
+
+
+def generate_pairs(
+    docs: list[tuple[str, dict[str, Any]]],
+    llm: LLMFn,
+    questions_per_doc: int = 5,
+    text_preview_chars: int = 3000,
+) -> list[EvalPair]:
+    """
+    Generate EvalPairs by prompting an LLM to produce questions from each document.
+
+    Args:
+        docs: List of (text, metadata) tuples. metadata becomes EvalPair.expected.
+        llm: Callable (str) -> str — use openai_llm() or anthropic_llm() as factories.
+        questions_per_doc: Number of questions to generate per document.
+        text_preview_chars: How many chars of each doc to send to the LLM.
+
+    Returns:
+        List of EvalPairs ready to pass to run_eval().
+    """
+    pairs: list[EvalPair] = []
+    for text, metadata in docs:
+        prompt = _PROMPT.format(n=questions_per_doc, text=text[:text_preview_chars])
+        response = llm(prompt)
+        questions = _extract_json_list(response)
+        for q in questions:
+            if isinstance(q, str) and q.strip():
+                pairs.append(EvalPair(query=q.strip(), expected=metadata))
+    return pairs
+
+
+def _extract_json_list(text: str) -> list:
+    match = re.search(r"\[.*?\]", text, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group())
+        except json.JSONDecodeError:
+            pass
+    try:
+        return json.loads(text.strip())
+    except json.JSONDecodeError:
+        return []
+
+
+def openai_llm(model: str = "gpt-4o-mini", api_key: str | None = None) -> LLMFn:
+    """LLM factory using OpenAI Chat Completions (install baked-vector[openai])."""
+    try:
+        from openai import OpenAI
+    except ImportError:
+        raise ImportError("pip install baked-vector[openai]")
+
+    client = OpenAI(api_key=api_key)
+
+    def llm(prompt: str) -> str:
+        resp = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.3,
+        )
+        return resp.choices[0].message.content
+
+    return llm
+
+
+def groq_llm(model: str = "openai/gpt-oss-120b", api_key: str | None = None) -> LLMFn:
+    """LLM factory using Groq's OpenAI-compatible API (install baked-vector[openai])."""
+    try:
+        from openai import OpenAI
+    except ImportError:
+        raise ImportError("pip install baked-vector[openai]")
+
+    client = OpenAI(
+        api_key=api_key,
+        base_url="https://api.groq.com/openai/v1",
+    )
+
+    def llm(prompt: str) -> str:
+        resp = client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.3,
+        )
+        return resp.choices[0].message.content
+
+    return llm
+
+
+def anthropic_llm(model: str = "claude-haiku-5-5", api_key: str | None = None) -> LLMFn:
+    """LLM factory using Anthropic Messages API (install baked-vector[anthropic])."""
+    try:
+        import anthropic
+    except ImportError:
+        raise ImportError("pip install baked-vector[anthropic]")
+
+    client = anthropic.Anthropic(api_key=api_key)
+
+    def llm(prompt: str) -> str:
+        msg = client.messages.create(
+            model=model,
+            max_tokens=1024,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return msg.content[0].text
+
+    return llm

baked/store.py

@@ -0,0 +1,194 @@
+import json
+import math
+import struct
+import os
+from typing import Any
+
+_HEADER_FMT = "II"  # dim, count — both unsigned int
+_HEADER_SIZE = struct.calcsize(_HEADER_FMT)
+
+
+def _normalize(vec: list[float]) -> list[float]:
+    magnitude = math.sqrt(sum(x * x for x in vec))
+    if magnitude == 0.0:
+        raise ValueError("Cannot normalize a zero vector.")
+    return [x / magnitude for x in vec]
+
+
+def _dot(a: list[float], b: list[float]) -> float:
+    return sum(x * y for x, y in zip(a, b))
+
+
+class VectorStore:
+    def __init__(self, path: str):
+        self._path = path
+        self._vecs_path = path + ".vecs"
+        self._meta_path = path + ".json"
+        self._dim: int | None = None
+        self._vectors: list[list[float]] = []
+        self._metadata: list[dict[str, Any]] = []
+        self._deleted: set[int] = set()
+
+        if os.path.exists(self._vecs_path) and os.path.exists(self._meta_path):
+            self._load()
+
+    # ------------------------------------------------------------------
+    # Write API
+    # ------------------------------------------------------------------
+
+    def add(self, embedding: list[float], metadata: dict[str, Any] | None = None) -> int:
+        """Add a vector with optional metadata. Returns its id."""
+        if self._dim is None:
+            self._dim = len(embedding)
+        elif len(embedding) != self._dim:
+            raise ValueError(f"Dimension mismatch: expected {self._dim}, got {len(embedding)}.")
+
+        normalized = _normalize(embedding)
+        idx = len(self._vectors)
+        self._vectors.append(normalized)
+        self._metadata.append(metadata or {})
+        return idx
+
+    def add_many(self, embeddings: list[list[float]], metadatas: list[dict[str, Any]] | None = None) -> list[int]:
+        """Add multiple vectors at once. Returns list of assigned ids."""
+        if metadatas is None:
+            metadatas = [{} for _ in embeddings]
+        if len(embeddings) != len(metadatas):
+            raise ValueError(
+                f"Length mismatch: {len(embeddings)} embeddings but {len(metadatas)} metadatas."
+            )
+        return [self.add(emb, meta) for emb, meta in zip(embeddings, metadatas)]
+
+    def remove(self, id: int) -> None:
+        """Soft-delete a vector by id. Call save() to persist."""
+        if id < 0 or id >= len(self._vectors):
+            raise IndexError(f"Id {id} out of range.")
+        self._deleted.add(id)
+
+    def update(
+        self,
+        id: int,
+        embedding: list[float] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Update vector and/or metadata for an existing id. At least one must be provided."""
+        if embedding is None and metadata is None:
+            raise ValueError("At least one of embedding or metadata must be provided.")
+        if id < 0 or id >= len(self._vectors) or id in self._deleted:
+            raise IndexError(f"Id {id} is out of range or has been deleted.")
+        if embedding is not None:
+            if self._dim is not None and len(embedding) != self._dim:
+                raise ValueError(f"Dimension mismatch: expected {self._dim}, got {len(embedding)}.")
+            self._vectors[id] = _normalize(embedding)
+        if metadata is not None:
+            self._metadata[id] = metadata
+
+    def save(self) -> None:
+        """Flush all data to disk, compacting deleted entries."""
+        active_ids = [i for i in range(len(self._vectors)) if i not in self._deleted]
+
+        if not active_ids:
+            # Wipe files if store is empty
+            for p in (self._vecs_path, self._meta_path):
+                if os.path.exists(p):
+                    os.remove(p)
+            self._vectors = []
+            self._metadata = []
+            self._deleted = set()
+            return
+
+        dim = self._dim
+        count = len(active_ids)
+        vec_fmt = f"{dim}f"
+
+        with open(self._vecs_path, "wb") as f:
+            f.write(struct.pack(_HEADER_FMT, dim, count))
+            for i in active_ids:
+                f.write(struct.pack(vec_fmt, *self._vectors[i]))
+
+        meta_out = [self._metadata[i] for i in active_ids]
+        with open(self._meta_path, "w", encoding="utf-8") as f:
+            json.dump(meta_out, f)
+
+        # Compact in-memory state to match saved state
+        self._vectors = [self._vectors[i] for i in active_ids]
+        self._metadata = meta_out
+        self._deleted = set()
+
+    # ------------------------------------------------------------------
+    # Search API
+    # ------------------------------------------------------------------
+
+    def search(
+        self,
+        query: list[float],
+        top_k: int = 5,
+        radius: float | None = None,
+        filter: dict[str, Any] | None = None,
+    ) -> list[tuple[dict[str, Any], float]]:
+        """
+        Return up to top_k results as [(metadata, score)], highest score first.
+        If radius is set, return all results with score >= radius (top_k ignored).
+        filter: only consider vectors whose metadata contains all key-value pairs.
+        """
+        if not self._vectors:
+            return []
+
+        q = _normalize(query)
+        results = []
+
+        for i, vec in enumerate(self._vectors):
+            if i in self._deleted:
+                continue
+            if filter and not _matches(self._metadata[i], filter):
+                continue
+            score = _dot(q, vec)
+            results.append((self._metadata[i], score))
+
+        if radius is not None:
+            results = [(m, s) for m, s in results if s >= radius]
+            results.sort(key=lambda x: x[1], reverse=True)
+        else:
+            results.sort(key=lambda x: x[1], reverse=True)
+            results = results[:top_k]
+
+        return results
+
+    def search_many(
+        self,
+        queries: list[list[float]],
+        top_k: int = 5,
+        radius: float | None = None,
+        filter: dict[str, Any] | None = None,
+    ) -> list[list[tuple[dict[str, Any], float]]]:
+        """Batch search. Returns a list of result lists, one per query."""
+        return [self.search(q, top_k=top_k, radius=radius, filter=filter) for q in queries]
+
+    # ------------------------------------------------------------------
+    # Introspection
+    # ------------------------------------------------------------------
+
+    def __len__(self) -> int:
+        return len(self._vectors) - len(self._deleted)
+
+    def __repr__(self) -> str:
+        return f"VectorStore(path={self._path!r}, vectors={len(self)}, dim={self._dim})"
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _load(self) -> None:
+        with open(self._vecs_path, "rb") as f:
+            dim, count = struct.unpack(_HEADER_FMT, f.read(_HEADER_SIZE))
+            self._dim = dim
+            vec_fmt = f"{dim}f"
+            vec_size = struct.calcsize(vec_fmt)
+            self._vectors = [list(struct.unpack(vec_fmt, f.read(vec_size))) for _ in range(count)]
+
+        with open(self._meta_path, "r", encoding="utf-8") as f:
+            self._metadata = json.load(f)
+
+
+def _matches(metadata: dict[str, Any], filter: dict[str, Any]) -> bool:
+    return all(metadata.get(k) == v for k, v in filter.items())

baked_vector-0.1.0.dist-info/METADATA

@@ -0,0 +1,225 @@
+Metadata-Version: 2.4
+Name: baked-vector
+Version: 0.1.0
+Summary: Zero-dependency vector store for small-scale embedding search.
+License: MIT
+Project-URL: Repository, https://github.com/sunnytuladhar/baked-vector
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Provides-Extra: st
+Requires-Dist: sentence-transformers>=3.0; extra == "st"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == "openai"
+Provides-Extra: bedrock
+Requires-Dist: boto3>=1.34; extra == "bedrock"
+Provides-Extra: google
+Requires-Dist: google-generativeai>=0.8; extra == "google"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == "anthropic"
+
+# baked-vector
+
+Zero-dependency vector store for small-scale embedding search. No numpy, no server, no setup — just a binary file and a JSON file baked into your project.
+
+Designed for under 10,000 vectors. Bring your own embeddings.
+
+## Install
+
+```bash
+pip install baked-vector
+# or
+uv add baked-vector
+```
+
+## Usage
+
+```python
+import baked
+
+store = baked.VectorStore("mydata")
+
+# add vectors with arbitrary metadata
+store.add(embedding, {"source": "doc1.pdf", "page": 3, "text": "..."})
+store.save()
+
+# reload from disk automatically
+store = baked.VectorStore("mydata")
+
+# search
+results = store.search(query_embedding, top_k=5)
+# [(metadata_dict, score), ...]
+
+# filter by metadata
+results = store.search(query_embedding, top_k=5, filter={"source": "doc1.pdf"})
+
+# range search — all results with score >= threshold
+results = store.search(query_embedding, radius=0.85)
+
+# batch search
+results = store.search_many([vec1, vec2], top_k=5)
+```
+
+## API
+
+### `VectorStore(path)`
+
+Loads from `<path>.vecs` and `<path>.json` if they exist, otherwise starts empty.
+
+| Method | Description |
+|--------|-------------|
+| `add(embedding, metadata)` | Add a vector. Returns its id. |
+| `remove(id)` | Soft-delete. Takes effect on next `save()`. |
+| `save()` | Flush to disk, compacting deleted entries. |
+| `search(query, top_k, radius, filter)` | Find similar vectors. |
+| `search_many(queries, top_k, radius, filter)` | Batch search. |
+| `len(store)` | Number of active vectors. |
+
+### Search parameters
+
+- `top_k` — return top N results (default: 5)
+- `radius` — return all results with score ≥ threshold (overrides `top_k`)
+- `filter` — dict of key-value pairs that must match the metadata exactly
+
+Scores are cosine similarity in range [-1, 1]. Higher = more similar.
+
+## Storage format
+
+Two files:
+
+- `<path>.vecs` — binary packed floats. 8-byte header (dim, count), then N × dim × 4 bytes.
+- `<path>.json` — JSON array of metadata dicts, one per vector.
+
+Vectors are L2-normalized at insert time. IDs are positional and shift after `save()` following a `remove()`.
+
+## Performance
+
+Pure Python, brute-force scan. Rough query times:
+
+| Vectors | 384-dim | 768-dim | 1536-dim |
+|---------|---------|---------|----------|
+| 1k | <0.1s | <0.1s | ~0.1s |
+| 5k | ~0.1s | ~0.3s | ~0.5s |
+| 10k | ~0.3s | ~0.6s | ~1–2s |
+
+## EmbeddingStore
+
+`EmbeddingStore` wraps `VectorStore` and handles embedding automatically. Install an extra for your provider, pass the adapter, and work with text directly.
+
+```python
+import baked
+
+embed = baked.openai_embedder()  # see adapters below
+store = baked.EmbeddingStore("mydata", embedder=embed)
+
+store.add("Wealth is not money...", {"title": "How to Make Wealth"})
+store.save()
+
+results = store.search("how startups create value", top_k=3)
+# [({"text": "...", "title": "..."}, score), ...]
+```
+
+`EmbeddingStore` exposes the same `add`, `remove`, `save`, `search`, and `len` interface as `VectorStore`, but takes `str` instead of `list[float]`. The original text is stored in metadata under `"text"` automatically.
+
+### Adapters
+
+| Provider | Function | Install extra |
+|----------|----------|---------------|
+| OpenAI | `openai_embedder(model, api_key)` | `baked-vector[openai]` |
+| Azure OpenAI | `azure_openai_embedder(deployment, endpoint, api_key, api_version)` | `baked-vector[openai]` |
+| AWS Bedrock | `bedrock_embedder(model_id, region_name)` | `baked-vector[bedrock]` |
+| Google | `google_embedder(model, api_key)` | `baked-vector[google]` |
+| Local (sentence-transformers) | `sentence_transformer_embedder(model_name)` | `baked-vector[st]` |
+
+```bash
+pip install baked-vector[openai]
+pip install baked-vector[bedrock]
+pip install baked-vector[google]
+pip install baked-vector[st]
+```
+
+Defaults: OpenAI → `text-embedding-3-small`, Bedrock → `amazon.titan-embed-text-v2:0`, Google → `models/text-embedding-004`, local → `all-MiniLM-L6-v2`.
+
+You can also pass any `(str) -> list[float]` callable as `embedder` — no adapter required.
+
+---
+
+## Providers (manual)
+
+baked-vector stores and searches vectors — bring embeddings from any provider.
+
+### OpenAI
+
+```python
+from openai import OpenAI
+
+client = OpenAI()
+
+def embed(text: str) -> list[float]:
+    resp = client.embeddings.create(model="text-embedding-3-small", input=text)
+    return resp.data[0].embedding
+
+store.add(embed("hello world"), {"text": "hello world"})
+results = store.search(embed("hi there"), top_k=5)
+```
+
+### AWS Bedrock (Titan)
+
+```python
+import json
+import boto3
+
+bedrock = boto3.client("bedrock-runtime", region_name="us-east-1")
+
+def embed(text: str) -> list[float]:
+    resp = bedrock.invoke_model(
+        modelId="amazon.titan-embed-text-v2:0",
+        body=json.dumps({"inputText": text, "dimensions": 512}),
+    )
+    return json.loads(resp["body"].read())["embedding"]
+
+store.add(embed("hello world"), {"text": "hello world"})
+results = store.search(embed("hi there"), top_k=5)
+```
+
+### Azure OpenAI
+
+```python
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+    api_key="YOUR_KEY",
+    api_version="2024-02-01",
+    azure_endpoint="https://YOUR_RESOURCE.openai.azure.com",
+)
+
+def embed(text: str) -> list[float]:
+    resp = client.embeddings.create(model="YOUR_DEPLOYMENT_NAME", input=text)
+    return resp.data[0].embedding
+
+store.add(embed("hello world"), {"text": "hello world"})
+results = store.search(embed("hi there"), top_k=5)
+```
+
+### Cohere
+
+```python
+import cohere
+
+co = cohere.Client("YOUR_API_KEY")
+
+def embed(text: str) -> list[float]:
+    resp = co.embed(texts=[text], model="embed-english-v3.0", input_type="search_document")
+    return resp.embeddings[0]
+
+store.add(embed("hello world"), {"text": "hello world"})
+
+# use input_type="search_query" for query embeddings
+results = store.search(
+    co.embed(texts=["hi there"], model="embed-english-v3.0", input_type="search_query").embeddings[0],
+    top_k=5,
+)
+```
+
+## License
+
+MIT

baked_vector-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+baked/__init__.py,sha256=6oq1qKrl91TlZAsSTjwREHpEXgudXjfqa9cch2CGXxM,614
+baked/chunking.py,sha256=RSpuKyt5N_aYGadTlqwuAWL-d9p0ZAwJJHnX4NBXBUI,8962
+baked/embed.py,sha256=7QRmjCZEgcdAVQ48tKnJ6fyt3mOXRQdIDW9Cqw9RwlQ,5464
+baked/store.py,sha256=OB_0EFST4CU8oNl0gkyxQ99IxfP1XIE1xq-qex7wc0w,7485
+baked/evals/__init__.py,sha256=qPx_cSYt15YdWi62ZIsxnnO7GrJ6MYl-OvPtpHvBvqo,1295
+baked/evals/harness.py,sha256=qRLKC7rcgvifnTSzd4Wy-aOZvxIE-0pgd2HbAzzCLSw,4642
+baked/evals/synthetic.py,sha256=ItJa5qM5Hmtyqh7oqOPNLiQaMMQC28N3Zl8lOIa3NKQ,4838
+baked_vector-0.1.0.dist-info/METADATA,sha256=O4kheCxsLylvXE3gORcN5488d7iTRwOvdnt7NvoD78A,6777
+baked_vector-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+baked_vector-0.1.0.dist-info/top_level.txt,sha256=NY-Zme5Vc7zB6CFbZVzkTmAZAhJYe4-smYnQie3ax1g,6
+baked_vector-0.1.0.dist-info/RECORD,,

baked_vector-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

baked_vector-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+baked