ragkit-dev 0.1.0__py3-none-any.whl

rag_kit/settings.py

@@ -0,0 +1,247 @@
+"""All tunable settings as one immutable object.
+
+    settings = Settings.load(project_dir)          # project_dir/.env + environment
+    settings = Settings.load()                     # find the project (see below)
+    pipeline = Pipeline(settings=settings)
+
+A project is a folder containing a `rag-kit.toml` marker file. Without an
+explicit project_dir, it is $RAG_PROJECT_DIR, else the nearest folder at or
+above the current directory that has the marker; if there is none, loading
+fails instead of guessing (a guess would scatter Qdrant stores around).
+
+`Settings.load` reads the project's `.env` without touching `os.environ`, and
+real environment variables win over `.env`. Relative paths in it (`QDRANT_PATH`,
+`PDF_PATH`) are resolved against the project directory - never against the
+current working directory - so running a command from somewhere else cannot
+silently point at a different, empty Qdrant store.
+
+Every function that needs settings takes an optional `settings=`; without one
+it uses `default_settings()`.
+"""
+
+import dataclasses
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Mapping, Optional
+
+from dotenv import dotenv_values
+
+from rag_kit.errors import ConfigError
+
+SUPPORTED_EXTENSIONS = (".pdf", ".txt", ".md")
+
+# Every variable Settings.load() reads. rag_kit.env never exports these to
+# os.environ (see rag_kit/env.py); RAG_* control variables are excluded there too.
+SETTING_NAMES = frozenset({
+    "PDF_PATH", "PDF_FONT_REPAIR", "CHUNK_SIZE", "CHUNK_OVERLAP", "GOOGLE_API_KEY",
+    "EMBEDDING_PROVIDER", "EMBEDDING_MODEL", "EMBEDDING_DIM", "EMBED_BATCH_SIZE", "EMBED_SLEEP",
+    "QDRANT_URL", "QDRANT_API_KEY", "QDRANT_PATH", "COLLECTION_NAME", "TOP_K", "MIN_SCORE",
+    "LLM_MODEL", "TEMPERATURE",
+})
+
+# Marks a project folder. Only its presence matters for now; the contents are
+# reserved for future settings.
+PROJECT_MARKER = "rag-kit.toml"
+
+_PACKAGE_DIR = Path(__file__).resolve().parent
+
+
+def find_project_dir(start=None, environ: Optional[Mapping[str, str]] = None) -> Path:
+    """$RAG_PROJECT_DIR if set, else the nearest folder at or above `start`
+    (default: the current directory) that contains rag-kit.toml."""
+    environ = os.environ if environ is None else environ
+    if environ.get("RAG_PROJECT_DIR"):
+        return _checked_project_dir(environ["RAG_PROJECT_DIR"], "RAG_PROJECT_DIR")
+
+    start = Path(start or os.getcwd()).resolve()
+    for folder in (start, *start.parents):
+        if (folder / PROJECT_MARKER).is_file():
+            return _checked_project_dir(folder, "the project marker")
+    raise ConfigError(
+        f"No rag-kit project found: neither {start} nor any folder above it contains "
+        f"{PROJECT_MARKER}. Run the command inside a project, pass --project DIR, or set "
+        "RAG_PROJECT_DIR."
+    )
+
+
+def _checked_project_dir(value, source: str) -> Path:
+    path = Path(value).expanduser().resolve()
+    if not path.is_dir():
+        raise ConfigError(f"Project folder from {source} does not exist: {path}")
+    if path == _PACKAGE_DIR or _PACKAGE_DIR in path.parents:
+        raise ConfigError(
+            f"Project folder {path} is inside the installed rag-kit package. Keep documents, "
+            ".env and Qdrant data in your own project folder."
+        )
+    return path
+
+
+def _resolve(project_dir: Path, value) -> Path:
+    """Absolute paths stay as they are; relative ones are taken from project_dir."""
+    path = Path(value).expanduser()
+    return path if path.is_absolute() else (project_dir / path).resolve()
+
+
+class _Reader:
+    """Typed access to merged .env + environment values. Empty means unset."""
+
+    def __init__(self, values: Mapping[str, Optional[str]]):
+        self.values = values
+
+    def text(self, name: str, default: Optional[str] = None) -> Optional[str]:
+        return self.values.get(name) or default
+
+    def integer(self, name: str, default: str) -> int:
+        raw = self.text(name, default)
+        try:
+            return int(raw)
+        except ValueError:
+            raise ConfigError(f"{name} must be a whole number, got {raw!r}.") from None
+
+    def number(self, name: str, default: str) -> float:
+        raw = self.text(name, default)
+        try:
+            return float(raw)
+        except ValueError:
+            raise ConfigError(f"{name} must be a number, got {raw!r}.") from None
+
+    def flag(self, name: str, default: str) -> bool:
+        return self.text(name, default).strip().lower() in {"1", "true", "yes", "on"}
+
+
+@dataclass(frozen=True)
+class Settings:
+    project_dir: Path
+    env_file: Path
+
+    # --- Step 1-2: input document and extraction ----------------------------
+    # Default file for `ingest` when no path is given on the command line.
+    pdf_path: Path
+    # Repairs PDFs whose subset fonts have no ToUnicode map (see
+    # rag_kit/pdf_font_repair.py). Off by default: ordinary PDFs do not need it.
+    pdf_font_repair: bool = False
+
+    # --- Step 4: chunking ---------------------------------------------------
+    chunk_size: int = 1000
+    chunk_overlap: int = 150
+
+    # --- Step 5: embedding --------------------------------------------------
+    google_api_key: Optional[str] = field(default=None, repr=False)
+    # "huggingface" runs a local model - no API key, no quota, no cost.
+    # "google" uses Gemini embeddings (needs a real API key with embedding quota).
+    embedding_provider: str = "huggingface"
+    embedding_model: str = "BAAI/bge-m3"
+    # None => detect the real dimension at runtime instead of hard-coding it.
+    embedding_dim: Optional[int] = None
+    embed_batch_size: int = 90
+    # Gemini free tier allows ~100 embedded texts / minute; pause between batches
+    # so a large PDF does not trip a 429. Not needed for a local model.
+    embed_sleep: float = 0.0
+
+    # --- Step 6: Qdrant -----------------------------------------------------
+    qdrant_url: str = "http://localhost:6333"
+    qdrant_api_key: Optional[str] = field(default=None, repr=False)
+    # Local on-disk mode, no server. Takes priority over qdrant_url when set.
+    qdrant_path: Optional[Path] = None
+    collection_name: str = "pdf_rag"
+
+    # --- Step 9: retrieval --------------------------------------------------
+    top_k: int = 4
+    # Chunks scoring below this are treated as "not relevant". None keeps every hit.
+    min_score: Optional[float] = None
+
+    # --- Step 10: Gemini LLM ------------------------------------------------
+    llm_model: str = "gemini-2.5-flash"
+    temperature: float = 0.2
+
+    @classmethod
+    def load(cls, project_dir=None, env_file=None,
+             environ: Optional[Mapping[str, str]] = None) -> "Settings":
+        """Read `env_file` (default: $RAG_ENV_FILE, else project_dir/.env) and
+        overlay `environ` (default: os.environ). Neither is modified.
+
+        Without project_dir the project is found with find_project_dir()."""
+        environ = os.environ if environ is None else environ
+        if project_dir is None:
+            project_dir = find_project_dir(environ=environ)
+        else:
+            project_dir = _checked_project_dir(project_dir, "project_dir")
+        env_file = Path(env_file or environ.get("RAG_ENV_FILE") or (project_dir / ".env"))
+        values = {**dotenv_values(env_file), **environ}  # environment wins, as before
+        env = _Reader(values)
+
+        provider = env.text("EMBEDDING_PROVIDER", "huggingface").lower()
+        qdrant_path = env.text("QDRANT_PATH")
+        min_score = env.text("MIN_SCORE")
+        return cls(
+            project_dir=project_dir,
+            env_file=env_file,
+            pdf_path=_resolve(project_dir, env.text("PDF_PATH", "data/sample.pdf")),
+            pdf_font_repair=env.flag("PDF_FONT_REPAIR", "false"),
+            chunk_size=env.integer("CHUNK_SIZE", "1000"),
+            chunk_overlap=env.integer("CHUNK_OVERLAP", "150"),
+            google_api_key=env.text("GOOGLE_API_KEY"),
+            embedding_provider=provider,
+            embedding_model=env.text("EMBEDDING_MODEL", "BAAI/bge-m3" if provider == "huggingface"
+                                    else "models/gemini-embedding-001"),
+            embedding_dim=env.integer("EMBEDDING_DIM", "0") or None,
+            embed_batch_size=env.integer("EMBED_BATCH_SIZE", "90"),
+            embed_sleep=env.number("EMBED_SLEEP", "60" if provider == "google" else "0"),
+            qdrant_url=env.text("QDRANT_URL", "http://localhost:6333"),
+            qdrant_api_key=env.text("QDRANT_API_KEY"),
+            qdrant_path=_resolve(project_dir, qdrant_path) if qdrant_path else None,
+            collection_name=env.text("COLLECTION_NAME", "pdf_rag"),
+            top_k=env.integer("TOP_K", "4"),
+            min_score=env.number("MIN_SCORE", "0") if min_score else None,
+            llm_model=env.text("LLM_MODEL", "gemini-2.5-flash"),
+            temperature=env.number("TEMPERATURE", "0.2"),
+        )
+
+    def replace(self, **changes) -> "Settings":
+        return dataclasses.replace(self, **changes)
+
+    def resolve_path(self, value) -> Path:
+        return _resolve(self.project_dir, value)
+
+    def require_api_key(self) -> None:
+        if not self.google_api_key:
+            raise ConfigError(
+                f"GOOGLE_API_KEY is not set. Copy .env.example to {self.env_file} and put your "
+                "Gemini API key there (https://aistudio.google.com/app/apikey)."
+            )
+
+    def validate(self) -> None:
+        """Fail early on settings that would only break halfway through a run."""
+        if self.embedding_provider not in {"huggingface", "google"}:
+            raise ConfigError(
+                "EMBEDDING_PROVIDER must be 'huggingface' or 'google', "
+                f"got {self.embedding_provider!r}."
+            )
+        if self.chunk_size <= 0 or not 0 <= self.chunk_overlap < self.chunk_size:
+            raise ConfigError(
+                f"Need CHUNK_SIZE > 0 and 0 <= CHUNK_OVERLAP < CHUNK_SIZE "
+                f"(got {self.chunk_size} / {self.chunk_overlap})."
+            )
+        if self.top_k <= 0:
+            raise ConfigError(f"TOP_K must be at least 1, got {self.top_k}.")
+        if self.embedding_provider == "google":
+            self.require_api_key()
+
+
+_default: Optional[Settings] = None
+
+
+def set_default_settings(settings: Settings) -> None:
+    """Settings used by functions that are not given any."""
+    global _default
+    _default = settings
+
+
+def default_settings() -> Settings:
+    """The settings registered with set_default_settings(); if none were, the
+    project found by find_project_dir() is loaded once and kept."""
+    global _default
+    if _default is None:
+        _default = Settings.load()
+    return _default

rag_kit/step01_load_pdf.py

@@ -0,0 +1,38 @@
+"""STEP 1 - Input document.
+
+Validate the file before anything else, so a wrong path fails here
+instead of halfway through the pipeline. PDF, plain text and Markdown are
+supported (see rag_kit.settings.SUPPORTED_EXTENSIONS).
+"""
+
+import logging
+from pathlib import Path
+
+from rag_kit.errors import UnsupportedFileError
+from rag_kit.settings import SUPPORTED_EXTENSIONS
+
+logger = logging.getLogger(__name__)
+
+
+def load_document(path) -> Path:
+    path = Path(path).expanduser().resolve()
+
+    if not path.exists():
+        raise UnsupportedFileError(f"File not found: {path}")
+    if not path.is_file():
+        raise UnsupportedFileError(f"Not a file: {path}")
+    if path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+        raise UnsupportedFileError(
+            f"Unsupported file type '{path.suffix or '(none)'}' for {path.name}. "
+            f"Supported: {', '.join(SUPPORTED_EXTENSIONS)}."
+        )
+    if path.stat().st_size == 0:
+        raise UnsupportedFileError(f"File is empty: {path.name}")
+
+    size_kb = path.stat().st_size / 1024
+    logger.info(f"[1] Document   : {path.name} ({size_kb:.1f} KB)")
+    return path
+
+
+# Kept for scripts written against the original PDF-only pipeline.
+load_pdf = load_document

rag_kit/step02_extract_text.py

@@ -0,0 +1,138 @@
+"""STEP 2 - Text extraction.
+
+PDFs become one LangChain Document per page, with the 1-based page number in
+metadata - that number is what step 11 cites. Text and Markdown files become a
+single Document with no page number.
+
+Every Document carries the same metadata keys:
+    source_file   file name, used to replace / delete a document later
+    doc_id        content hash, changes when the file changes
+    file_type     "pdf", "txt" or "md"
+    page          0-based page index (None for text files)
+    page_number   1-based page number (None for text files)
+
+Optional font repair (PDF_FONT_REPAIR=true or font_repair=True): some PDFs embed
+subset fonts with no ToUnicode map, and every extractor then returns scrambled
+glyph codes like `& ? ! & &`. `pdf_font_repair.repair_page` undoes the specific
+substitution cipher of this project's grammar PDF. Leave it off for ordinary
+PDFs.
+
+After extraction every PDF page is scored (alphabetic ratio + English-word
+ratio). Pages that still look like garbage are reported, not silently embedded -
+that is the point where OCR would be the next step.
+"""
+
+import hashlib
+import logging
+from pathlib import Path
+from typing import List, Optional
+
+from langchain_core.documents import Document
+
+from rag_kit.errors import ExtractionError
+from rag_kit.pdf_font_repair import is_readable, repair_page, text_quality
+from rag_kit.settings import Settings, default_settings
+
+logger = logging.getLogger(__name__)
+
+
+def document_id(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
+
+
+def extract_text(path: Path, font_repair: Optional[bool] = None,
+                 settings: Optional[Settings] = None) -> List[Document]:
+    path = Path(path)
+    if path.suffix.lower() == ".pdf":
+        if font_repair is None:
+            font_repair = (settings or default_settings()).pdf_font_repair
+        return _extract_pdf(path, font_repair)
+    return _extract_plain_text(path)
+
+
+def _metadata(path: Path, doc_id: str, page: Optional[int]) -> dict:
+    return {
+        "source_file": path.name,
+        "doc_id": doc_id,
+        "file_type": path.suffix.lower().lstrip("."),
+        "page": page,
+        "page_number": None if page is None else page + 1,
+    }
+
+
+def _extract_plain_text(path: Path) -> List[Document]:
+    try:
+        text = path.read_text(encoding="utf-8-sig")
+    except UnicodeDecodeError as exc:
+        raise ExtractionError(
+            f"{path.name} is not valid UTF-8 text ({exc.reason} at byte {exc.start}). "
+            "Re-save it as UTF-8 and ingest again."
+        ) from exc
+    except OSError as exc:
+        raise ExtractionError(f"Could not read {path.name}: {exc}") from exc
+
+    logger.info(f"[2] Extracted  : 1 text document, {len(text):,} characters")
+    return [Document(page_content=text, metadata=_metadata(path, document_id(path), None))]
+
+
+def _extract_pdf(path: Path, font_repair: bool) -> List[Document]:
+    import pymupdf
+
+    try:
+        doc = pymupdf.open(str(path))
+    except Exception as exc:  # pymupdf raises several types for broken files
+        raise ExtractionError(f"Could not open {path.name} as a PDF: {exc}") from exc
+
+    if doc.needs_pass:
+        doc.close()
+        raise ExtractionError(
+            f"{path.name} is password-protected. Decrypt it first (e.g. `qpdf --decrypt`)."
+        )
+
+    doc_id = document_id(path)
+    pages: List[Document] = []
+    readable = corrupted = blank = 0
+    worst: List[tuple] = []
+
+    try:
+        for index in range(doc.page_count):
+            page = doc[index]
+            text = repair_page(page) if font_repair else page.get_text("text")
+            page_number = index + 1
+
+            if len(text.strip()) < 20:
+                blank += 1
+            elif is_readable(text):
+                readable += 1
+            else:
+                corrupted += 1
+                alpha_ratio, word_ratio = text_quality(text)
+                worst.append((round(word_ratio, 2), round(alpha_ratio, 2), page_number))
+
+            pages.append(Document(page_content=text, metadata=_metadata(path, doc_id, index)))
+    except Exception as exc:
+        raise ExtractionError(f"Text extraction failed on {path.name}: {exc}") from exc
+    finally:
+        doc.close()
+
+    total_chars = sum(len(p.page_content) for p in pages)
+    logger.info(f"[2] Extracted  : {len(pages)} pages, {total_chars:,} characters"
+                f"{' (font repair on)' if font_repair else ''}")
+    logger.info(f"    quality    : {readable} readable, {corrupted} corrupted, {blank} blank")
+
+    if corrupted:
+        worst.sort()
+        preview = ", ".join(f"p{pn}" for _, _, pn in worst[:10])
+        logger.warning(f"    ! {corrupted} page(s) look corrupted: {preview}")
+        if font_repair:
+            logger.warning("      The maps in pdf_font_repair.py are PDF-specific; "
+                           "OCR is the fallback.")
+        else:
+            logger.warning("      If the text is scrambled glyphs, try PDF_FONT_REPAIR=true; "
+                           "otherwise add OCR (pytesseract / ocrmypdf).")
+
+    if total_chars < 100:
+        logger.warning("    ! Almost no text found - this PDF is probably a scan. Add OCR "
+                       "before this pipeline can work.")
+
+    return pages

rag_kit/step03_clean_text.py

@@ -0,0 +1,76 @@
+"""STEP 3 - Text cleaning.
+
+PDF text arrives with hard line breaks, hyphen-split words, repeated
+headers/footers and stray page numbers. All of that pollutes the embeddings,
+so it gets removed before chunking.
+"""
+
+import logging
+import re
+from collections import Counter
+from typing import List
+
+from langchain_core.documents import Document
+
+logger = logging.getLogger(__name__)
+
+_CONTROL_CHARS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
+_HYPHEN_BREAK = re.compile(r"(\w)-\n(\w)")
+_SINGLE_NEWLINE = re.compile(r"(?<![.!?:;।])\n(?![\n\-\u2022\d])")
+_PAGE_NUMBER_LINE = re.compile(r"^\s*(page\s*)?[\divxlIVXL]{1,4}\s*$", re.MULTILINE | re.IGNORECASE)
+_SPACES = re.compile(r"[ \t]{2,}")
+_BLANK_LINES = re.compile(r"\n{3,}")
+
+
+def clean_text(text: str) -> str:
+    text = text.replace("\u00ad", "")            # soft hyphen
+    text = text.replace("\ufeff", "")            # BOM
+    text = _CONTROL_CHARS.sub(" ", text)
+    text = _HYPHEN_BREAK.sub(r"\1\2", text)      # "informa-\ntion" -> "information"
+    text = _PAGE_NUMBER_LINE.sub("", text)
+    text = _SINGLE_NEWLINE.sub(" ", text)        # unwrap mid-sentence line breaks
+    text = _SPACES.sub(" ", text)
+    text = _BLANK_LINES.sub("\n\n", text)
+    return text.strip()
+
+
+def _find_repeating_lines(docs: List[Document], threshold: float = 0.6) -> set:
+    """Lines that show up on most pages are headers/footers, not content."""
+    if len(docs) < 4:
+        return set()
+
+    counter = Counter()
+    for doc in docs:
+        for line in {ln.strip() for ln in doc.page_content.splitlines() if ln.strip()}:
+            if len(line) < 80:
+                counter[line] += 1
+
+    cutoff = len(docs) * threshold
+    return {line for line, count in counter.items() if count >= cutoff}
+
+
+def _drop_lines(text: str, boilerplate: set) -> str:
+    """Blank out whole lines that are boilerplate. Matching whole lines (not
+    substrings) keeps a short header like "Chapter" from being cut out of the
+    sentences that happen to contain it."""
+    if not boilerplate:
+        return text
+    return "\n".join("" if line.strip() in boilerplate else line for line in text.split("\n"))
+
+
+def clean_documents(docs: List[Document], min_chars: int = 50) -> List[Document]:
+    """Return cleaned copies; the input Documents are left untouched."""
+    boilerplate = _find_repeating_lines(docs)
+
+    cleaned: List[Document] = []
+    for doc in docs:
+        body = clean_text(_drop_lines(doc.page_content, boilerplate))
+        if len(body) < min_chars:      # blank or near-blank page
+            continue
+
+        cleaned.append(Document(id=doc.id, page_content=body, metadata=dict(doc.metadata)))
+
+    dropped = len(docs) - len(cleaned)
+    logger.info(f"[3] Cleaned    : {len(cleaned)} pages kept, {dropped} dropped, "
+                f"{len(boilerplate)} repeating header/footer lines removed")
+    return cleaned

rag_kit/step04_chunking.py

@@ -0,0 +1,36 @@
+"""STEP 4 - Chunking.
+
+Split each page into overlapping chunks. The overlap keeps a sentence that
+straddles a boundary retrievable from either side.
+"""
+
+import logging
+from typing import List
+
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+logger = logging.getLogger(__name__)
+
+# Bengali danda (।) is included so Bangla PDFs split on sentence ends too.
+SEPARATORS = ["\n\n", "\n", "। ", ". ", "? ", "! ", "; ", ", ", " ", ""]
+
+
+def chunk_documents(docs: List[Document], chunk_size: int, chunk_overlap: int) -> List[Document]:
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=SEPARATORS,
+        length_function=len,
+        add_start_index=True,
+    )
+
+    chunks = splitter.split_documents(docs)
+
+    for i, chunk in enumerate(chunks):
+        chunk.metadata["chunk_id"] = i
+
+    avg = sum(len(c.page_content) for c in chunks) / max(len(chunks), 1)
+    logger.info(f"[4] Chunked    : {len(chunks)} chunks "
+                f"(size={chunk_size}, overlap={chunk_overlap}, avg={avg:.0f} chars)")
+    return chunks

rag_kit/step05_embedding.py

@@ -0,0 +1,77 @@
+"""STEP 5 - Embedding.
+
+One embedding object is shared by ingestion (step 6) and querying (step 8) -
+the same model must be used on both sides or the vectors are not comparable.
+
+Two providers, chosen by EMBEDDING_PROVIDER in .env:
+  huggingface : a local sentence-transformers model, no API key, no quota
+  google      : Gemini embeddings (needs an API key with embedding quota)
+"""
+
+import logging
+from typing import Optional
+
+from rag_kit.errors import EmbeddingProviderError
+from rag_kit.settings import Settings, default_settings
+
+logger = logging.getLogger(__name__)
+
+LOCAL_EXTRA_HINT = ('EMBEDDING_PROVIDER=huggingface needs the local extra: '
+                    'pip install "ragkit-dev[local]"')
+GEMINI_EXTRA_HINT = ('EMBEDDING_PROVIDER=google needs the gemini extra: '
+                     'pip install "ragkit-dev[gemini]"')
+
+
+def get_embeddings(settings: Optional[Settings] = None):
+    settings = settings or default_settings()
+    settings.validate()
+    if settings.embedding_provider == "huggingface":
+        try:
+            from langchain_huggingface import HuggingFaceEmbeddings
+        except ImportError as exc:
+            raise EmbeddingProviderError(f"{LOCAL_EXTRA_HINT} ({exc}).") from exc
+        try:
+            embeddings = HuggingFaceEmbeddings(model_name=settings.embedding_model)
+        except ImportError as exc:  # sentence-transformers is imported only here
+            raise EmbeddingProviderError(f"{LOCAL_EXTRA_HINT} ({exc}).") from exc
+        except Exception as exc:
+            raise EmbeddingProviderError(
+                f"Could not load embedding model {settings.embedding_model!r}: {exc}. "
+                "Check the model name and your network (the first run downloads it "
+                "into the Hugging Face cache)."
+            ) from exc
+        logger.info(f"[5] Embedding  : {settings.embedding_model} (local)")
+        return embeddings
+
+    try:
+        from langchain_google_genai import GoogleGenerativeAIEmbeddings
+    except ImportError as exc:
+        raise EmbeddingProviderError(f"{GEMINI_EXTRA_HINT} ({exc}).") from exc
+
+    embeddings = GoogleGenerativeAIEmbeddings(
+        model=settings.embedding_model,
+        google_api_key=settings.google_api_key,
+    )
+    logger.info(f"[5] Embedding  : {settings.embedding_model}")
+    return embeddings
+
+
+def detect_dimension(embeddings, settings: Optional[Settings] = None) -> int:
+    """Ask the model for one vector and measure it.
+
+    Safer than hard-coding a size: it changes with the model (384 for
+    all-MiniLM-L6-v2, 3072 for gemini-embedding-001), and a mismatch only
+    shows up later as a Qdrant dimension error.
+    """
+    settings = settings or default_settings()
+    if settings.embedding_dim:
+        return settings.embedding_dim
+
+    try:
+        dim = len(embeddings.embed_query("dimension probe"))
+    except Exception as exc:
+        raise EmbeddingProviderError(
+            f"Embedding model {settings.embedding_model!r} failed on a test input: {exc}"
+        ) from exc
+    logger.info(f"    vector size: {dim}")
+    return dim