ragkit-dev 0.1.0__py3-none-any.whl

rag_kit/env.py

@@ -0,0 +1,84 @@
+"""Explicitly copy selected .env values into os.environ.
+
+`Settings.load()` never touches os.environ. Some libraries, however, only read
+their configuration from the environment - Hugging Face reads HF_HOME when it
+is first imported, and HTTP clients read proxy variables - so a value placed
+in the project's .env would otherwise be ignored.
+
+`export_env_file(settings)` closes that gap with an allow-list. Only the
+application entry points call it: the CLI, the default API app (`main.app`)
+at startup, and the Streamlit UI. Importing the package, building a Pipeline
+or calling create_app() never exports anything.
+
+Policy:
+  * Exported: variables other libraries read from the environment (see
+    EXPORT_PREFIXES / EXPORT_NAMES), plus any names listed in RAG_EXPORT_ENV
+    (comma-separated, in .env or the environment).
+  * Never exported: rag-kit's own settings, including GOOGLE_API_KEY and
+    QDRANT_API_KEY, and any RAG_* variable. Settings reads those directly, and
+    server/UI child processes re-read .env themselves.
+  * Variables already set in the process win; empty values are skipped.
+  * Only names are ever logged or returned, never values. HF_TOKEN is a secret
+    but is exported, because huggingface_hub reads it only from the environment.
+"""
+
+import logging
+import os
+from typing import List, MutableMapping, Optional
+
+from dotenv import dotenv_values
+
+from rag_kit.settings import SETTING_NAMES, Settings
+
+logger = logging.getLogger(__name__)
+
+OPT_IN_VARIABLE = "RAG_EXPORT_ENV"
+
+# Hugging Face, transformers, sentence-transformers and torch runtime settings.
+EXPORT_PREFIXES = ("HF_", "HUGGINGFACE_", "TRANSFORMERS_", "SENTENCE_TRANSFORMERS_", "TORCH_")
+EXPORT_NAMES = frozenset({
+    "TOKENIZERS_PARALLELISM", "CUDA_VISIBLE_DEVICES", "OMP_NUM_THREADS",
+    "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR", "CURL_CA_BUNDLE",
+})
+# Proxy variables are honoured in either case by most HTTP clients.
+PROXY_NAMES = frozenset({"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "ALL_PROXY"})
+
+
+def is_rag_kit_setting(name: str) -> bool:
+    return name in SETTING_NAMES or name.upper().startswith("RAG_")
+
+
+def _allow_listed(name: str) -> bool:
+    return (name in EXPORT_NAMES or name.upper() in PROXY_NAMES
+            or name.startswith(EXPORT_PREFIXES))
+
+
+def _opt_in_names(file_values: dict, environ: MutableMapping[str, str]) -> set:
+    raw = environ.get(OPT_IN_VARIABLE) or file_values.get(OPT_IN_VARIABLE) or ""
+    return {name.strip() for name in raw.split(",") if name.strip()}
+
+
+def export_env_file(settings: Settings, environ: Optional[MutableMapping[str, str]] = None,
+                    override: bool = False) -> List[str]:
+    """Copy allow-listed and opted-in keys from settings.env_file into `environ`
+    (default: os.environ). Returns the sorted names that were set."""
+    environ = os.environ if environ is None else environ
+    file_values = dotenv_values(settings.env_file)
+    opt_in = _opt_in_names(file_values, environ)
+
+    refused = sorted(name for name in opt_in if is_rag_kit_setting(name))
+    if refused:
+        logger.warning(f"    ! {OPT_IN_VARIABLE} lists rag-kit settings, which are never "
+                       f"exported: {', '.join(refused)}")
+
+    exported = []
+    for name, value in file_values.items():
+        if not value or is_rag_kit_setting(name):
+            continue
+        if not (_allow_listed(name) or name in opt_in):
+            continue
+        if name in environ and not override:
+            continue  # the process environment wins
+        environ[name] = value
+        exported.append(name)
+    return sorted(exported)

rag_kit/errors.py

@@ -0,0 +1,37 @@
+"""Errors the pipeline raises on purpose.
+
+Every message says what went wrong *and* what to do about it, so the CLI, the
+FastAPI service and the Streamlit UI can show it to the user as-is.
+"""
+
+
+class RagError(RuntimeError):
+    """Base class for expected, user-fixable failures."""
+
+
+class ConfigError(RagError):
+    """A setting is missing or invalid. The message says how to fix it."""
+
+
+class UnsupportedFileError(RagError):
+    """Step 1: the input is missing, empty, or not a supported type."""
+
+
+class ExtractionError(RagError):
+    """Step 2: the file could not be opened or read."""
+
+
+class EmptyDocumentError(RagError):
+    """Steps 2-4: the file was read but produced no usable text."""
+
+
+class EmbeddingProviderError(RagError):
+    """Step 5: the embedding model could not be loaded or called."""
+
+
+class VectorStoreError(RagError):
+    """Step 6/9: Qdrant is unreachable, locked, missing, or incompatible."""
+
+
+class GenerationError(RagError):
+    """Step 10: the LLM could not be created or called."""

rag_kit/log.py

@@ -0,0 +1,42 @@
+"""Step progress goes through logging, not print().
+
+Library code logs to the "rag_kit" logger, which has only a NullHandler, so a
+program that imports the pipeline sees no output unless it asks for it.
+Applications (the CLI, the API service, the Streamlit UI) call
+`show_progress()` to print the same "[1] Document : ..." lines as before.
+"""
+
+import logging
+import sys
+
+LOGGER_NAME = "rag_kit"  # rag_kit/__init__.py gives it a NullHandler
+
+
+class _CurrentStdout(logging.StreamHandler):
+    """Writes to whatever sys.stdout is when a record is emitted, so
+    contextlib.redirect_stdout (the CLI's --json mode) and test capture still
+    apply - exactly like the print() calls this replaces."""
+
+    def __init__(self):
+        super().__init__(sys.stdout)
+
+    @property
+    def stream(self):
+        return sys.stdout
+
+    @stream.setter
+    def stream(self, _value):
+        pass
+
+
+def show_progress(level: int = logging.INFO) -> logging.Handler:
+    """Print step progress (message text only) to stdout. Safe to call twice."""
+    logger = logging.getLogger(LOGGER_NAME)
+    for handler in [h for h in logger.handlers if isinstance(h, _CurrentStdout)]:
+        logger.removeHandler(handler)
+    handler = _CurrentStdout()
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    logger.propagate = False  # do not print twice if the root logger is configured
+    return handler

rag_kit/pdf_font_repair.py

@@ -0,0 +1,132 @@
+"""Font-encoding repair for PDFs with broken glyph maps.
+
+Some PDFs (this project's `-Free-English-Grammar.pdf` among them - watermarked by
+pdforall.com) embed subset fonts with *no* ToUnicode CMap. Every PDF text
+extractor - pypdf and PyMuPDF alike - then returns the raw glyph indices, which
+come out as `& ? ! & & ? !` garbage.
+
+The scramble is a plain, document-wide substitution cipher, so it is fully
+recoverable without OCR. Two subset fonts carry the body:
+
+    TTE1AEB1B8t00  regular body text  (~93 % of characters)
+    TTE1AEB0C8t00  bold headings / emphasised grammar terms
+
+The maps below were reconstructed by decoding known words. Anything that is not
+one of those two fonts (Times-Roman digits, the Helvetica watermark) already
+extracts correctly and is passed through untouched.
+
+`repair_page` rebuilds a page's text span by span, so page boundaries - and
+therefore the page numbers cited in step 11 - stay exactly where PyMuPDF puts
+them.
+"""
+
+from __future__ import annotations
+
+import re
+
+BODY_FONT = "TTE1AEB1B8t00"
+BOLD_FONT = "TTE1AEB0C8t00"
+
+# --- regular body font -------------------------------------------------------
+BODY_MAP = {
+    0x01: " ", 0x02: "b", 0x03: "y", 0x04: "S", 0x05: "e", 0x06: "c",
+    0x07: "o", 0x08: "n", 0x09: "d", 0x0A: "i", 0x0B: "t", 0x0C: ".",
+    0x0D: "C", 0x0E: "p", 0x0F: "r", 0x10: "g", 0x11: "h", 0x13: "2",
+    0x14: "0", 0x15: "T", 0x16: "m", 0x17: "a", 0x18: "l", 0x19: "s",
+    0x1A: "k", 0x1B: "u", 0x1C: "f", 0x1F: "w", 0x20: "v",
+    0x21: ",", 0x22: "x", 0x23: "I", 0x24: ":", 0x26: "A", 0x27: "E",
+    0x28: "F", 0x29: "G", 0x2A: "N", 0x2B: "z", 0x2C: "H", 0x2D: "U",
+    0x2E: "D", 0x2F: "O", 0x30: "R", 0x31: "M", 0x32: "L", 0x33: "V",
+    0x34: "B", 0x35: "W", 0x36: "P", 0x37: "1", 0x38: "3", 0x39: "4",
+    0x3A: "Q", 0x3B: "q", 0x3C: "5", 0x3D: "6", 0x3E: "7", 0x3F: "j",
+    0x40: "8", 0x42: "9", 0x43: "/", 0x44: "(", 0x45: ")", 0x46: "?",
+    0x47: "’", 0x48: "Y", 0x49: " ", 0x4A: '"', 0x4B: '"', 0x4C: "J",
+    0x4E: "!", 0x25: "•",
+}
+
+# --- bold headings / emphasis font ----------------------------------------
+BOLD_MAP = {
+    0x01: " ", 0x02: "s", 0x03: "e", 0x04: "y", 0x05: "f", 0x06: "i",
+    0x08: "o", 0x09: "c", 0x0A: "a", 0x0B: "w", 0x0D: "s", 0x0E: "h",
+    0x0F: "m", 0x11: "n", 0x13: "l", 0x15: "r", 0x1A: "a", 0x1F: "t",
+    0x22: "C", 0x24: "f", 0x26: "t", 0x27: "b", 0x29: "v", 0x2C: "d",
+    0x30: "u", 0x36: "p", 0x37: "’", 0x38: "W", 0x39: ",", 0x3A: "u",
+    0x3B: "v", 0x3C: "j", 0x3E: "k", 0x3F: "Q", 0x41: "Q", 0x46: ")",
+}
+
+# Capital glyphs the font draws as "<capital><phantom 0x20>"; the trailing
+# glyph 0x20 (otherwise 'v') must be swallowed.
+_CAP_ABSORB_V = {0x29, 0x2C, 0x2D, 0x2F, 0x31}
+
+# characters worth keeping verbatim when a glyph is not in the map
+_KEEP = set(" .,;:!?()[]{}\"'‘’“”-–—/&%$#@*+=<>0123456789\n\t")
+
+_PHANTOM_V = re.compile(r"\b([HOMGU])v(?=[a-z])")
+
+
+def _decode_span(raw: str, fmap: dict, absorb_v: bool) -> str:
+    out = []
+    i, n = 0, len(raw)
+    while i < n:
+        cp = ord(raw[i])
+        if cp in fmap:
+            out.append(fmap[cp])
+            if absorb_v and cp in _CAP_ABSORB_V and i + 1 < n and ord(raw[i + 1]) == 0x20:
+                i += 1  # swallow the phantom 'v' that trails this capital
+        elif cp < 0x20:
+            pass  # unmapped control glyph - drop it rather than emit noise
+        elif raw[i] in _KEEP or raw[i].isalnum():
+            out.append(raw[i])
+        i += 1
+    return "".join(out)
+
+
+def repair_page(page) -> str:
+    """Return the readable text of one ``pymupdf`` page.
+
+    Broken subset fonts are decoded via the substitution maps above; every other
+    font is passed through as PyMuPDF extracted it.
+    """
+    data = page.get_text("rawdict")
+    lines = []
+    for block in data["blocks"]:
+        for line in block.get("lines", []):
+            buf = []
+            for span in line["spans"]:
+                raw = "".join(ch["c"] for ch in span["chars"])
+                font = span["font"]
+                if font == BODY_FONT:
+                    buf.append(_decode_span(raw, BODY_MAP, absorb_v=True))
+                elif font == BOLD_FONT:
+                    buf.append(_decode_span(raw, BOLD_MAP, absorb_v=False))
+                else:
+                    buf.append(raw)
+            lines.append("".join(buf))
+    return _PHANTOM_V.sub(r"\1", "\n".join(lines))
+
+
+# --- quality scoring -------------------------------------------------------
+_WORD = re.compile(r"[A-Za-z][A-Za-z'’-]*")
+_VOWEL = re.compile(r"[aeiouyAEIOUY]")
+
+
+def text_quality(text: str) -> tuple[float, float]:
+    """(alphabetic-character ratio, plausible-English-word ratio) for `text`."""
+    stripped = [c for c in text if not c.isspace()]
+    if not stripped:
+        return 0.0, 0.0
+    alpha_ratio = sum(c.isalpha() for c in stripped) / len(stripped)
+
+    words = _WORD.findall(text)
+    if not words:
+        return alpha_ratio, 0.0
+    good = sum(
+        1 for w in words
+        if 1 <= len(w) <= 20 and (len(w) <= 2 or _VOWEL.search(w))
+    )
+    return alpha_ratio, good / len(words)
+
+
+def is_readable(text: str, min_alpha: float = 0.70, min_words: float = 0.80) -> bool:
+    alpha_ratio, word_ratio = text_quality(text)
+    return alpha_ratio >= min_alpha and word_ratio >= min_words

rag_kit/pipeline.py

@@ -0,0 +1,169 @@
+"""The 11 steps wired together, shared by the CLI, FastAPI and Streamlit.
+
+    pipeline = Pipeline()                      # nothing is loaded yet
+    pipeline.ingest("data/book.pdf")           # steps 1-6
+    result = pipeline.ask("What is a noun?")   # steps 7-11 (minus printing)
+    pipeline.close()
+
+The embedding model, Qdrant client and LLM are created on first use and then
+reused. A local Qdrant store allows one client per process, so keep one
+Pipeline per process.
+"""
+
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from langchain_core.documents import Document
+
+from rag_kit.errors import EmptyDocumentError
+from rag_kit.settings import Settings, default_settings
+from rag_kit.step01_load_pdf import load_document
+from rag_kit.step02_extract_text import extract_text
+from rag_kit.step03_clean_text import clean_documents
+from rag_kit.step04_chunking import chunk_documents
+from rag_kit.step05_embedding import get_embeddings
+from rag_kit.step06_vector_store import (
+    delete_document,
+    get_client,
+    list_documents,
+    open_vector_store,
+    store_chunks,
+)
+from rag_kit.step08_query_embedding import embed_question
+from rag_kit.step09_retrieve import retrieve
+from rag_kit.step10_generate import NO_RESULTS_ANSWER, generate_answer, get_llm
+from rag_kit.step11_answer import build_sources, strip_markers
+
+ScoredDocs = List[Tuple[Document, float]]
+
+
+@dataclass
+class AnswerResult:
+    question: str
+    answer: str                      # raw model output, with [S1] markers
+    status: str                      # "answered" or "no_results"
+    sources: List[dict] = field(default_factory=list)
+    scored_docs: ScoredDocs = field(default_factory=list, repr=False)
+
+    @property
+    def answer_without_markers(self) -> str:
+        return strip_markers(self.answer)
+
+
+class Pipeline:
+    def __init__(self, collection: Optional[str] = None, embeddings=None, client=None, llm=None,
+                 settings: Optional[Settings] = None):
+        self.settings = settings or default_settings()
+        self.collection = collection or self.settings.collection_name
+        self._embeddings = embeddings
+        self._client = client
+        self._llm = llm
+        self._vector_store = None
+        self._lock = threading.RLock()
+
+    # --- lazily created resources -------------------------------------------
+    @property
+    def embeddings(self):
+        with self._lock:
+            if self._embeddings is None:
+                self._embeddings = get_embeddings(self.settings)          # 5
+            return self._embeddings
+
+    def client(self, create: bool = False):
+        with self._lock:
+            if self._client is None:
+                self._client = get_client(create=create, settings=self.settings)
+            return self._client
+
+    @property
+    def vector_store(self):
+        with self._lock:
+            if self._vector_store is None:                                 # 6 (read side)
+                self._vector_store = open_vector_store(self.embeddings, self.client(),
+                                                       self.collection, self.settings)
+            return self._vector_store
+
+    @property
+    def llm(self):
+        with self._lock:
+            if self._llm is None:
+                self._llm = get_llm(self.settings)                         # 10
+            return self._llm
+
+    @property
+    def ready(self) -> bool:
+        """True once the collection has been opened and validated."""
+        return self._vector_store is not None
+
+    def warm_up(self, llm: bool = True) -> None:
+        """Load everything a query needs, so the first request is not slow."""
+        _ = self.vector_store
+        if llm:
+            _ = self.llm
+
+    def close(self) -> None:
+        with self._lock:
+            if self._client is not None:
+                self._client.close()
+            self._client = self._vector_store = None
+
+    # --- steps 1-6 -----------------------------------------------------------
+    def ingest(self, path, recreate: bool = False, font_repair: Optional[bool] = None) -> dict:
+        source = load_document(path)                                       # 1
+        pages = extract_text(source, font_repair=font_repair,              # 2
+                             settings=self.settings)
+        pages = clean_documents(pages)                                     # 3
+        if not pages:
+            raise EmptyDocumentError(
+                f"No usable text in {source.name} after cleaning. If it is a scanned "
+                "PDF, run OCR first (e.g. `ocrmypdf in.pdf out.pdf`)."
+            )
+        chunks = chunk_documents(pages, self.settings.chunk_size,             # 4
+                                 self.settings.chunk_overlap)
+        with self._lock:
+            report = store_chunks(chunks, self.embeddings, recreate=recreate,     # 5 + 6
+                                  client=self.client(create=True), collection=self.collection,
+                                  settings=self.settings)
+            self._vector_store = None  # re-validate on next read (collection may be new)
+        report["file"] = source.name
+        report["pages"] = len(pages)
+        return report
+
+    # --- steps 8-11 ----------------------------------------------------------
+    def search(self, question: str, k: Optional[int] = None,
+               min_score: Optional[float] = None) -> ScoredDocs:
+        question = _require_question(question)
+        query_vector = embed_question(self.embeddings, question)           # 8
+        return retrieve(self.vector_store, query_vector, k or self.settings.top_k,  # 9
+                        self.settings.min_score if min_score is None else min_score)
+
+    def ask(self, question: str, k: Optional[int] = None,
+            min_score: Optional[float] = None) -> AnswerResult:
+        question = _require_question(question)
+        scored_docs = self.search(question, k=k, min_score=min_score)
+        if not scored_docs:
+            return AnswerResult(question, NO_RESULTS_ANSWER, "no_results")
+        answer = generate_answer(question, scored_docs, llm=self.llm,     # 10
+                                 settings=self.settings)
+        return AnswerResult(question, answer, "answered",                  # 11
+                            build_sources(scored_docs), scored_docs)
+
+    # --- maintenance ---------------------------------------------------------
+    def documents(self) -> dict:
+        return list_documents(self.client(), self.collection, self.settings)
+
+    def delete(self, source_file: str) -> int:
+        with self._lock:
+            removed = delete_document(self.client(), Path(source_file).name, self.collection,
+                                      self.settings)
+            self._vector_store = None
+            return removed
+
+
+def _require_question(question: str) -> str:
+    question = (question or "").strip()
+    if not question:
+        raise ValueError("Question is empty.")
+    return question