ragdesk 0.1.0__py3-none-any.whl

ragdesk/__init__.py

@@ -0,0 +1,3 @@
+"""ragdesk — personal, local-first RAG over your own sources."""
+
+__version__ = "0.1.0"

ragdesk/__main__.py

@@ -0,0 +1,4 @@
+from ragdesk.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ragdesk/answer.py

@@ -0,0 +1,178 @@
+"""Grounded answer generation over any :mod:`ragdesk.llm` backend.
+
+The grounding gate refuses to call the LLM when retrieval is weak — the
+cosine threshold is embedder-specific, so calibrate it per embedder with the
+eval harness instead of trusting the default.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from typing import Any
+
+from ragdesk.search import Hit
+
+DEFAULT_LLM_MODEL = "qwen3.5:4b"
+# Grounded answers are short; the cap also stops small models from looping
+# (a 1k-token ramble holds the model for minutes on laptop hardware).
+# ``num_ctx`` is Ollama-specific and ignored by other backends.
+ANSWER_OPTIONS = {"num_predict": 400, "temperature": 0.2, "num_ctx": 8192}
+ANSWER_LENGTHS = {"short": 200, "medium": 400, "long": 700}
+LENGTH_HINTS = {
+    "short": "Keep the answer to two or three sentences.\n",
+    "medium": "",
+    "long": "Answer thoroughly, but stay under ten sentences.\n",
+}
+
+REFUSAL = "I could not find this in your indexed sources."
+
+PROMPT_TEMPLATE = """You are ragdesk, a retrieval assistant. Answer from the sources below, plus any
+correction the user added — a correction outranks the sources.
+Structure the answer for scanning: a one- or two-sentence summary, then short
+paragraphs; when the sources enumerate facts (names, codes, amounts, dates),
+list them as `- ` bullets on their own lines. No LaTeX, no headings, no tables.
+Cite the sources you used inline as [1], [2] and so on — end each factual
+sentence with its citation marker, but never reply with citations alone.
+If the sources do not contain the answer, say exactly:
+"{refusal}". Never use outside knowledge.
+{history}{memory}{diagram}
+Sources:
+{context}
+{corrections}
+Question: {question}
+Answer:"""
+
+HISTORY_HEADER = "Recent conversation (context only — the sources are the only source of truth):\n"
+HISTORY_TURNS = 3
+HISTORY_CHARS = 400
+MEMORY_HEADER = (
+    "Durable notes the user asked you to remember (context, still answer from the sources):\n"
+)
+CORRECTION_HEADER = (
+    "IMPORTANT — the user fixed an earlier answer to a matching question. Use the "
+    "fixed answer even when a source says otherwise, and keep citing sources:\n"
+)
+CORRECTION_QUESTION_CHARS = 200
+CORRECTION_ANSWER_CHARS = 800
+DIAGRAM_NOTE = """The user asked for a diagram. Reply with ONE fenced ```mermaid block and at
+most two sentences of context. Use ONLY this Mermaid vocabulary: `flowchart TD` or
+`flowchart LR`, nodes `id[Label]`, edges `A --> B` or `A -->|label| B`, optional grouping
+`subgraph Name ... end`, plus `classDef accent fill:#2e6b58,color:#f6f7f2,stroke:#2e6b58;`
+and `class NodeId accent;` for the node to notice first. Never invent keywords such as
+`subregion`, `flowchart graph`, `linkStyle`, or HTML labels. At most 12 nodes, short labels
+in the same language as the question.
+"""
+# Words that mean "draw me something" rather than "tell me something".
+DIAGRAM_WORDS = (
+    "diagram",
+    "flowchart",
+    "flow chart",
+    "sequence diagram",
+    "architecture diagram",
+    "draw",
+    "chart",
+    "graph",
+    "sơ đồ",
+    "biểu đồ",
+    "vẽ",
+)
+_DIAGRAM_RE = re.compile(r"\b(?:" + "|".join(re.escape(word) for word in DIAGRAM_WORDS) + r")\b")
+
+
+def wants_diagram(question: str) -> bool:
+    # Word boundaries: "graph" must not match "paragraph", "draw" not "drawback".
+    return bool(_DIAGRAM_RE.search(question.lower()))
+
+
+def _answer_length() -> str:
+    from ragdesk import settings  # noqa: PLC0415 - avoids an import cycle at load
+
+    return str(settings.load().get("answer_length") or "medium")
+
+
+def length_hint() -> str:
+    return LENGTH_HINTS.get(_answer_length(), "")
+
+
+def answer_options() -> dict:
+    return {**ANSWER_OPTIONS, "num_predict": ANSWER_LENGTHS.get(_answer_length(), 400)}
+
+
+def build_prompt(
+    question: str,
+    hits: list[Hit],
+    history: list[tuple[str, str]] | None = None,
+    memory: list[str] | None = None,
+    diagram: bool = False,
+    corrections: list[dict] | None = None,
+) -> str:
+    blocks = [f"[{i}] {hit.path}\n{hit.context}" for i, hit in enumerate(hits, start=1)]
+    lines: list[str] = []
+    for role, text in (history or [])[-HISTORY_TURNS * 2 :]:
+        speaker = "User" if role == "user" else "ragdesk"
+        lines.append(f"{speaker}: {text[:HISTORY_CHARS]}")
+    notes = [f"- {note}" for note in (memory or [])]
+    fixes = [
+        f"Fixed answer (to {str(item.get('question', ''))[:CORRECTION_QUESTION_CHARS]!r}): "
+        f"{str(item.get('answer', ''))[:CORRECTION_ANSWER_CHARS]}"
+        for item in (corrections or [])
+    ]
+    return PROMPT_TEMPLATE.format(
+        refusal=REFUSAL,
+        history=(HISTORY_HEADER + "\n".join(lines) + "\n\n") if lines else "",
+        memory=(MEMORY_HEADER + "\n".join(notes) + "\n\n") if notes else "",
+        corrections=(CORRECTION_HEADER + "\n".join(fixes) + "\n\n") if fixes else "",
+        diagram=(DIAGRAM_NOTE if diagram else "") + length_hint(),
+        context="\n\n".join(blocks),
+        question=question,
+    )
+
+
+def answer(
+    question: str,
+    hits: list[Hit],
+    llm: Any,
+    *,
+    min_cosine: float = 0.0,
+    history: list[tuple[str, str]] | None = None,
+    memory: list[str] | None = None,
+    diagram: bool = False,
+    corrections: list[dict] | None = None,
+) -> str:
+    best_cosine = max((hit.cosine for hit in hits), default=0.0)
+    if not hits or best_cosine < min_cosine:
+        return REFUSAL
+    prompt = build_prompt(question, hits, history, memory, diagram, corrections)
+    # Small models occasionally return an empty completion; one retry.
+    for _ in range(2):
+        text = str(llm.generate(prompt, answer_options())).strip()
+        if text:
+            return text
+    return REFUSAL
+
+
+def answer_stream(
+    question: str,
+    hits: list[Hit],
+    llm: Any,
+    *,
+    min_cosine: float = 0.0,
+    history: list[tuple[str, str]] | None = None,
+    memory: list[str] | None = None,
+    diagram: bool = False,
+    corrections: list[dict] | None = None,
+) -> Iterator[str]:
+    """Same contract as :func:`answer`, but yields text pieces as they arrive."""
+    best_cosine = max((hit.cosine for hit in hits), default=0.0)
+    if not hits or best_cosine < min_cosine:
+        yield REFUSAL
+        return
+    emitted = False
+    prompt = build_prompt(question, hits, history, memory, diagram, corrections)
+    for piece in llm.generate_stream(prompt, answer_options()):
+        if piece:
+            emitted = True
+            yield str(piece)
+    if not emitted:
+        yield REFUSAL

ragdesk/archive.py

@@ -0,0 +1,39 @@
+"""Shared tarball helpers for repository connectors (GitHub, GitLab)."""
+
+from __future__ import annotations
+
+import io
+import tarfile
+from pathlib import Path
+
+from ragdesk.index import extract_bytes, is_indexable
+
+
+def tar_text_files(data: bytes, subdir: str = "") -> list[tuple[str, str]]:
+    """Return ``(relative_path, text)`` for indexable files in a repo tarball.
+
+    Repository archives wrap everything in one top-level ``<project>-<sha>/``
+    directory, which is stripped here.
+    """
+    files: list[tuple[str, str]] = []
+    prefix = Path(subdir) if subdir else None
+    with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
+        for member in tar.getmembers():
+            if not member.isfile():
+                continue
+            rel = Path(*Path(member.name).parts[1:])  # strip '<owner>-<repo>-<sha>/'
+            if not rel.parts:
+                continue
+            if prefix is not None and rel != prefix and prefix not in rel.parents:
+                continue
+            if not is_indexable(rel, member.size):
+                continue
+            extracted = tar.extractfile(member)
+            if extracted is None:
+                continue
+            raw = extracted.read()
+            text = extract_bytes(raw, str(rel))
+            if text is None or not text.strip():
+                continue
+            files.append((str(rel), text))
+    return files

ragdesk/buildenv.py

@@ -0,0 +1,94 @@
+"""Bake local ``.env`` values into a gitignored module for release builds.
+
+    python -m ragdesk.buildenv      # ./.env -> src/ragdesk/_build_env.py
+    uv build                        # the wheel now carries the values
+
+The generated module is gitignored — the repository never contains
+credentials, while distributed artifacts (PyPI wheel, bundled sidecars) can
+still offer one-click connects. Only non-confidential values belong in a
+public build (see SECURITY.md): GitHub client ID, Google Desktop client
+credentials. The Atlassian 3LO secret is a real credential and is refused
+here.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from ragdesk.envfile import parse_env_file
+
+ENV_TO_CONST = {
+    "RAGDESK_GITHUB_CLIENT_ID": "GITHUB_CLIENT_ID",
+    "RAGDESK_ATLASSIAN_CLIENT_ID": "ATLASSIAN_CLIENT_ID",
+    "RAGDESK_ATLASSIAN_CLIENT_SECRET": "ATLASSIAN_CLIENT_SECRET",
+    "GDRIVE_CLIENT_ID": "GOOGLE_CLIENT_ID",
+    "GDRIVE_CLIENT_SECRET": "GOOGLE_CLIENT_SECRET",
+    "RAGDESK_MS_CLIENT_ID": "MS_CLIENT_ID",
+}
+CONST_ORDER = (
+    "GITHUB_CLIENT_ID",
+    "ATLASSIAN_CLIENT_ID",
+    "ATLASSIAN_CLIENT_SECRET",
+    "GOOGLE_CLIENT_ID",
+    "GOOGLE_CLIENT_SECRET",
+    "MS_CLIENT_ID",
+)
+REFUSED = {"ATLASSIAN_CLIENT_SECRET"}
+
+
+def render_build_env(values: dict[str, str]) -> str:
+    lines = [
+        '"""Generated by `python -m ragdesk.buildenv` — do not edit, do not commit."""',
+        "",
+    ]
+    for const in CONST_ORDER:
+        lines.append(f"{const} = {values.get(const, '')!r}")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def write_build_env(env_path: Path, out_path: Path) -> dict[str, str]:
+    """Map ``.env`` values to constants; refuses confidential values."""
+    raw = parse_env_file(env_path.read_text()) if env_path.is_file() else {}
+    mapped = {const: raw.get(env_name, "") for env_name, const in ENV_TO_CONST.items()}
+    for const in REFUSED:
+        if mapped.get(const):
+            mapped[const] = ""
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(render_build_env(mapped))
+    return mapped
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m ragdesk.buildenv",
+        description="Bake .env values into src/ragdesk/_build_env.py for release builds.",
+    )
+    parser.add_argument("--env", type=Path, default=Path(".env"))
+    parser.add_argument("--out", type=Path, default=Path("src/ragdesk/_build_env.py"))
+    args = parser.parse_args(argv)
+
+    if not args.env.is_file():
+        print(f"no {args.env} found — nothing baked (fine for a repo-only build)")
+        return 0
+    mapped = write_build_env(args.env, args.out)
+    baked = [const for const, value in mapped.items() if value]
+    refused = [
+        const
+        for const in REFUSED
+        if parse_env_file(args.env.read_text()).get(
+            next(env for env, name in ENV_TO_CONST.items() if name == const), ""
+        )
+    ]
+    print(f"baked {args.out} from {args.env}: {', '.join(sorted(baked)) or 'no values'}")
+    if refused:
+        print(
+            f"refused to bake confidential values: {', '.join(sorted(refused))} "
+            "(Atlassian 3LO secret belongs to the user, see SECURITY.md)"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ragdesk/chunk.py

@@ -0,0 +1,93 @@
+"""Paragraph-aware chunking with character overlap and line tracking.
+
+Symbol-aware chunking was measured three ways and rejected on this corpus:
+per-symbol segmentation (fragmentation + BM25 term density), symbol headers in
+the chunk text (test files out-rank the implementation) and a symbol lane over
+definition sites (loose LIKE matching dilutes RRF). See AGENTS.md for the
+numbers; symbol intelligence belongs in a call-graph index, not in chunking.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# Lines that open a named block in the languages we index most.
+SYMBOL_RE = re.compile(
+    r"^\s*(?:async\s+)?(?:def|class|func|function|impl|struct|interface|enum|"
+    r"fn|pub\s+fn)\s+([A-Za-z_][\w]*)"
+)
+
+DEFAULT_MAX_CHARS = 1000
+DEFAULT_OVERLAP = 150
+
+
+@dataclass(frozen=True)
+class Chunk:
+    ordinal: int
+    text: str
+    line_start: int = 1
+
+
+def _line_number(text: str, offset: int) -> int:
+    return text.count("\n", 0, offset) + 1
+
+
+def chunk_text(
+    text: str,
+    max_chars: int = DEFAULT_MAX_CHARS,
+    overlap: int = DEFAULT_OVERLAP,
+) -> list[Chunk]:
+    """Split text into overlapping, paragraph-aware chunks.
+
+    Paragraphs are packed until ``max_chars``; oversized paragraphs are
+    hard-split. The overlap is taken from the tail of the previous chunk so
+    context survives boundaries. Each chunk remembers the source line it starts
+    on, which is what a citation's ``:line`` points at.
+    """
+    if overlap >= max_chars:
+        raise ValueError("overlap must be smaller than max_chars")
+
+    paragraphs: list[tuple[int, str]] = []
+    cursor = 0
+    for raw in text.split("\n\n"):
+        stripped = raw.strip()
+        if stripped:
+            offset = cursor + raw.index(stripped)
+            paragraphs.append((offset, stripped))
+        cursor += len(raw) + 2
+
+    pieces: list[tuple[int, str]] = []
+    for offset, para in paragraphs:
+        if len(para) <= max_chars:
+            pieces.append((offset, para))
+        else:
+            step = max_chars - overlap
+            pieces.extend((offset + i, para[i : i + max_chars]) for i in range(0, len(para), step))
+
+    chunks: list[tuple[int, str]] = []
+    buf = ""
+    buf_offset = 0
+    for offset, piece in pieces:
+        if not buf:
+            buf = piece
+            buf_offset = offset
+            continue
+        candidate = f"{buf}\n\n{piece}"
+        if len(candidate) <= max_chars:
+            buf = candidate
+            continue
+        chunks.append((buf_offset, buf))
+        tail = buf[-overlap:]
+        joined = f"{tail}\n\n{piece}"
+        if len(joined) <= max_chars:
+            buf = joined
+        else:
+            buf = piece
+            buf_offset = offset
+    if buf:
+        chunks.append((buf_offset, buf))
+    return [
+        Chunk(ordinal=i, text=chunk, line_start=_line_number(text, offset))
+        for i, (offset, chunk) in enumerate(chunks)
+    ]