docforge-cli 0.2.0__py3-none-any.whl

docforge/ranking.py

@@ -0,0 +1,20 @@
+"""Ranking helpers — pure Python mirror of the boost formula in search SQL."""
+
+from __future__ import annotations
+
+
+def compute_boosted_score(
+    similarity: float,
+    source_tags: list[str],
+    user_tags: list[str],
+    tag_weight: float,
+    org_weight: float,
+) -> float:
+    """Apply tag-overlap + org-tag boost to a similarity score.
+
+    Formula mirrors the SQL used in mcp_server.py and api.py search queries.
+    Kept in a pure function so the ranking math is unit-testable without SQL.
+    """
+    overlap = len(set(source_tags) & set(user_tags))
+    has_org = "org" in source_tags
+    return similarity * (1 + tag_weight * overlap + org_weight * (1 if has_org else 0))

docforge/scripts/__init__.py

@@ -0,0 +1 @@
+"""Operator scripts for docforge (run via `python -m docforge.scripts.<name>`)."""

docforge/scripts/eval_search.py

@@ -0,0 +1,226 @@
+"""Evaluate docforge retrieval quality against a ground-truth query set.
+
+Usage:
+    python -m docforge.scripts.eval_search \\
+      --api-url https://<fqdn> \\
+      --ground-truth rag/eval/ground_truth.yml \\
+      --user tobias.ens --team ccl --area cloud \\
+      --k 5
+
+Prints per-query detail + summary (recall@1, recall@k, MRR) to stdout. Exits 0
+on successful run regardless of retrieval quality — this tool measures, it does
+not gate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+import yaml
+
+
+@dataclass(frozen=True)
+class QueryResult:
+    query: str
+    expected_substring: str
+    returned_titles: list[str]
+    returned_scores: list[float]
+    match_rank: int | None  # 1-based; None if not in top-k
+
+
+def score_query(returned_titles: list[str], expected_substring: str) -> int | None:
+    """Return 1-based rank where expected_substring is contained in any title,
+    case-insensitively; or None if no match. Pure function."""
+    needle = expected_substring.lower()
+    for rank, title in enumerate(returned_titles, start=1):
+        if needle in title.lower():
+            return rank
+    return None
+
+
+def summarize(results: list[QueryResult], k: int) -> dict[str, float | int]:
+    """Return {queries, recall@1, recall@k, mrr}. Pure function."""
+    total = len(results)
+    if total == 0:
+        return {"queries": 0, "recall@1": 0.0, f"recall@{k}": 0.0, "mrr": 0.0}
+    hits_at_1 = sum(1 for r in results if r.match_rank == 1)
+    hits_at_k = sum(1 for r in results if r.match_rank is not None and r.match_rank <= k)
+    mrr = sum(1.0 / r.match_rank for r in results if r.match_rank is not None) / total
+    return {
+        "queries": total,
+        "recall@1": hits_at_1 / total,
+        f"recall@{k}": hits_at_k / total,
+        "mrr": mrr,
+    }
+
+
+async def run_queries(
+    api_url: str,
+    ground_truth: list[dict],
+    user_name: str,
+    team_name: str,
+    area_name: str | None,
+    k: int,
+    audience: str | None = None,
+) -> list[QueryResult]:
+    """POST each query to <api_url>/search via httpx; collect results. Sequential.
+
+    When audience is provided, attach an Entra Bearer token obtained via
+    DefaultAzureCredential. When not, send unauthenticated (auth.mode==none path)."""
+    results: list[QueryResult] = []
+    credential = None
+    if audience:
+        from azure.identity.aio import DefaultAzureCredential
+
+        credential = DefaultAzureCredential()
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            for entry in ground_truth:
+                q: str = entry["q"]
+                expected: str = entry["expected_title_contains"]
+                headers: dict[str, str] = {}
+                if credential is not None:
+                    token = await credential.get_token(f"{audience}/.default")
+                    headers["Authorization"] = f"Bearer {token.token}"
+                try:
+                    resp = await client.post(
+                        f"{api_url}/search",
+                        headers=headers,
+                        json={
+                            "query": q,
+                            "user_name": user_name,
+                            "team_name": team_name,
+                            "area_name": area_name,
+                            "limit": k,
+                        },
+                    )
+                    resp.raise_for_status()
+                    payload = resp.json()
+                    hits = payload.get("results", [])
+                except (httpx.HTTPError, ValueError) as e:
+                    print(f"  Query failed ({q!r}): {e}", file=sys.stderr)
+                    hits = []
+                titles = [h.get("source_title", "") for h in hits]
+                scores = [float(h.get("similarity", 0.0)) for h in hits]
+                results.append(
+                    QueryResult(
+                        query=q,
+                        expected_substring=expected,
+                        returned_titles=titles,
+                        returned_scores=scores,
+                        match_rank=score_query(titles, expected),
+                    )
+                )
+    finally:
+        if credential is not None:
+            await credential.close()
+    return results
+
+
+def format_report(results: list[QueryResult], summary: dict[str, float | int], k: int) -> str:
+    """Per-query detail + summary. Human-readable stdout."""
+    lines: list[str] = []
+    for r in results:
+        lines.append(f"Query: {r.query!r}")
+        lines.append(f"  Expected: contains {r.expected_substring!r}")
+        if r.returned_titles:
+            lines.append(f"  Top {len(r.returned_titles)}:")
+            for i, (title, score) in enumerate(
+                zip(r.returned_titles, r.returned_scores, strict=False), start=1
+            ):
+                marker = "  <-- MATCH" if r.match_rank == i else ""
+                lines.append(f"    {i}. [{score:.2f}] {title}{marker}")
+        else:
+            lines.append("  Top: (no results)")
+        if r.match_rank is not None and r.match_rank <= k:
+            lines.append(f"  recall@{k}: HIT  rank: {r.match_rank}")
+        else:
+            lines.append(f"  recall@{k}: MISS")
+        lines.append("")
+
+    lines.append("Summary:")
+    lines.append(f"  queries:               {summary['queries']}")
+    recall1 = summary["recall@1"]
+    recall_k = summary[f"recall@{k}"]
+    total = summary["queries"] or 1
+    r1_pct = f"{recall1 * 100:.0f}%"
+    rk_pct = f"{recall_k * 100:.0f}%"
+    lines.append(f"  recall@1:              {int(recall1 * total)}/{total} ({r1_pct})")
+    lines.append(f"  recall@{k}:              {int(recall_k * total)}/{total} ({rk_pct})")
+    lines.append(f"  mean reciprocal rank:  {summary['mrr']:.3f}")
+
+    misses = [r for r in results if r.match_rank is None or r.match_rank > k]
+    if misses:
+        lines.append("")
+        lines.append(f"  Missed (no match in top {k}):")
+        for r in misses:
+            lines.append(f"    - {r.query!r}  (expected {r.expected_substring!r})")
+
+    return "\n".join(lines)
+
+
+def _load_ground_truth(path: Path) -> list[dict]:
+    """Load, validate, and return the `queries` list from a ground-truth YAML file."""
+    if not path.is_file():
+        raise FileNotFoundError(f"Ground truth file not found: {path}")
+    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    queries = data.get("queries")
+    if not isinstance(queries, list) or not queries:
+        raise ValueError(f"{path}: missing or empty 'queries' list")
+    for i, q in enumerate(queries):
+        if "q" not in q or "expected_title_contains" not in q:
+            raise ValueError(f"{path}: entry {i} must have 'q' and 'expected_title_contains' keys")
+    return queries
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--api-url", required=True, help="Base URL of the search API (no trailing slash)"
+    )
+    parser.add_argument("--ground-truth", required=True, type=Path, help="Path to ground_truth.yml")
+    parser.add_argument("--user", required=True, help="Your identity — forwarded as user_name")
+    parser.add_argument("--team", required=True, help="Your team tag — forwarded as team_name")
+    parser.add_argument("--area", default=None, help="Optional area tag — forwarded as area_name")
+    parser.add_argument("--k", type=int, default=5, help="Top-k cutoff for recall@k")
+    parser.add_argument(
+        "--audience",
+        default=None,
+        help=(
+            "Entra API audience (e.g., api://<app-id>). When set, attaches a "
+            "Bearer token via DefaultAzureCredential. Omit for auth.mode=none."
+        ),
+    )
+    args = parser.parse_args()
+
+    try:
+        ground_truth = _load_ground_truth(args.ground_truth)
+    except (FileNotFoundError, ValueError) as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    results = asyncio.run(
+        run_queries(
+            api_url=args.api_url.rstrip("/"),
+            ground_truth=ground_truth,
+            user_name=args.user,
+            team_name=args.team,
+            area_name=args.area,
+            k=args.k,
+            audience=args.audience,
+        )
+    )
+    summary = summarize(results, args.k)
+    print(format_report(results, summary, args.k))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docforge/scripts/latency_report.py

@@ -0,0 +1,142 @@
+"""Compute P50 / P95 / P99 latency over recent query_log entries.
+
+Usage:
+    python -m docforge.scripts.latency_report --since '7 days' [--database-url ...]
+
+Reads DATABASE_URL from the environment (or --database-url flag) so it can
+run against prod with the admin connection string from Key Vault.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+import re
+import sys
+from dataclasses import dataclass
+
+import asyncpg
+
+# Only these interval shapes are accepted. The value is embedded as a SQL
+# literal (asyncpg can't bind str to $1::interval), so the regex is the
+# injection safety boundary — keep it strict.
+_SINCE_PATTERN = re.compile(
+    r"^\s*\d+\s+(seconds?|minutes?|hours?|days?|weeks?|months?)\s*$",
+    re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class LatencySummary:
+    n: int
+    p50_ms: float | None
+    p95_ms: float | None
+    p99_ms: float | None
+    earliest_request_ms_at: str | None  # ISO timestamp of first post-C4.3 row
+
+
+async def compute_summary(database_url: str, since: str) -> LatencySummary:
+    """Query query_log.request_ms within the given interval. Returns
+    percentiles + row count + the earliest-seen request_ms timestamp (the
+    effective C4.3 cutover date for this DB).
+
+    `since` must match _SINCE_PATTERN (N + unit). It is embedded as a SQL
+    literal because asyncpg rejects str for $1::interval; the regex
+    validation is the injection boundary."""
+    if not _SINCE_PATTERN.match(since):
+        raise ValueError(
+            f"Invalid --since {since!r}. Expected 'N unit' where unit is "
+            "seconds/minutes/hours/days/weeks/months (e.g. '7 days')."
+        )
+    conn = await asyncpg.connect(database_url)
+    try:
+        row = await conn.fetchrow(
+            f"""
+            SELECT
+                percentile_cont(0.50) WITHIN GROUP (ORDER BY request_ms) AS p50,
+                percentile_cont(0.95) WITHIN GROUP (ORDER BY request_ms) AS p95,
+                percentile_cont(0.99) WITHIN GROUP (ORDER BY request_ms) AS p99,
+                count(*)                                                 AS n
+              FROM query_log
+             WHERE request_ms IS NOT NULL
+               AND created_at > now() - interval '{since.strip()}'
+            """
+        )
+        earliest = await conn.fetchval(
+            "SELECT min(created_at) FROM query_log WHERE request_ms IS NOT NULL"
+        )
+        return LatencySummary(
+            n=int(row["n"]),
+            p50_ms=float(row["p50"]) if row["p50"] is not None else None,
+            p95_ms=float(row["p95"]) if row["p95"] is not None else None,
+            p99_ms=float(row["p99"]) if row["p99"] is not None else None,
+            earliest_request_ms_at=earliest.isoformat() if earliest is not None else None,
+        )
+    finally:
+        await conn.close()
+
+
+def format_summary(summary: LatencySummary, since: str) -> str:
+    """Human-readable stdout report."""
+    lines = [
+        f"Window:                 last {since}",
+        f"Queries with timing:    {summary.n}",
+    ]
+    if summary.n == 0:
+        lines.append(
+            "No rows with request_ms in the window — has the C4.3 migration been applied "
+            "and the /search handler redeployed?"
+        )
+        return "\n".join(lines)
+    lines.extend(
+        [
+            f"P50:                    {summary.p50_ms:.0f} ms",
+            f"P95:                    {summary.p95_ms:.0f} ms",
+            f"P99:                    {summary.p99_ms:.0f} ms",
+        ]
+    )
+    if summary.earliest_request_ms_at is not None:
+        lines.append(f"request_ms cutover at:  {summary.earliest_request_ms_at}")
+    lines.append("")
+    lines.append("Note: the earliest ~1-2 rows after each revision deployment include")
+    lines.append("the 15-30 s embedding-model warm-up cost; this is kept in the data as")
+    lines.append("honest signal. P95 therefore reflects warm-up+steady-state.")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--since",
+        default="7 days",
+        help="Postgres interval string (e.g., '7 days', '24 hours'). Default: 7 days.",
+    )
+    parser.add_argument(
+        "--database-url",
+        default=None,
+        help="Postgres URL. Falls back to DATABASE_URL env var.",
+    )
+    args = parser.parse_args()
+
+    db_url = args.database_url or os.environ.get("DATABASE_URL")
+    if not db_url:
+        print("Error: DATABASE_URL not set (and --database-url not provided)", file=sys.stderr)
+        return 1
+
+    try:
+        summary = asyncio.run(compute_summary(db_url, args.since))
+    except ValueError as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 2
+    except (OSError, asyncpg.PostgresError) as e:
+        print(f"Error connecting to the database: {e}", file=sys.stderr)
+        return 1
+    print(format_summary(summary, args.since))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docforge/sources.py

@@ -0,0 +1,46 @@
+"""Source configuration — pydantic models + YAML loader.
+
+Each entry in `sources.yml` is a ConfluenceSourceConfig or a
+GitRepoSourceConfig, discriminated by the `type` field.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Literal
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class ConfluenceSourceConfig(BaseModel):
+    type: Literal["confluence_page"]
+    page_id: str
+    space_key: str
+    title: str
+    tags: list[str] = []
+
+
+class GitRepoSourceConfig(BaseModel):
+    type: Literal["git_repo"]
+    repo_path: str
+    include_patterns: list[str] = ["README.md", "CLAUDE.md", "docs/**/*.md"]
+    title: str
+    tags: list[str] = []
+
+
+SourceConfig = Annotated[
+    ConfluenceSourceConfig | GitRepoSourceConfig,
+    Field(discriminator="type"),
+]
+
+
+class SourcesFile(BaseModel):
+    sources: list[SourceConfig]
+
+
+def load_sources(path: str | Path) -> list[SourceConfig]:
+    """Load source configurations from a YAML file."""
+    with open(path) as f:
+        data = yaml.safe_load(f)
+    return SourcesFile.model_validate(data).sources

docforge/sql/migrations/001_add_source_identifier.sql

@@ -0,0 +1,3 @@
+ALTER TABLE sources ADD COLUMN IF NOT EXISTS source_identifier TEXT;
+CREATE UNIQUE INDEX IF NOT EXISTS sources_source_identifier_unique
+    ON sources (source_identifier) WHERE source_identifier IS NOT NULL;

docforge/sql/migrations/002_add_status_index.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS sources_status_idx ON sources (status);

docforge/sql/migrations/003_add_source_tags.sql

@@ -0,0 +1,4 @@
+ALTER TABLE sources
+    ADD COLUMN IF NOT EXISTS tags TEXT[] NOT NULL DEFAULT '{}';
+
+CREATE INDEX IF NOT EXISTS sources_tags_idx ON sources USING gin (tags);

docforge/sql/migrations/004_add_query_log.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS query_log (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    user_name TEXT NOT NULL,
+    team_name TEXT NOT NULL,
+    area_name TEXT,
+    query TEXT NOT NULL,
+    result_count INT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS query_log_created_at_idx ON query_log (created_at);

docforge/sql/migrations/005_add_query_log_user_oid.sql

@@ -0,0 +1,2 @@
+ALTER TABLE query_log ADD COLUMN IF NOT EXISTS user_oid TEXT;
+CREATE INDEX IF NOT EXISTS query_log_user_oid_idx ON query_log (user_oid);

docforge/sql/migrations/006_add_query_log_request_ms.sql

@@ -0,0 +1 @@
+ALTER TABLE query_log ADD COLUMN IF NOT EXISTS request_ms INT;

docforge/sql/schema.sql

@@ -0,0 +1,29 @@
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE IF NOT EXISTS sources (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    type TEXT NOT NULL,
+    url TEXT NOT NULL,
+    title TEXT NOT NULL,
+    confluence_page_id TEXT,
+    confluence_space_key TEXT,
+    source_identifier TEXT,
+    last_crawled_at TIMESTAMPTZ,
+    content_hash TEXT,
+    status TEXT DEFAULT 'pending',
+    created_at TIMESTAMPTZ DEFAULT now(),
+    CONSTRAINT sources_confluence_page_id_unique UNIQUE (confluence_page_id)
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
+    chunk_index INT NOT NULL,
+    text TEXT NOT NULL,
+    embedding vector(768),
+    section_title TEXT,
+    created_at TIMESTAMPTZ DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS chunks_embedding_idx ON chunks
+    USING hnsw (embedding vector_cosine_ops);

docforge/templates/docforge.yml

@@ -0,0 +1,11 @@
+# docforge.yml — main configuration
+# Secrets (API tokens, passwords) go in .env, not here.
+
+database_url: postgresql://docforge:localdev@localhost:5432/docforge
+
+embedding:
+  model: google/embeddinggemma-300m
+  dimensions: 768
+  chunk_max_tokens: 500
+
+sources_file: sources.yml

docforge/templates/docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  db:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: docforge
+      POSTGRES_USER: docforge
+      POSTGRES_PASSWORD: localdev
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+
+volumes:
+  pgdata:

docforge/templates/mcp_client.py

@@ -0,0 +1,83 @@
+"""Lightweight MCP client for docforge.
+
+Calls a hosted search API over HTTP. No local database or model needed.
+
+Usage:
+    pip install httpx fastmcp
+    claude mcp add -s user docforge -- python mcp_client.py
+
+Environment:
+    DOCFORGE_API_URL: Base URL of the search API
+"""
+
+from __future__ import annotations
+
+import os
+
+import httpx
+from fastmcp import FastMCP
+
+API_URL = os.environ.get("DOCFORGE_API_URL", "http://localhost:8000")
+
+mcp = FastMCP(
+    "docforge",
+    instructions=(
+        "Search across your team's indexed documentation including architecture, "
+        "coding guidelines, and cross-team interfaces. "
+        "Use the search_documentation tool when you need information about "
+        "other teams, shared practices, or organizational knowledge."
+    ),
+)
+
+
+@mcp.tool()
+async def search_documentation(query: str, limit: int = 5) -> str:
+    """Search across indexed documentation from Confluence pages and git repos.
+
+    Args:
+        query: Natural language search query.
+        limit: Maximum number of results to return (default 5).
+    """
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        resp = await client.post(
+            f"{API_URL}/search",
+            json={"query": query, "limit": limit},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+    if not data["results"]:
+        return "No documentation found matching your query."
+
+    parts: list[str] = []
+    for i, result in enumerate(data["results"], 1):
+        header = f"**Result {i}** (relevance: {result['similarity']:.2f})"
+        header += f" -- {result['source_title']}"
+        if result.get("section_title"):
+            header += f" > {result['section_title']}"
+        header += f"\nSource: {result['source_url']}"
+        parts.append(f"{header}\n\n{result['text']}")
+
+    return "\n\n---\n\n".join(parts)
+
+
+@mcp.tool()
+async def list_sources() -> str:
+    """List all documentation sources currently indexed."""
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        resp = await client.get(f"{API_URL}/sources")
+        resp.raise_for_status()
+        data = resp.json()
+
+    if not data["sources"]:
+        return "No sources indexed."
+
+    lines = [f"**{data['count']} indexed sources:**\n"]
+    for src in data["sources"]:
+        lines.append(f"- **{src['title']}** ({src['chunk_count']} chunks, {src['status']})")
+
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    mcp.run()

docforge/templates/sources.yml

@@ -0,0 +1,21 @@
+# sources.yml — documentation sources to index
+# Uncomment and edit the examples below.
+
+sources: []
+
+  # Confluence pages (need CONFLUENCE_* vars in .env):
+  # - type: confluence_page
+  #   page_id: "12345"
+  #   space_key: MYSPACE
+  #   title: "My Team's Documentation"
+  #
+  # - type: confluence_page
+  #   page_id: "67890"
+  #   space_key: MYSPACE
+  #   title: "Architecture Guidelines"
+
+  # Local git repos (no auth needed):
+  # - type: git_repo
+  #   repo_path: "/path/to/my-repo"
+  #   include_patterns: ["README.md", "CLAUDE.md", "docs/**/*.md"]
+  #   title: "My Repo"