docforge-cli 0.2.0__py3-none-any.whl

docforge/lint.py

@@ -0,0 +1,92 @@
+"""Lint a repo's README + CLAUDE.md + docs/ for Spec B banned-content patterns.
+
+Pure logic — see `docforge.cli.lint_docs` for the user entry point. Banned-content
+only in v1; structural/required-topics linting is deferred (see Spec C2 follow-ups).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from docforge.crawlers.git import crawl_repo
+
+
+@dataclass(frozen=True)
+class LintFinding:
+    file: str
+    line: int
+    rule: str
+    message: str
+
+
+@dataclass(frozen=True)
+class LintReport:
+    scanned: list[str] = field(default_factory=list)
+    findings: list[LintFinding] = field(default_factory=list)
+
+
+BANNED_RULES: list[tuple[str, str, str]] = [
+    (
+        "todo-placeholder",
+        r"TODO.*(Explain|Contribute)",
+        "Placeholder TODO — delete and write real content",
+    ),
+    (
+        "readme-inspiration-link",
+        r"create-a-readme",
+        "Microsoft README inspirational link — delete",
+    ),
+    (
+        "readme-boilerplate",
+        r"(github\.com/aspnet/Home|Microsoft/vscode|Microsoft/ChakraCore)",
+        "Azure DevOps default boilerplate — delete whole block",
+    ),
+    (
+        "lastpass-reference",
+        r"LastPass",
+        "Credential-source reference — move to Teams channel, not indexed docs",
+    ),
+]
+
+_COMPILED_BANNED_RULES = [(name, re.compile(pat), msg) for name, pat, msg in BANNED_RULES]
+
+
+def lint_repo(repo_root: Path) -> LintReport:
+    """Walk the repo's doc surface and scan for banned-content patterns. Read-only."""
+    files = crawl_repo(str(repo_root))
+    scanned = [f.file_path.replace("\\", "/") for f in files]
+    findings: list[LintFinding] = []
+    for f in files:
+        rel = f.file_path.replace("\\", "/")
+        for lineno, line in enumerate(f.content.splitlines(), start=1):
+            for rule_name, pattern, message in _COMPILED_BANNED_RULES:
+                if pattern.search(line):
+                    findings.append(LintFinding(rel, lineno, rule_name, message))
+    return LintReport(scanned=scanned, findings=findings)
+
+
+def format_report(report: LintReport, repo_root: Path) -> str:
+    """Human-readable stdout, grouped by file with summary line."""
+    lines: list[str] = []
+    header = "FAIL" if report.findings else "PASS"
+    lines.append(f"{repo_root} — {header}")
+    lines.append(f"  {len(report.scanned)} files scanned")
+    if not report.findings:
+        lines.append("  No banned content")
+        return "\n".join(lines)
+
+    lines.append("")
+    lines.append("  Banned content:")
+    by_file: dict[str, list[LintFinding]] = {}
+    for f in report.findings:
+        by_file.setdefault(f.file, []).append(f)
+    for file_rel, file_findings in by_file.items():
+        for f in file_findings:
+            location = f"{file_rel}:{f.line}"
+            lines.append(f"    FAIL  {location:<30}  {f.rule:<22}  {f.message}")
+
+    lines.append("")
+    lines.append(f"  Summary: {len(report.findings)} banned-content hits")
+    return "\n".join(lines)

docforge/mcp_server.py

@@ -0,0 +1,188 @@
+"""MCP server exposing documentation search to AI coding assistants.
+
+Run with: python -m docforge.mcp_server
+"""
+
+from __future__ import annotations
+
+import logging
+
+import numpy as np
+from fastmcp import FastMCP
+
+from docforge.config import Settings
+from docforge.db import get_pool
+from docforge.processors.embedder import Embedder
+
+logger = logging.getLogger(__name__)
+
+mcp = FastMCP(
+    "knowledge-hub",
+    instructions=(
+        "Search across your team's indexed documentation including team responsibilities, "
+        "coding guidelines, architecture standards, and cross-team interfaces. "
+        "Use the search_documentation tool when you need information about other teams, "
+        "shared coding practices, or organizational knowledge."
+    ),
+)
+
+# Initialized lazily on first search
+_embedder: Embedder | None = None
+_settings: Settings | None = None
+
+
+def _get_settings() -> Settings:
+    global _settings
+    if _settings is None:
+        _settings = Settings()
+    return _settings
+
+
+def _get_embedder() -> Embedder:
+    global _embedder
+    if _embedder is None:
+        settings = _get_settings()
+        logger.info("Loading embedding model (this may take a few seconds)...")
+        _embedder = Embedder(
+            settings.embedding_model, hf_token=settings.hf_token.get_secret_value()
+        )
+    return _embedder
+
+
+@mcp.tool()
+async def search_documentation(
+    query: str,
+    user_name: str,
+    team_name: str,
+    area_name: str | None = None,
+    limit: int = 5,
+) -> str:
+    """Search across indexed documentation from Confluence pages and git repos.
+
+    Returns relevant documentation chunks with source attribution. Use this to find
+    information about team ownership, coding guidelines, architecture decisions,
+    and cross-team interfaces.
+
+    Args:
+        query: Natural language search query.
+        user_name: Your name (e.g., "tobias.ens"). Used for usage telemetry.
+        team_name: Your team tag (e.g., "ccl"). Boosts team-tagged docs.
+        area_name: Your area tag (e.g., "cloud"). Optional; boosts area-tagged docs.
+        limit: Maximum number of results to return (default 5).
+    """
+    settings = _get_settings()
+    embedder = _get_embedder()
+
+    query_vector = embedder.embed_query(query)
+    user_tags = [team_name] + ([area_name] if area_name else [])
+
+    pool = await get_pool(settings.database_url)
+    async with pool.acquire() as conn:
+        rows = await conn.fetch(
+            """
+            SELECT
+                c.text,
+                c.section_title,
+                s.title AS source_title,
+                s.url AS source_url,
+                s.tags AS source_tags,
+                1 - (c.embedding <=> $1::vector) AS similarity,
+                (1 - (c.embedding <=> $1::vector)) *
+                    (1
+                     + $2::float * cardinality(
+                         ARRAY(SELECT unnest(s.tags) INTERSECT SELECT unnest($3::text[]))
+                       )
+                     + $4::float * (CASE WHEN 'org' = ANY(s.tags) THEN 1 ELSE 0 END)
+                    ) AS boosted_score
+            FROM chunks c
+            JOIN sources s ON c.source_id = s.id
+            WHERE s.status = 'active'
+            ORDER BY boosted_score DESC
+            LIMIT $5
+            """,
+            np.array(query_vector, dtype=np.float32),
+            settings.tag_match_weight,
+            user_tags,
+            settings.org_tag_weight,
+            limit,
+        )
+
+    from docforge.query_log import log_query
+
+    await log_query(pool, user_name, team_name, area_name, query, len(rows))
+
+    if not rows:
+        return (
+            "No documentation found matching your query. "
+            "The index may be empty -- run `python -m docforge ingest` to populate it."
+        )
+
+    parts: list[str] = []
+    for i, row in enumerate(rows, 1):
+        similarity = row["similarity"]
+        source = row["source_title"]
+        url = row["source_url"]
+        section = row["section_title"]
+        text = row["text"]
+        tags = list(row["source_tags"] or [])
+
+        header = f"**Result {i}** (relevance: {similarity:.2f}) — {source}"
+        if section:
+            header += f" > {section}"
+        header += f"\nSource: {url}"
+        if tags:
+            header += f"\nTags: {', '.join(tags)}"
+
+        parts.append(f"{header}\n\n{text}")
+
+    return "\n\n---\n\n".join(parts)
+
+
+@mcp.tool()
+async def list_sources() -> str:
+    """List all documentation sources currently indexed in the knowledge hub.
+
+    Returns the title, URL, status, and last crawl time for each source.
+    Use this to see what documentation is available for searching.
+    """
+    settings = _get_settings()
+    pool = await get_pool(settings.database_url)
+
+    async with pool.acquire() as conn:
+        rows = await conn.fetch(
+            """
+            SELECT title, url, status, last_crawled_at,
+                   (SELECT count(*) FROM chunks WHERE source_id = s.id) AS chunk_count
+            FROM sources s
+            ORDER BY title
+            """
+        )
+
+    if not rows:
+        return "No sources indexed yet. Run `python -m docforge ingest` to populate."
+
+    lines: list[str] = []
+    for row in rows:
+        last = row["last_crawled_at"]
+        crawled = last.strftime("%Y-%m-%d %H:%M") if last else "never"
+        lines.append(
+            f"- **{row['title']}** ({row['chunk_count']} chunks, {row['status']})\n"
+            f"  Last crawled: {crawled}\n"
+            f"  {row['url']}"
+        )
+
+    return f"**{len(rows)} indexed sources:**\n\n" + "\n\n".join(lines)
+
+
+def main() -> None:
+    """Configure logging and start the FastMCP server on stdio transport."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)-8s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

docforge/processors/__init__.py

@@ -0,0 +1 @@
+"""Text processors — HTML parser, token-aware chunker, embedder."""

docforge/processors/chunker.py

@@ -0,0 +1,141 @@
+"""Token-aware chunker — splits sections into chunks under a token limit."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from docforge.processors.parser import Section
+
+
+@dataclass
+class Chunk:
+    text: str
+    section_title: str
+    chunk_index: int
+
+
+def chunk_sections(
+    sections: list[Section],
+    max_tokens: int = 500,
+    tokenizer_fn: callable | None = None,
+) -> list[Chunk]:
+    """Split sections into chunks of roughly max_tokens size.
+
+    Splits on section boundaries first, then on paragraph boundaries
+    if a section exceeds max_tokens.
+
+    Args:
+        sections: Parsed sections from the document.
+        max_tokens: Maximum tokens per chunk.
+        tokenizer_fn: Function that counts tokens in a string.
+                      If None, uses a simple word-count approximation
+                      (1 token ~ 0.75 words).
+    """
+    if tokenizer_fn is None:
+        tokenizer_fn = _approximate_token_count
+
+    chunks: list[Chunk] = []
+    chunk_index = 0
+
+    for section in sections:
+        text = section.text.strip()
+        if not text:
+            continue
+
+        # Add section title as context prefix
+        if section.title:
+            text = f"{section.title}\n\n{text}"
+
+        if tokenizer_fn(text) <= max_tokens:
+            chunks.append(Chunk(text=text, section_title=section.title, chunk_index=chunk_index))
+            chunk_index += 1
+        else:
+            # Split on paragraph boundaries
+            sub_chunks = _split_by_paragraphs(text, max_tokens, tokenizer_fn)
+            for sub_text in sub_chunks:
+                chunks.append(
+                    Chunk(text=sub_text, section_title=section.title, chunk_index=chunk_index)
+                )
+                chunk_index += 1
+
+    return chunks
+
+
+def _split_by_paragraphs(text: str, max_tokens: int, tokenizer_fn: callable) -> list[str]:
+    """Split text into chunks by paragraph boundaries."""
+    paragraphs = text.split("\n")
+    result: list[str] = []
+    current_parts: list[str] = []
+    current_tokens = 0
+
+    for paragraph in paragraphs:
+        paragraph = paragraph.strip()
+        if not paragraph:
+            continue
+
+        para_tokens = tokenizer_fn(paragraph)
+
+        if para_tokens > max_tokens:
+            # Flush current buffer
+            if current_parts:
+                result.append("\n".join(current_parts))
+                current_parts = []
+                current_tokens = 0
+
+            # Split long paragraph by sentences
+            for sentence_chunk in _split_long_text(paragraph, max_tokens, tokenizer_fn):
+                result.append(sentence_chunk)
+            continue
+
+        if current_tokens + para_tokens > max_tokens and current_parts:
+            result.append("\n".join(current_parts))
+            current_parts = []
+            current_tokens = 0
+
+        current_parts.append(paragraph)
+        current_tokens += para_tokens
+
+    if current_parts:
+        result.append("\n".join(current_parts))
+
+    return result
+
+
+def _split_long_text(text: str, max_tokens: int, tokenizer_fn: callable) -> list[str]:
+    """Split text that exceeds max_tokens by sentence boundaries, falling back to words."""
+    # Try splitting by sentences (period followed by space)
+    sentences = text.replace(". ", ".\n").split("\n")
+
+    result: list[str] = []
+    current_parts: list[str] = []
+    current_tokens = 0
+
+    for sentence in sentences:
+        sentence = sentence.strip()
+        if not sentence:
+            continue
+
+        sent_tokens = tokenizer_fn(sentence)
+
+        if current_tokens + sent_tokens > max_tokens and current_parts:
+            result.append(" ".join(current_parts))
+            current_parts = []
+            current_tokens = 0
+
+        current_parts.append(sentence)
+        current_tokens += sent_tokens
+
+    if current_parts:
+        result.append(" ".join(current_parts))
+
+    return result
+
+
+def _approximate_token_count(text: str) -> int:
+    """Approximate token count using word count.
+
+    Roughly 1 token ~ 0.75 words for English text.
+    This is used as fallback when no model tokenizer is available.
+    """
+    words = len(text.split())
+    return int(words / 0.75)

docforge/processors/embedder.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import logging
+import os
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+
+class Embedder:
+    """Generates text embeddings using a sentence-transformers model.
+
+    Loads the model once at initialization and reuses it for all calls.
+    Default model is EmbeddingGemma-300M (768 dimensions).
+    Falls back to all-MiniLM-L6-v2 (384 dimensions) if the primary model fails to load.
+    """
+
+    def __init__(self, model_name: str = "google/embeddinggemma-300m", hf_token: str = "") -> None:
+        from sentence_transformers import SentenceTransformer
+
+        # Use provided token, fall back to environment variable
+        if not hf_token:
+            hf_token = os.environ.get("HF_TOKEN", "")
+
+        try:
+            logger.info("Loading embedding model: %s", model_name)
+            self._model = SentenceTransformer(model_name, token=hf_token or None)
+            self.model_name = model_name
+            self.dimensions = self._model.get_embedding_dimension()
+            logger.info("Model loaded: %s (%d dimensions)", self.model_name, self.dimensions)
+        except Exception:
+            fallback = "sentence-transformers/all-MiniLM-L6-v2"
+            logger.warning(
+                "Failed to load %s, falling back to %s",
+                model_name,
+                fallback,
+                exc_info=True,
+            )
+            try:
+                self._model = SentenceTransformer(fallback)
+                self.model_name = fallback
+                self.dimensions = self._model.get_embedding_dimension()
+                logger.info(
+                    "Fallback model loaded: %s (%d dimensions)",
+                    self.model_name,
+                    self.dimensions,
+                )
+            except Exception:
+                logger.error("Failed to load fallback model %s", fallback, exc_info=True)
+                raise RuntimeError(
+                    f"No embedding model available. "
+                    f"Primary ({model_name}) and fallback ({fallback}) both failed."
+                )
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Generate embeddings for a list of texts.
+
+        Returns a list of float vectors, one per input text.
+        """
+        if not texts:
+            return []
+
+        embeddings = self._model.encode(texts, show_progress_bar=False, normalize_embeddings=True)
+        return embeddings.tolist()
+
+    def embed_query(self, query: str) -> list[float]:
+        """Generate embedding for a single search query."""
+        result = self.embed([query])
+        return result[0]
+
+    def get_tokenizer_fn(self) -> Callable[[str], int]:
+        """Return a token-counting function using this model's tokenizer."""
+        tokenizer = self._model.tokenizer
+
+        def count_tokens(text: str) -> int:
+            return len(tokenizer.encode(text, add_special_tokens=False))
+
+        return count_tokens

docforge/processors/parser.py

@@ -0,0 +1,143 @@
+"""Confluence storage-format HTML parser — yields Section objects."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from bs4 import BeautifulSoup, Tag
+
+
+@dataclass
+class Section:
+    title: str
+    text: str
+    level: int = 0
+
+
+def parse_confluence_html(html: str) -> list[Section]:
+    """Parse Confluence storage-format HTML into a list of text sections.
+
+    Handles Confluence-specific elements:
+    - Headings become section boundaries
+    - Tables are converted to readable text
+    - Custom macros (smartlinks, status, emoji) are handled
+    - Empty sections are dropped
+    """
+    soup = BeautifulSoup(html, "html.parser")
+
+    _clean_confluence_macros(soup)
+
+    sections: list[Section] = []
+    current_title = ""
+    current_level = 0
+    current_parts: list[str] = []
+
+    for element in soup.children:
+        if not isinstance(element, Tag):
+            text = element.get_text(strip=True)
+            if text:
+                current_parts.append(text)
+            continue
+
+        if element.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            # Flush previous section
+            if current_parts:
+                combined = "\n".join(current_parts).strip()
+                if combined:
+                    sections.append(
+                        Section(title=current_title, text=combined, level=current_level)
+                    )
+                current_parts = []
+
+            current_title = element.get_text(strip=True)
+            current_level = int(element.name[1])
+
+        elif element.name == "table":
+            current_parts.append(_table_to_text(element))
+
+        else:
+            text = element.get_text(separator=" ", strip=True)
+            if text:
+                current_parts.append(text)
+
+    # Flush last section
+    if current_parts:
+        combined = "\n".join(current_parts).strip()
+        if combined:
+            sections.append(Section(title=current_title, text=combined, level=current_level))
+
+    return sections
+
+
+def _clean_confluence_macros(soup: BeautifulSoup) -> None:
+    """Process Confluence custom elements in-place."""
+    for custom in soup.find_all("custom"):
+        data_type = custom.get("data-type", "")
+
+        if data_type == "smartlink":
+            # Replace smart links with their URL
+            href = custom.get_text(strip=True)
+            if href.startswith("http"):
+                custom.replace_with(href)
+            else:
+                custom.replace_with(custom.get_text(strip=True))
+
+        elif data_type == "emoji":
+            # Strip emojis
+            custom.decompose()
+
+        elif data_type == "status":
+            # Convert status badges to text
+            status_text = custom.get_text(strip=True)
+            custom.replace_with(f"[{status_text}]")
+
+        else:
+            # Unknown custom element — keep as text
+            custom.replace_with(custom.get_text(strip=True))
+
+    # Also handle ac:structured-macro, ac:rich-text-body etc. (Confluence Server format)
+    for macro in soup.find_all("ac:structured-macro"):
+        body = macro.find("ac:rich-text-body")
+        if body:
+            macro.replace_with(body.get_text(separator=" ", strip=True))
+        else:
+            macro.decompose()
+
+
+def _table_to_text(table: Tag) -> str:
+    """Convert an HTML table to readable plain text.
+
+    For tables with headers, produces "header: value" pairs per row.
+    For tables without headers, produces pipe-separated rows.
+    """
+    rows = table.find_all("tr")
+    if not rows:
+        return ""
+
+    # Extract headers from first row
+    headers: list[str] = []
+    first_row = rows[0]
+    header_cells = first_row.find_all(["th"])
+    if header_cells:
+        headers = [cell.get_text(separator=" ", strip=True) for cell in header_cells]
+        data_rows = rows[1:]
+    else:
+        data_rows = rows
+
+    lines: list[str] = []
+
+    for row in data_rows:
+        cells = row.find_all(["td", "th"])
+        values = [cell.get_text(separator=" ", strip=True) for cell in cells]
+
+        if headers and len(values) == len(headers):
+            # Format as "header: value" pairs, skip empty values
+            pairs = [f"{h}: {v}" for h, v in zip(headers, values) if v]
+            if pairs:
+                lines.append(" | ".join(pairs))
+        elif values:
+            non_empty = [v for v in values if v]
+            if non_empty:
+                lines.append(" | ".join(non_empty))
+
+    return "\n".join(lines)

docforge/query_log.py

@@ -0,0 +1,45 @@
+"""Async helper for inserting rows into query_log.
+
+Failures are logged and swallowed — query logging must never break a search.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import asyncpg
+
+logger = logging.getLogger(__name__)
+
+
+async def log_query(
+    pool: asyncpg.Pool,
+    user_name: str,
+    team_name: str,
+    area_name: str | None,
+    query: str,
+    result_count: int,
+    user_oid: str | None = None,
+    request_ms: int | None = None,
+) -> None:
+    """Record a search request. user_oid is the Entra object ID (post-auth)
+    or None (pre-auth rows). request_ms is the handler's wall-clock time in
+    milliseconds (post-C4.3) or None (pre-C4.3 rows). Never raises."""
+    try:
+        async with pool.acquire() as conn:
+            await conn.execute(
+                """
+                INSERT INTO query_log
+                    (user_name, team_name, area_name, query, result_count, user_oid, request_ms)
+                VALUES ($1, $2, $3, $4, $5, $6, $7)
+                """,
+                user_name,
+                team_name,
+                area_name,
+                query,
+                result_count,
+                user_oid,
+                request_ms,
+            )
+    except Exception as e:
+        logger.warning("query_log insert failed: %s", e)