docforge-cli 0.2.0__py3-none-any.whl

docforge/__main__.py

@@ -0,0 +1,5 @@
+"""Module entrypoint — `python -m docforge` dispatches to the Typer app."""
+
+from docforge.cli import app
+
+app()

docforge/api.py

@@ -0,0 +1,266 @@
+"""FastAPI search API for docforge.
+
+Runs on Azure Container Apps. Loads embedding model at startup,
+serves search queries over HTTP.
+
+Run locally: uvicorn docforge.api:app --reload
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from contextlib import asynccontextmanager
+from typing import Any
+
+import numpy as np
+from fastapi import Depends, FastAPI, HTTPException, Request
+from fastapi.security import SecurityScopes
+from pydantic import BaseModel
+
+from docforge.config import Settings
+from docforge.db import close_pool, get_pool
+from docforge.processors.embedder import Embedder
+
+logger = logging.getLogger(__name__)
+
+_embedder: Embedder | None = None
+_settings: Settings | None = None
+_azure_scheme = None  # Populated in lifespan when auth.mode == "entra"
+_cleanup_task: asyncio.Task | None = None
+
+_CLEANUP_INTERVAL_SECONDS = 3600  # one hour — overridable in tests
+
+
+async def _query_log_cleanup_loop(database_url: str, retention_days: int) -> None:
+    """Deletes query_log rows older than retention_days every
+    _CLEANUP_INTERVAL_SECONDS. Idempotent, so multi-replica is safe."""
+    # int() coercion makes the f-string SQL below injection-safe. asyncpg's
+    # $1::interval parameter binding doesn't accept str, hence the literal.
+    days = int(retention_days)
+    while True:
+        try:
+            pool = await get_pool(database_url)
+            async with pool.acquire() as conn:
+                result = await conn.execute(
+                    f"DELETE FROM query_log WHERE created_at < now() - interval '{days} days'"
+                )
+            logger.info("query_log cleanup: %s", result)
+        except Exception as e:
+            logger.exception("query_log cleanup failed: %s", e)
+        await asyncio.sleep(_CLEANUP_INTERVAL_SECONDS)
+
+
+def _get_settings() -> Settings:
+    global _settings
+    if _settings is None:
+        _settings = Settings()
+    return _settings
+
+
+def _build_auth_scheme(settings: Settings):
+    """Return a SingleTenantAzureAuthorizationCodeBearer if mode==entra, else None."""
+    if settings.auth.mode != "entra":
+        return None
+    from fastapi_azure_auth import SingleTenantAzureAuthorizationCodeBearer
+
+    app_client_id = settings.auth.audience.removeprefix("api://")
+    return SingleTenantAzureAuthorizationCodeBearer(
+        app_client_id=app_client_id,
+        tenant_id=settings.auth.tenant_id,
+        scopes={f"{settings.auth.audience}/search": "Search docforge"},
+    )
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Load the embedding model at startup; close the DB pool on shutdown."""
+    global _embedder, _azure_scheme, _cleanup_task
+    settings = _get_settings()
+    _azure_scheme = _build_auth_scheme(settings)
+    if _azure_scheme is not None:
+        await _azure_scheme.openid_config.load_config()
+        logger.info(
+            "Entra auth enabled (tenant=%s, audience=%s)",
+            settings.auth.tenant_id,
+            settings.auth.audience,
+        )
+    logger.info("Loading embedding model...")
+    _embedder = Embedder(settings.embedding_model, hf_token=settings.hf_token.get_secret_value())
+    logger.info("Model loaded: %s (%dd)", _embedder.model_name, _embedder.dimensions)
+
+    _cleanup_task = asyncio.create_task(
+        _query_log_cleanup_loop(settings.database_url, settings.query_log_retention_days)
+    )
+
+    yield
+
+    if _cleanup_task is not None:
+        _cleanup_task.cancel()
+        try:
+            await _cleanup_task
+        except asyncio.CancelledError:
+            pass
+    await close_pool()
+
+
+app = FastAPI(title="docforge", lifespan=lifespan)
+
+
+async def _auth_dependency(request: Request):
+    """Return the authenticated User under auth.mode=entra, None otherwise."""
+    if _azure_scheme is None:
+        return None
+    # Empty SecurityScopes: we don't enforce scope-level authorization beyond
+    # the token validation the scheme itself does. Without this arg the call
+    # signature mismatches what fastapi-azure-auth expects.
+    return await _azure_scheme(request, SecurityScopes())
+
+
+class SearchRequest(BaseModel):
+    query: str
+    user_name: str
+    team_name: str
+    area_name: str | None = None
+    limit: int = 5
+
+
+class SearchResult(BaseModel):
+    text: str
+    section_title: str | None
+    source_title: str
+    source_url: str
+    source_tags: list[str]
+    similarity: float
+
+
+class SearchResponse(BaseModel):
+    results: list[SearchResult]
+    query: str
+    count: int
+
+
+@app.get("/health")
+async def health() -> dict[str, Any]:
+    """Health check endpoint."""
+    return {
+        "status": "ok",
+        "model": _embedder.model_name if _embedder else "not loaded",
+    }
+
+
+@app.post("/search", response_model=SearchResponse)
+async def search(req: SearchRequest, user=Depends(_auth_dependency)) -> SearchResponse:
+    """Search indexed documentation by semantic similarity."""
+    start = time.perf_counter()
+    if not _embedder:
+        raise HTTPException(status_code=503, detail="Embedding model not loaded yet")
+
+    try:
+        query_vector = _embedder.embed_query(req.query)
+    except Exception as e:
+        logger.error("Embedding failed: %s", e)
+        raise HTTPException(status_code=500, detail="Failed to embed query")
+
+    settings = _get_settings()
+    user_tags = [req.team_name] + ([req.area_name] if req.area_name else [])
+
+    try:
+        pool = await get_pool(settings.database_url)
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                """
+                SELECT
+                    c.text,
+                    c.section_title,
+                    s.title AS source_title,
+                    s.url AS source_url,
+                    s.tags AS source_tags,
+                    1 - (c.embedding <=> $1::vector) AS similarity,
+                    (1 - (c.embedding <=> $1::vector)) *
+                        (1
+                         + $2::float * cardinality(
+                             ARRAY(SELECT unnest(s.tags) INTERSECT SELECT unnest($3::text[]))
+                           )
+                         + $4::float * (CASE WHEN 'org' = ANY(s.tags) THEN 1 ELSE 0 END)
+                        ) AS boosted_score
+                FROM chunks c
+                JOIN sources s ON c.source_id = s.id
+                WHERE s.status = 'active'
+                ORDER BY boosted_score DESC
+                LIMIT $5
+                """,
+                np.array(query_vector, dtype=np.float32),
+                settings.tag_match_weight,
+                user_tags,
+                settings.org_tag_weight,
+                req.limit,
+            )
+    except Exception as e:
+        logger.error("Database error during search: %s", e)
+        raise HTTPException(status_code=503, detail="Database unavailable")
+
+    from docforge.query_log import log_query
+
+    request_ms = int((time.perf_counter() - start) * 1000)
+
+    # team_name and area_name remain self-declared (routing hints, not identity).
+    # user_name and user_oid come from the token when present.
+    await log_query(
+        pool,
+        user.preferred_username if user else req.user_name,
+        req.team_name,
+        req.area_name,
+        req.query,
+        len(rows),
+        user_oid=user.oid if user else None,
+        request_ms=request_ms,
+    )
+
+    results = [
+        SearchResult(
+            text=row["text"],
+            section_title=row["section_title"],
+            source_title=row["source_title"],
+            source_url=row["source_url"],
+            source_tags=list(row["source_tags"] or []),
+            similarity=float(row["similarity"]),
+        )
+        for row in rows
+    ]
+
+    return SearchResponse(results=results, query=req.query, count=len(results))
+
+
+@app.get("/sources")
+async def list_sources(user=Depends(_auth_dependency)) -> dict[str, Any]:
+    """List all indexed documentation sources."""
+    settings = _get_settings()
+    try:
+        pool = await get_pool(settings.database_url)
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                """
+                SELECT title, url, status, last_crawled_at,
+                       (SELECT count(*) FROM chunks WHERE source_id = s.id) AS chunk_count
+                FROM sources s
+                ORDER BY title
+                """
+            )
+    except Exception as e:
+        logger.error("Database error listing sources: %s", e)
+        raise HTTPException(status_code=503, detail="Database unavailable")
+
+    return {
+        "count": len(rows),
+        "sources": [
+            {
+                "title": row["title"],
+                "url": row["url"],
+                "status": row["status"],
+                "chunk_count": row["chunk_count"],
+            }
+            for row in rows
+        ],
+    }

docforge/cli.py

@@ -0,0 +1,296 @@
+"""docforge CLI — forge searchable context from documentation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+import typer
+
+app = typer.Typer(
+    help="Forge searchable context from Confluence and git repos for AI coding assistants.",
+)
+
+
+@app.command()
+def init(name: str = typer.Argument(help="Project directory name")):
+    """Scaffold a new docforge project with config templates."""
+    target = Path(name)
+    if target.exists():
+        typer.echo(f"Error: directory '{name}' already exists.", err=True)
+        raise typer.Exit(1)
+
+    import importlib.resources as resources
+
+    templates_dir = resources.files("docforge") / "templates"
+    target.mkdir(parents=True)
+
+    for item in templates_dir.iterdir():
+        dest = target / item.name
+        if hasattr(item, "read_bytes"):
+            dest.write_bytes(item.read_bytes())
+            typer.echo(f"  Created {dest}")
+
+    typer.echo(f"\nProject scaffolded in {target}/")
+    typer.echo("Next steps:")
+    typer.echo(f"  cd {name}")
+    typer.echo("  # Edit docforge.yml with your Confluence URL")
+    typer.echo("  # Edit sources.yml with your page IDs")
+    typer.echo("  # Edit .env with your credentials")
+    typer.echo("  docker compose up -d db")
+    typer.echo("  docforge init-db")
+    typer.echo("  docforge ingest")
+    typer.echo("  docforge serve")
+
+
+@app.command(name="init-db")
+def init_db():
+    """Initialize the database schema."""
+    asyncio.run(_init_db())
+
+
+@app.command()
+def ingest(
+    purge_orphans: bool = typer.Option(
+        False,
+        "--purge-orphans",
+        help="Report DB sources absent from sources.yml. Dry-run; use --confirm to delete.",
+    ),
+    confirm: bool = typer.Option(
+        False,
+        "--confirm",
+        help="Required alongside --purge-orphans to actually delete orphans.",
+    ),
+):
+    """Crawl all sources, embed, and store in PostgreSQL."""
+    _setup_logging()
+    if confirm and not purge_orphans:
+        typer.echo("Error: --confirm only applies to --purge-orphans", err=True)
+        raise typer.Exit(1)
+    asyncio.run(_ingest(purge_orphans=purge_orphans, confirm=confirm))
+
+
+@app.command()
+def search(
+    query: str = typer.Argument(help="Search query"),
+    user_name: str = typer.Option(
+        None,
+        "--user",
+        help="Your name (required; falls back to default_user_name setting)",
+    ),
+    team_name: str = typer.Option(
+        None,
+        "--team",
+        help="Your team tag (required; falls back to default_team_name setting)",
+    ),
+    area_name: str = typer.Option(
+        None,
+        "--area",
+        help="Your area tag (optional; falls back to default_area_name setting)",
+    ),
+    limit: int = typer.Option(5, help="Max results"),
+):
+    """Search the documentation index."""
+    _setup_logging()
+    from docforge.config import Settings
+
+    settings = Settings()
+    resolved_user = user_name or settings.default_user_name
+    resolved_team = team_name or settings.default_team_name
+    resolved_area = area_name or (settings.default_area_name or None) or None
+
+    if not resolved_user:
+        typer.echo(
+            "Error: --user is required (or set default_user_name in docforge.yml).",
+            err=True,
+        )
+        raise typer.Exit(1)
+    if not resolved_team:
+        typer.echo(
+            "Error: --team is required (or set default_team_name in docforge.yml).",
+            err=True,
+        )
+        raise typer.Exit(1)
+
+    asyncio.run(_search(query, resolved_user, resolved_team, resolved_area, limit))
+
+
+@app.command()
+def serve(api: bool = typer.Option(False, help="Run FastAPI search API instead of MCP")):
+    """Run the MCP server (or FastAPI API with --api)."""
+    _setup_logging()
+    if api:
+        import uvicorn
+
+        from docforge.api import app as fastapi_app
+
+        uvicorn.run(fastapi_app, host="0.0.0.0", port=8000)
+    else:
+        from docforge.mcp_server import mcp
+
+        mcp.run()
+
+
+@app.command()
+def status():
+    """Show index statistics and health."""
+    asyncio.run(_status())
+
+
+@app.command(name="lint-docs")
+def lint_docs(
+    repo_path: Path = typer.Argument(..., help="Path to the repo root to lint"),
+) -> None:
+    """Lint a repo's README + CLAUDE.md + docs/ for banned-content rules."""
+    from docforge.lint import format_report, lint_repo
+
+    if not repo_path.is_dir():
+        typer.echo(f"Error: {repo_path} is not a directory", err=True)
+        raise typer.Exit(1)
+
+    report = lint_repo(repo_path)
+    typer.echo(format_report(report, repo_path))
+    if report.findings:
+        raise typer.Exit(1)
+
+
+def _setup_logging():
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)-8s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+
+async def _init_db():
+    from docforge.config import Settings
+    from docforge.db import init_db as do_init_db
+
+    settings = Settings()
+    typer.echo(f"Initializing database: {settings.database_url.split('@')[-1]}")
+    try:
+        await do_init_db(settings.database_url)
+    except OSError as e:
+        typer.echo(
+            f"Error: Cannot connect to database. Is PostgreSQL running?\n{e}",
+            err=True,
+        )
+        raise typer.Exit(1)
+    except Exception as e:
+        typer.echo(f"Error initializing database: {e}", err=True)
+        raise typer.Exit(1)
+    typer.echo("Database initialized successfully.")
+
+
+async def _ingest(purge_orphans: bool = False, confirm: bool = False):
+    from docforge.config import Settings
+    from docforge.db import close_pool
+    from docforge.ingest import ingest_all
+
+    settings = Settings()
+    try:
+        await ingest_all(settings, purge_orphans=purge_orphans, confirm=confirm)
+    except OSError as e:
+        typer.echo(
+            f"Error: Cannot connect to database. Is PostgreSQL running?\n{e}",
+            err=True,
+        )
+        raise typer.Exit(1)
+    except RuntimeError as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+    except Exception as e:
+        typer.echo(f"Error during ingest: {e}", err=True)
+        raise typer.Exit(1)
+    finally:
+        await close_pool()
+
+
+async def _search(query: str, user_name: str, team_name: str, area_name: str | None, limit: int):
+    import numpy as np
+
+    from docforge.config import Settings
+    from docforge.db import close_pool, get_pool
+    from docforge.processors.embedder import Embedder
+    from docforge.query_log import log_query
+
+    settings = Settings()
+    try:
+        embedder = Embedder(settings.embedding_model, hf_token=settings.hf_token.get_secret_value())
+    except RuntimeError as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+    query_vector = embedder.embed_query(query)
+    user_tags = [team_name] + ([area_name] if area_name else [])
+
+    try:
+        pool = await get_pool(settings.database_url)
+        async with pool.acquire() as conn:
+            rows = await conn.fetch(
+                """
+                SELECT c.text, c.section_title, s.title AS source_title,
+                       s.tags AS source_tags,
+                       1 - (c.embedding <=> $1::vector) AS similarity,
+                       (1 - (c.embedding <=> $1::vector)) *
+                         (1
+                          + $2::float * cardinality(
+                              ARRAY(SELECT unnest(s.tags) INTERSECT SELECT unnest($3::text[]))
+                            )
+                          + $4::float * (CASE WHEN 'org' = ANY(s.tags) THEN 1 ELSE 0 END)
+                         ) AS boosted_score
+                FROM chunks c JOIN sources s ON c.source_id = s.id
+                WHERE s.status = 'active'
+                ORDER BY boosted_score DESC LIMIT $5
+                """,
+                np.array(query_vector, dtype=np.float32),
+                settings.tag_match_weight,
+                user_tags,
+                settings.org_tag_weight,
+                limit,
+            )
+        await log_query(pool, user_name, team_name, area_name, query, len(rows))
+    except OSError as e:
+        typer.echo(
+            f"Error: Cannot connect to database. Is PostgreSQL running?\n{e}",
+            err=True,
+        )
+        raise typer.Exit(1)
+    finally:
+        await close_pool()
+
+    if not rows:
+        typer.echo("No results found.")
+        return
+
+    for i, row in enumerate(rows, 1):
+        sim = row["similarity"]
+        src = row["source_title"]
+        sec = row["section_title"] or ""
+        tags = list(row["source_tags"] or [])
+        typer.echo(f"\n--- Result {i} (relevance: {sim:.2f}) --- {src}")
+        if sec:
+            typer.echo(f"Section: {sec}")
+        if tags:
+            typer.echo(f"Tags: {', '.join(tags)}")
+        typer.echo(row["text"][:500])
+
+
+async def _status():
+    from docforge.config import Settings
+    from docforge.db import close_pool, get_pool
+
+    settings = Settings()
+    try:
+        pool = await get_pool(settings.database_url)
+        async with pool.acquire() as conn:
+            sources = await conn.fetchval("SELECT count(*) FROM sources")
+            chunks = await conn.fetchval("SELECT count(*) FROM chunks")
+        typer.echo(f"Sources: {sources}")
+        typer.echo(f"Chunks:  {chunks}")
+        typer.echo(f"DB:      {settings.database_url.split('@')[-1]}")
+    except Exception as e:
+        typer.echo(f"Error connecting to database: {e}", err=True)
+    finally:
+        await close_pool()

docforge/config.py

@@ -0,0 +1,99 @@
+"""Settings loading — merges defaults, docforge.yml, .env, env vars, and kwargs.
+
+Precedence: kwargs > yml > env > .env > defaults. yml values are passed to
+pydantic-settings via `super().__init__(**merged)`, which treats them as
+init-kwargs (highest priority after explicit kwargs).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, SecretStr, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AuthSettings(BaseModel):
+    mode: Literal["none", "entra"] = "none"
+    tenant_id: str = ""
+    audience: str = ""
+
+    @model_validator(mode="after")
+    def _validate_entra_fields(self):
+        if self.mode == "entra":
+            if not self.tenant_id:
+                raise ValueError(
+                    "auth.mode=entra requires auth.tenant_id to be set "
+                    "(via docforge.yml or AUTH__TENANT_ID env var)"
+                )
+            if not self.audience:
+                raise ValueError(
+                    "auth.mode=entra requires auth.audience to be set "
+                    "(via docforge.yml or AUTH__AUDIENCE env var)"
+                )
+        return self
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        env_nested_delimiter="__",
+    )
+
+    # Database
+    database_url: str = "postgresql://docforge:localdev@localhost:5432/docforge"
+
+    # Confluence
+    confluence_base_url: str = ""
+    confluence_email: str = ""
+    confluence_api_token: SecretStr = SecretStr("")
+
+    # HuggingFace token for model access
+    hf_token: SecretStr = SecretStr("")
+
+    # Embedding model
+    embedding_model: str = "google/embeddinggemma-300m"
+    embedding_dimensions: int = 768
+    chunk_max_tokens: int = 500
+
+    # Sources config
+    sources_file: str = "sources.yml"
+
+    # Ranking weights (see docforge.ranking.compute_boosted_score)
+    tag_match_weight: float = 0.1
+    org_tag_weight: float = 0.05
+
+    # Default identity (used as CLI flag defaults when set via env/yml)
+    default_user_name: str = ""
+    default_team_name: str = ""
+    default_area_name: str = ""
+
+    # Auth (opt-in Entra ID for /search + /sources)
+    auth: AuthSettings = AuthSettings()
+
+    # query_log retention — app-level cleanup loop deletes rows older than this
+    query_log_retention_days: int = 180
+
+    def __init__(self, **kwargs) -> None:
+        # Load from docforge.yml if present, then overlay with env vars
+        yml_path = Path("docforge.yml")
+        yml_values = {}
+        if yml_path.exists():
+            with open(yml_path) as f:
+                yml = yaml.safe_load(f) or {}
+            # Flatten nested embedding config
+            if "embedding" in yml:
+                emb = yml.pop("embedding")
+                if "model" in emb:
+                    yml_values["embedding_model"] = emb["model"]
+                if "dimensions" in emb:
+                    yml_values["embedding_dimensions"] = emb["dimensions"]
+                if "chunk_max_tokens" in emb:
+                    yml_values["chunk_max_tokens"] = emb["chunk_max_tokens"]
+            yml_values.update(yml)
+        # YAML values are defaults; explicit kwargs and env vars override
+        merged = {**yml_values, **kwargs}
+        super().__init__(**merged)

docforge/crawlers/__init__.py

@@ -0,0 +1 @@
+"""Source crawlers — Confluence REST API and local git repo file walkers."""