docforge-cli 0.2.0__py3-none-any.whl

docforge/crawlers/confluence.py

@@ -0,0 +1,109 @@
+"""Confluence REST API v2 page crawler with retry logic for transient errors."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+TRANSIENT_STATUS_CODES = {429, 502, 503, 504}
+MAX_RETRIES = 3
+BACKOFF_BASE = 2.0
+
+
+@dataclass
+class CrawledPage:
+    page_id: str
+    title: str
+    space_key: str
+    html_content: str
+    content_hash: str
+    version: int
+    url: str
+
+
+async def crawl_page(
+    page_id: str,
+    *,
+    base_url: str,
+    email: str,
+    api_token: str,
+) -> CrawledPage:
+    """Fetch a Confluence page via REST API v2 and return its content."""
+    api_url = f"{base_url}/wiki/api/v2/pages/{page_id}"
+    params = {"body-format": "storage"}
+    auth = httpx.BasicAuth(email, api_token)
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        response = await _request_with_retry(client, api_url, params=params, auth=auth)
+
+    data = response.json()
+    html_content = data.get("body", {}).get("storage", {}).get("value", "")
+    title = data.get("title", "")
+    version = data.get("version", {}).get("number", 0)
+    space_id = data.get("spaceId", "")
+
+    content_hash = hashlib.sha256(html_content.encode()).hexdigest()
+    page_url = f"{base_url}/wiki/spaces/{space_id}/pages/{page_id}"
+
+    return CrawledPage(
+        page_id=page_id,
+        title=title,
+        space_key=space_id,
+        html_content=html_content,
+        content_hash=content_hash,
+        version=version,
+        url=page_url,
+    )
+
+
+async def _request_with_retry(
+    client: httpx.AsyncClient,
+    url: str,
+    *,
+    params: dict | None = None,
+    auth: httpx.BasicAuth | None = None,
+) -> httpx.Response:
+    """Make an HTTP GET request with retry logic for transient failures."""
+    import asyncio
+
+    for attempt in range(MAX_RETRIES + 1):
+        try:
+            response = await client.get(url, params=params, auth=auth)
+
+            if response.status_code == 200:
+                return response
+
+            if response.status_code in TRANSIENT_STATUS_CODES:
+                retry_after = float(response.headers.get("Retry-After", BACKOFF_BASE**attempt))
+                logger.warning(
+                    "Transient error %d for %s, retrying in %.1fs (attempt %d/%d)",
+                    response.status_code,
+                    url,
+                    retry_after,
+                    attempt + 1,
+                    MAX_RETRIES,
+                )
+                await asyncio.sleep(retry_after)
+                continue
+
+            # Permanent failure
+            response.raise_for_status()
+
+        except httpx.TimeoutException:
+            if attempt < MAX_RETRIES:
+                wait = BACKOFF_BASE**attempt
+                logger.warning("Timeout for %s, retrying in %.1fs", url, wait)
+                await asyncio.sleep(wait)
+                continue
+            raise
+
+    raise httpx.HTTPStatusError(
+        f"Max retries exceeded for {url}",
+        request=httpx.Request("GET", url),
+        response=response,  # type: ignore[possibly-undefined]
+    )

docforge/crawlers/git.py

@@ -0,0 +1,79 @@
+"""Crawler for local git repository documentation files.
+
+Reads markdown files (README.md, CLAUDE.md, docs/**/*.md) from a local
+git repo directory. No git clone — the repo must already be on disk.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CrawledFile:
+    file_path: str
+    title: str
+    content: str
+    content_hash: str
+    repo_path: str
+
+
+def crawl_repo(
+    repo_path: str,
+    include_patterns: list[str] | None = None,
+) -> list[CrawledFile]:
+    """Read documentation files from a local git repository.
+
+    Args:
+        repo_path: Absolute path to the repo root (e.g., "E:/MyRepo").
+        include_patterns: Glob patterns for files to include.
+                          Defaults to ["README.md", "CLAUDE.md", "docs/**/*.md"].
+    """
+    if include_patterns is None:
+        include_patterns = ["README.md", "CLAUDE.md", "docs/**/*.md"]
+
+    root = Path(repo_path)
+    if not root.is_dir():
+        logger.warning("Repo path does not exist: %s", repo_path)
+        return []
+
+    results: list[CrawledFile] = []
+    seen: set[Path] = set()
+
+    for pattern in include_patterns:
+        for file_path in root.glob(pattern):
+            if not file_path.is_file():
+                continue
+            if file_path in seen:
+                continue
+            seen.add(file_path)
+
+            try:
+                content = file_path.read_text(encoding="utf-8")
+            except (UnicodeDecodeError, OSError) as e:
+                logger.warning("Cannot read %s: %s", file_path, e)
+                continue
+
+            if not content.strip():
+                continue
+
+            relative = file_path.relative_to(root)
+            content_hash = hashlib.sha256(content.encode()).hexdigest()
+
+            results.append(
+                CrawledFile(
+                    file_path=str(relative),
+                    title=str(relative),
+                    content=content,
+                    content_hash=content_hash,
+                    repo_path=str(root),
+                )
+            )
+
+    logger.info("Found %d files in %s", len(results), repo_path)
+    return results

docforge/db.py

@@ -0,0 +1,57 @@
+"""asyncpg connection pool + pgvector registration.
+
+Module-level `_pool` is created lazily on first `get_pool()` call and
+shared across all callers. `init_db()` applies the packaged schema.sql
+and any migration scripts.
+"""
+
+from __future__ import annotations
+
+import asyncpg
+from pgvector.asyncpg import register_vector
+
+_pool: asyncpg.Pool | None = None
+
+
+async def get_pool(database_url: str) -> asyncpg.Pool:
+    """Return the module-level asyncpg pool, creating it on first call."""
+    global _pool
+    if _pool is None:
+        _pool = await asyncpg.create_pool(
+            database_url,
+            min_size=1,
+            max_size=5,
+            init=_init_connection,
+        )
+    return _pool
+
+
+async def _init_connection(conn: asyncpg.Connection) -> None:
+    await register_vector(conn)
+
+
+async def close_pool() -> None:
+    """Close and clear the module-level asyncpg pool if it exists."""
+    global _pool
+    if _pool is not None:
+        await _pool.close()
+        _pool = None
+
+
+async def init_db(database_url: str) -> None:
+    """Apply schema and migrations from the docforge package."""
+    import importlib.resources as resources
+
+    sql_dir = resources.files("docforge") / "sql"
+    schema_sql = (sql_dir / "schema.sql").read_text(encoding="utf-8")
+
+    conn = await asyncpg.connect(database_url)
+    try:
+        await conn.execute(schema_sql)
+
+        migrations_dir = sql_dir / "migrations"
+        for migration in sorted(migrations_dir.iterdir()):
+            if str(migration).endswith(".sql"):
+                await conn.execute(migration.read_text(encoding="utf-8"))
+    finally:
+        await conn.close()

docforge/ingest.py

@@ -0,0 +1,401 @@
+"""Ingest pipeline — crawl → parse → chunk → embed → store.
+
+`ingest_all` loads the sources list and runs the appropriate crawler for
+each source type (Confluence page or local git repo). Per-source failures
+are logged but do not abort the run.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Callable
+
+import asyncpg
+import numpy as np
+
+from docforge.config import Settings
+from docforge.crawlers.confluence import crawl_page
+from docforge.crawlers.git import crawl_repo
+from docforge.db import get_pool
+from docforge.processors.chunker import chunk_sections
+from docforge.processors.embedder import Embedder
+from docforge.processors.parser import Section, parse_confluence_html
+from docforge.sources import (
+    ConfluenceSourceConfig,
+    GitRepoSourceConfig,
+    load_sources,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _git_source_identifier(repo_path: str, file_path: str) -> str:
+    """Canonical identifier for a git-repo source row. Must stay in sync with
+    what _ingest_git_source INSERTs and what _purge_orphans matches against."""
+    return f"git:{repo_path}:{file_path}"
+
+
+async def ingest_all(
+    settings: Settings,
+    *,
+    purge_orphans: bool = False,
+    confirm: bool = False,
+) -> None:
+    """Run the full ingest pipeline for all configured sources.
+
+    When purge_orphans=True, after all sources have been ingested, any
+    `sources` rows whose identifier is not in the current sources.yml are
+    reported (and — if confirm=True — deleted). See _purge_orphans."""
+    sources = load_sources(settings.sources_file)
+    logger.info("Loaded %d sources from %s", len(sources), settings.sources_file)
+
+    logger.info("Loading embedding model...")
+    embedder = Embedder(settings.embedding_model, hf_token=settings.hf_token.get_secret_value())
+
+    pool = await get_pool(settings.database_url)
+    tokenizer_fn = embedder.get_tokenizer_fn()
+
+    succeeded = 0
+    failed = 0
+    failed_names: list[str] = []
+    current_identifiers: set[str] = set()
+
+    for source in sources:
+        try:
+            if isinstance(source, ConfluenceSourceConfig):
+                await _ingest_confluence_source(source, settings, pool, embedder, tokenizer_fn)
+                current_identifiers.add(source.page_id)
+            elif isinstance(source, GitRepoSourceConfig):
+                git_identifiers = await _ingest_git_source(source, pool, embedder, tokenizer_fn)
+                current_identifiers.update(git_identifiers)
+            succeeded += 1
+        except Exception:
+            failed += 1
+            failed_names.append(source.title)
+            logger.error("Failed to ingest source: %s", source.title, exc_info=True)
+
+    logger.info(
+        "Ingest complete: %d succeeded, %d failed out of %d sources",
+        succeeded,
+        failed,
+        len(sources),
+    )
+    if failed_names:
+        logger.warning("Failed sources: %s", ", ".join(failed_names))
+
+    if purge_orphans:
+        # Only purge if ALL sources ingested cleanly. A failed source would
+        # leave its identifier out of current_identifiers and get purged as
+        # an orphan — data loss. Require zero failures before allowing purge.
+        if failed > 0:
+            logger.warning(
+                "Skipping --purge-orphans: %d source(s) failed to ingest; "
+                "their identifiers would be incorrectly classified as orphans.",
+                failed,
+            )
+        else:
+            await _purge_orphans(pool, current_identifiers, confirm=confirm)
+
+
+async def _ingest_confluence_source(
+    source: ConfluenceSourceConfig,
+    settings: Settings,
+    pool: asyncpg.Pool,
+    embedder: Embedder,
+    tokenizer_fn: Callable[[str], int],
+) -> None:
+    """Ingest a single Confluence page: crawl, parse HTML, chunk, embed, store."""
+    logger.info("Crawling Confluence: %s (page_id=%s)", source.title, source.page_id)
+
+    page = await crawl_page(
+        source.page_id,
+        base_url=settings.confluence_base_url,
+        email=settings.confluence_email,
+        api_token=settings.confluence_api_token.get_secret_value(),
+    )
+
+    existing_hash = await _get_existing_hash(pool, source.page_id)
+    if existing_hash == page.content_hash:
+        logger.info("No changes detected for: %s", source.title)
+        return
+
+    logger.info("Parsing: %s", source.title)
+    sections = parse_confluence_html(page.html_content)
+    logger.info("Found %d sections", len(sections))
+
+    chunks = chunk_sections(sections, max_tokens=500, tokenizer_fn=tokenizer_fn)
+    logger.info("Created %d chunks", len(chunks))
+
+    if not chunks:
+        logger.warning("No chunks produced for: %s", source.title)
+        return
+
+    logger.info("Embedding %d chunks...", len(chunks))
+    texts = [chunk.text for chunk in chunks]
+    embeddings = embedder.embed(texts)
+
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            source_id = await conn.fetchval(
+                """
+                INSERT INTO sources (type, url, title, confluence_page_id,
+                                     confluence_space_key, last_crawled_at,
+                                     content_hash, status, tags)
+                VALUES ($1, $2, $3, $4, $5, $6, $7, 'active', $8)
+                ON CONFLICT (confluence_page_id)
+                DO UPDATE SET
+                    title = EXCLUDED.title,
+                    url = EXCLUDED.url,
+                    last_crawled_at = EXCLUDED.last_crawled_at,
+                    content_hash = EXCLUDED.content_hash,
+                    status = 'active',
+                    tags = EXCLUDED.tags
+                RETURNING id
+                """,
+                source.type,
+                page.url,
+                page.title,
+                source.page_id,
+                source.space_key,
+                datetime.now(timezone.utc),
+                page.content_hash,
+                source.tags,
+            )
+
+            await conn.execute("DELETE FROM chunks WHERE source_id = $1", source_id)
+
+            for chunk, embedding in zip(chunks, embeddings):
+                await conn.execute(
+                    """
+                    INSERT INTO chunks (source_id, chunk_index, text,
+                                        embedding, section_title)
+                    VALUES ($1, $2, $3, $4, $5)
+                    """,
+                    source_id,
+                    chunk.chunk_index,
+                    chunk.text,
+                    np.array(embedding, dtype=np.float32),
+                    chunk.section_title,
+                )
+
+    logger.info("Stored %d chunks for: %s", len(chunks), source.title)
+
+
+async def _ingest_git_source(
+    source: GitRepoSourceConfig,
+    pool: asyncpg.Pool,
+    embedder: Embedder,
+    tokenizer_fn: Callable[[str], int],
+) -> list[str]:
+    """Ingest documentation files from a local git repository.
+
+    Returns the list of source identifiers enumerated from the repo (one per
+    file crawled, not only those actually re-embedded). The caller can feed
+    this into _purge_orphans without re-walking the filesystem.
+
+    Raises FileNotFoundError if the configured repo path does not exist —
+    important because crawl_repo otherwise silently returns [] for a missing
+    path. A silent empty walk would let --purge-orphans delete all of the
+    repo's historical sources as "orphans" on a transient mount failure."""
+    from pathlib import Path
+
+    logger.info("Crawling git repo: %s (%s)", source.title, source.repo_path)
+
+    if not Path(source.repo_path).is_dir():
+        raise FileNotFoundError(f"Configured git repo path does not exist: {source.repo_path}")
+
+    files = crawl_repo(source.repo_path, source.include_patterns)
+    identifiers = [_git_source_identifier(source.repo_path, f.file_path) for f in files]
+
+    for file in files:
+        identifier = _git_source_identifier(source.repo_path, file.file_path)
+
+        existing_hash = await _get_hash_by_identifier(pool, identifier)
+        if existing_hash == file.content_hash:
+            logger.info("No changes: %s/%s", source.title, file.title)
+            continue
+
+        sections = _parse_markdown(file.content)
+        chunks = chunk_sections(sections, max_tokens=500, tokenizer_fn=tokenizer_fn)
+
+        if not chunks:
+            continue
+
+        logger.info("Embedding %d chunks for %s/%s", len(chunks), source.title, file.title)
+        texts = [chunk.text for chunk in chunks]
+        embeddings = embedder.embed(texts)
+
+        url = f"file://{source.repo_path}/{file.file_path}"
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                source_id = await conn.fetchval(
+                    """
+                    INSERT INTO sources (type, url, title, source_identifier,
+                                         last_crawled_at, content_hash, status, tags)
+                    VALUES ($1, $2, $3, $4, $5, $6, 'active', $7)
+                    ON CONFLICT (source_identifier)
+                        WHERE source_identifier IS NOT NULL
+                    DO UPDATE SET
+                        title = EXCLUDED.title,
+                        last_crawled_at = EXCLUDED.last_crawled_at,
+                        content_hash = EXCLUDED.content_hash,
+                        status = 'active',
+                        tags = EXCLUDED.tags
+                    RETURNING id
+                    """,
+                    "git_repo",
+                    url,
+                    f"{source.title}/{file.title}",
+                    identifier,
+                    datetime.now(timezone.utc),
+                    file.content_hash,
+                    source.tags,
+                )
+
+                await conn.execute("DELETE FROM chunks WHERE source_id = $1", source_id)
+
+                for chunk, embedding in zip(chunks, embeddings):
+                    await conn.execute(
+                        """
+                        INSERT INTO chunks (source_id, chunk_index, text,
+                                            embedding, section_title)
+                        VALUES ($1, $2, $3, $4, $5)
+                        """,
+                        source_id,
+                        chunk.chunk_index,
+                        chunk.text,
+                        np.array(embedding, dtype=np.float32),
+                        chunk.section_title,
+                    )
+
+        logger.info("Stored %d chunks for: %s/%s", len(chunks), source.title, file.title)
+
+    return identifiers
+
+
+def _parse_markdown(content: str) -> list[Section]:
+    """Parse markdown content into sections by headings."""
+    sections: list[Section] = []
+    current_title = ""
+    current_level = 0
+    current_parts: list[str] = []
+
+    for line in content.split("\n"):
+        if line.startswith("#"):
+            if current_parts:
+                text = "\n".join(current_parts).strip()
+                if text:
+                    sections.append(Section(title=current_title, text=text, level=current_level))
+                current_parts = []
+
+            level = len(line) - len(line.lstrip("#"))
+            current_title = line.lstrip("#").strip()
+            current_level = level
+        else:
+            current_parts.append(line)
+
+    if current_parts:
+        text = "\n".join(current_parts).strip()
+        if text:
+            sections.append(Section(title=current_title, text=text, level=current_level))
+
+    return sections
+
+
+async def _get_existing_hash(pool: asyncpg.Pool, page_id: str) -> str | None:
+    """Get the content hash of a Confluence source."""
+    async with pool.acquire() as conn:
+        return await conn.fetchval(
+            "SELECT content_hash FROM sources WHERE confluence_page_id = $1",
+            page_id,
+        )
+
+
+async def _get_hash_by_identifier(pool: asyncpg.Pool, identifier: str) -> str | None:
+    """Get the content hash of a source by its identifier."""
+    async with pool.acquire() as conn:
+        return await conn.fetchval(
+            "SELECT content_hash FROM sources WHERE source_identifier = $1",
+            identifier,
+        )
+
+
+async def _purge_orphans(
+    pool: asyncpg.Pool,
+    current_identifiers: set[str],
+    confirm: bool,
+) -> tuple[int, int]:
+    """Find `sources` rows whose identifier is not in the current sources.yml,
+    report them, and (if confirm=True) delete them along with their chunks.
+
+    Identifier format:
+        - Confluence: the page_id string (e.g., "5108006937")
+        - Git:        f"git:{repo_path}:{file_path}"
+
+    Returns (orphan_source_count, orphan_chunk_count). When confirm=False,
+    returns the counts of what WOULD be deleted and leaves the DB untouched.
+
+    chunks.source_id has ON DELETE CASCADE, so deleting from sources
+    cascades to chunks automatically.
+    """
+    if not current_identifiers and confirm:
+        logger.error(
+            "_purge_orphans called with empty current_identifiers and confirm=True. "
+            "This would delete every source in the DB. Aborting — this is almost "
+            "certainly a caller bug (e.g., load_sources returned empty)."
+        )
+        return (0, 0)
+
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            rows = await conn.fetch(
+                """
+                SELECT id,
+                       title,
+                       COALESCE(confluence_page_id, source_identifier) AS identifier
+                  FROM sources
+                 WHERE COALESCE(confluence_page_id, source_identifier) IS NOT NULL
+                """
+            )
+            db_identifiers = {r["identifier"]: r for r in rows}
+            orphan_ids = [
+                r["id"] for ident, r in db_identifiers.items() if ident not in current_identifiers
+            ]
+
+            if not orphan_ids:
+                logger.info("No orphan sources detected.")
+                return (0, 0)
+
+            chunk_count = await conn.fetchval(
+                "SELECT count(*) FROM chunks WHERE source_id = ANY($1::uuid[])",
+                orphan_ids,
+            )
+
+            logger.info(
+                "Orphans detected: %d sources / %d chunks not in current sources.yml",
+                len(orphan_ids),
+                chunk_count,
+            )
+            for ident, r in db_identifiers.items():
+                if ident not in current_identifiers:
+                    logger.debug("  orphan: %s  (%s)", r["title"], ident)
+
+            if not confirm:
+                logger.info(
+                    "Would delete %d orphan sources (%d chunks). Re-run with --confirm to execute.",
+                    len(orphan_ids),
+                    chunk_count,
+                )
+                return (len(orphan_ids), chunk_count)
+
+            await conn.execute(
+                "DELETE FROM sources WHERE id = ANY($1::uuid[])",
+                orphan_ids,
+            )
+            logger.info(
+                "Purged %d orphan sources (%d chunks).",
+                len(orphan_ids),
+                chunk_count,
+            )
+            return (len(orphan_ids), chunk_count)