iil-docs-agent 0.2.0__py3-none-any.whl

docs_agent/__init__.py

@@ -0,0 +1,7 @@
+"""Docs Agent — AI-assisted documentation quality tool.
+
+Provides AST-based docstring coverage scanning, DIATAXIS classification,
+and LLM-powered docstring generation with non-destructive code insertion.
+"""
+
+__version__ = "0.2.0"

docs_agent/analyzer/__init__.py

@@ -0,0 +1 @@
+"""Analyzer components — AST scanner and DIATAXIS classifier."""

docs_agent/analyzer/ast_scanner.py

@@ -0,0 +1,231 @@
+"""AST-based docstring coverage scanner.
+
+Scans Python modules via the `ast` module and reports undocumented items:
+classes, functions, methods. No LLM required — pure static analysis.
+"""
+
+from __future__ import annotations
+
+import ast
+import logging
+from pathlib import Path
+
+from docs_agent.models import CodeItem, ItemKind, ModuleCoverage, RepoCoverage
+
+logger = logging.getLogger(__name__)
+
+SKIP_DIRS: set[str] = {
+    "__pycache__",
+    ".git",
+    ".venv",
+    "venv",
+    "node_modules",
+    "migrations",
+    "_build",
+    "_archive",
+    "static",
+    "staticfiles",
+    ".tox",
+    ".mypy_cache",
+    ".pytest_cache",
+    "dist",
+    "build",
+    "eggs",
+    "*.egg-info",
+}
+
+SKIP_FILES: set[str] = {
+    "__init__.py",
+    "conftest.py",
+    "manage.py",
+    "wsgi.py",
+    "asgi.py",
+}
+
+
+def _should_skip_dir(dirname: str) -> bool:
+    """Check if directory should be skipped."""
+    return dirname in SKIP_DIRS or dirname.startswith(".")
+
+
+def _should_skip_file(filename: str) -> bool:
+    """Check if file should be skipped."""
+    return filename in SKIP_FILES or filename.startswith("test_")
+
+
+def _has_docstring(node: ast.AST) -> bool:
+    """Check if an AST node has a docstring."""
+    if not hasattr(node, "body") or not node.body:
+        return False
+    first = node.body[0]
+    if isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant):
+        return isinstance(first.value.value, str)
+    return False
+
+
+def _is_private(name: str) -> bool:
+    """Check if name is private (single underscore prefix, not dunder)."""
+    return name.startswith("_") and not name.startswith("__")
+
+
+def _is_dunder(name: str) -> bool:
+    """Check if name is a dunder method."""
+    return name.startswith("__") and name.endswith("__")
+
+
+SKIP_DUNDERS: set[str] = {
+    "__init__",
+    "__str__",
+    "__repr__",
+    "__eq__",
+    "__hash__",
+    "__lt__",
+    "__le__",
+    "__gt__",
+    "__ge__",
+    "__len__",
+    "__bool__",
+    "__contains__",
+    "__iter__",
+    "__next__",
+    "__enter__",
+    "__exit__",
+    "__call__",
+    "__getattr__",
+    "__setattr__",
+    "__delattr__",
+    "__getitem__",
+    "__setitem__",
+    "__delitem__",
+}
+
+
+def scan_module(file_path: Path) -> ModuleCoverage:
+    """Scan a single Python module for docstring coverage.
+
+    Args:
+        file_path: Path to the Python file.
+
+    Returns:
+        ModuleCoverage with all documentable items and their status.
+    """
+    coverage = ModuleCoverage(file_path=file_path)
+
+    try:
+        source = file_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as exc:
+        logger.warning("Cannot read %s: %s", file_path, exc)
+        return coverage
+
+    try:
+        tree = ast.parse(source, filename=str(file_path))
+    except SyntaxError as exc:
+        logger.warning("Syntax error in %s: %s", file_path, exc)
+        return coverage
+
+    # Module-level docstring
+    has_module_doc = _has_docstring(tree)
+    coverage.items.append(
+        CodeItem(
+            name=file_path.stem,
+            kind=ItemKind.MODULE,
+            line=1,
+            has_docstring=has_module_doc,
+            file_path=file_path,
+        )
+    )
+    coverage.total_items += 1
+    coverage.documented_items += int(has_module_doc)
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            has_doc = _has_docstring(node)
+            coverage.items.append(
+                CodeItem(
+                    name=node.name,
+                    kind=ItemKind.CLASS,
+                    line=node.lineno,
+                    has_docstring=has_doc,
+                    file_path=file_path,
+                )
+            )
+            coverage.total_items += 1
+            coverage.documented_items += int(has_doc)
+
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            name = node.name
+
+            if _is_dunder(name) and name in SKIP_DUNDERS:
+                continue
+            if _is_private(name):
+                continue
+
+            # Determine if method or function
+            is_method = any(
+                isinstance(parent, ast.ClassDef)
+                for parent in ast.walk(tree)
+                if isinstance(getattr(parent, "body", None), list)
+                and node in parent.body
+            )
+            kind = ItemKind.METHOD if is_method else ItemKind.FUNCTION
+
+            has_doc = _has_docstring(node)
+            coverage.items.append(
+                CodeItem(
+                    name=name,
+                    kind=kind,
+                    line=node.lineno,
+                    has_docstring=has_doc,
+                    file_path=file_path,
+                )
+            )
+            coverage.total_items += 1
+            coverage.documented_items += int(has_doc)
+
+    return coverage
+
+
+def scan_repo(
+    repo_path: Path,
+    *,
+    apps_only: bool = False,
+) -> RepoCoverage:
+    """Scan an entire repository for docstring coverage.
+
+    Args:
+        repo_path: Root path of the repository.
+        apps_only: If True, only scan `apps/` subdirectory.
+
+    Returns:
+        RepoCoverage with per-module and aggregate statistics.
+    """
+    repo_path = repo_path.resolve()
+    result = RepoCoverage(repo_path=repo_path)
+
+    search_root = repo_path
+    if apps_only:
+        apps_dir = repo_path / "apps"
+        if apps_dir.is_dir():
+            search_root = apps_dir
+        else:
+            src_apps = repo_path / "src" / "apps"
+            if src_apps.is_dir():
+                search_root = src_apps
+
+    for py_file in sorted(search_root.rglob("*.py")):
+        # Skip excluded directories
+        if any(
+            _should_skip_dir(part)
+            for part in py_file.relative_to(repo_path).parts
+        ):
+            continue
+
+        # Skip excluded files
+        if _should_skip_file(py_file.name):
+            continue
+
+        module_cov = scan_module(py_file)
+        if module_cov.total_items > 0:
+            result.modules.append(module_cov)
+
+    return result

docs_agent/analyzer/diataxis_classifier.py

@@ -0,0 +1,206 @@
+"""DIATAXIS heuristic classifier for documentation files.
+
+Classifies Markdown/RST files into DIATAXIS quadrants using
+trigger-word pattern matching. No LLM required for the heuristic pass.
+
+LLM refinement (confidence < 0.7) is deferred to Phase 4.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+from docs_agent.models import DiaxisClassification, DiaxisQuadrant
+
+logger = logging.getLogger(__name__)
+
+TRIGGER_PATTERNS: dict[DiaxisQuadrant, list[str]] = {
+    DiaxisQuadrant.TUTORIAL: [
+        r"step\s+\d",
+        r"getting\s+started",
+        r"\blearn\b",
+        r"we\s+will",
+        r"in\s+this\s+tutorial",
+        r"let[\u2018\u2019']s\s+",
+        r"follow\s+along",
+        r"by\s+the\s+end\b",
+        r"prerequisites",
+    ],
+    DiaxisQuadrant.GUIDE: [
+        r"how\s+to\b",
+        r"\bconfigure\b",
+        r"\bdeploy\b",
+        r"\bfix\b",
+        r"troubleshoot",
+        r"set\s+up\b",
+        r"install\b",
+        r"upgrade\b",
+        r"migrat",
+        r"recipe",
+    ],
+    DiaxisQuadrant.REFERENCE: [
+        r"\bAPI\b",
+        r"\bparameter[s]?\b",
+        r"\breturn[s]?\b",
+        r"type\s*:",
+        r"automodule",
+        r"endpoint[s]?\b",
+        r"\bschema\b",
+        r"class\s+\w+",
+        r"field[s]?\b",
+        r"\bargs\b",
+        r"\bkwargs\b",
+    ],
+    DiaxisQuadrant.EXPLANATION: [
+        r"\bwhy\b",
+        r"architecture",
+        r"\bdesign\b",
+        r"rationale",
+        r"background",
+        r"concept[s]?\b",
+        r"overview",
+        r"philosophy",
+        r"trade-?off",
+        r"decision",
+    ],
+}
+
+DOC_EXTENSIONS: set[str] = {".md", ".rst", ".txt"}
+
+SKIP_DIRS: set[str] = {
+    "_archive",
+    "_build",
+    "source",
+    "node_modules",
+    ".git",
+    ".venv",
+    "venv",
+}
+
+
+def _count_triggers(
+    text: str,
+    patterns: list[str],
+) -> tuple[int, list[str]]:
+    """Count trigger pattern matches in text.
+
+    Args:
+        text: Document text to scan.
+        patterns: Regex patterns to match.
+
+    Returns:
+        Tuple of (match_count, list_of_matched_patterns).
+    """
+    count = 0
+    matched: list[str] = []
+    for pattern in patterns:
+        hits = len(re.findall(pattern, text, re.IGNORECASE))
+        if hits > 0:
+            count += hits
+            matched.append(pattern)
+    return count, matched
+
+
+def classify_file(file_path: Path) -> DiaxisClassification:
+    """Classify a single document file into a DIATAXIS quadrant.
+
+    Args:
+        file_path: Path to the document file.
+
+    Returns:
+        DiaxisClassification with quadrant, confidence, and triggers.
+    """
+    try:
+        text = file_path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        logger.warning("Cannot read %s: %s", file_path, exc)
+        return DiaxisClassification(
+            file_path=file_path,
+            quadrant=DiaxisQuadrant.UNKNOWN,
+            confidence=0.0,
+        )
+
+    scores: dict[DiaxisQuadrant, int] = {}
+    all_triggers: dict[DiaxisQuadrant, list[str]] = {}
+
+    for quadrant, patterns in TRIGGER_PATTERNS.items():
+        count, matched = _count_triggers(text, patterns)
+        scores[quadrant] = count
+        all_triggers[quadrant] = matched
+
+    total = sum(scores.values())
+    if total == 0:
+        return DiaxisClassification(
+            file_path=file_path,
+            quadrant=DiaxisQuadrant.UNKNOWN,
+            confidence=0.0,
+        )
+
+    best_quadrant = max(scores, key=lambda q: scores[q])
+    best_score = scores[best_quadrant]
+    confidence = best_score / total
+
+    # Boost confidence if file path hints at quadrant
+    path_lower = str(file_path).lower()
+    path_hints: dict[str, DiaxisQuadrant] = {
+        "tutorial": DiaxisQuadrant.TUTORIAL,
+        "getting-started": DiaxisQuadrant.TUTORIAL,
+        "guide": DiaxisQuadrant.GUIDE,
+        "howto": DiaxisQuadrant.GUIDE,
+        "how-to": DiaxisQuadrant.GUIDE,
+        "reference": DiaxisQuadrant.REFERENCE,
+        "api": DiaxisQuadrant.REFERENCE,
+        "explanation": DiaxisQuadrant.EXPLANATION,
+        "adr": DiaxisQuadrant.EXPLANATION,
+        "architecture": DiaxisQuadrant.EXPLANATION,
+    }
+    for hint, quadrant in path_hints.items():
+        if hint in path_lower:
+            if quadrant == best_quadrant:
+                confidence = min(confidence + 0.15, 1.0)
+            break
+
+    return DiaxisClassification(
+        file_path=file_path,
+        quadrant=best_quadrant,
+        confidence=round(confidence, 3),
+        triggers=all_triggers[best_quadrant],
+    )
+
+
+def classify_repo(
+    repo_path: Path,
+) -> list[DiaxisClassification]:
+    """Classify all documentation files in a repository.
+
+    Args:
+        repo_path: Root path of the repository.
+
+    Returns:
+        List of DiaxisClassification results.
+    """
+    repo_path = repo_path.resolve()
+    results: list[DiaxisClassification] = []
+
+    docs_dir = repo_path / "docs"
+    if not docs_dir.is_dir():
+        logger.info("No docs/ directory in %s", repo_path)
+        return results
+
+    for doc_file in sorted(docs_dir.rglob("*")):
+        if not doc_file.is_file():
+            continue
+        if doc_file.suffix not in DOC_EXTENSIONS:
+            continue
+        if any(
+            part in SKIP_DIRS
+            for part in doc_file.relative_to(repo_path).parts
+        ):
+            continue
+
+        classification = classify_file(doc_file)
+        results.append(classification)
+
+    return results

docs_agent/analyzer/llm_classifier.py

@@ -0,0 +1,149 @@
+"""LLM-based DIATAXIS classifier for low-confidence documents.
+
+Re-classifies documents where the heuristic classifier has
+confidence < threshold, using the llm_mcp HTTP gateway or
+direct OpenAI API.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Optional
+
+from docs_agent.llm_client import LLMConfig, generate
+from docs_agent.models import DiaxisClassification, DiaxisQuadrant
+from docs_agent.prompts import PROMPT_DIATAXIS_CLASSIFY, SYSTEM_DIATAXIS
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_THRESHOLD = 0.7
+PREVIEW_CHARS = 500
+
+
+async def reclassify_low_confidence(
+    classifications: list[DiaxisClassification],
+    *,
+    threshold: float = DEFAULT_THRESHOLD,
+    config: Optional[LLMConfig] = None,
+) -> list[DiaxisClassification]:
+    """Re-classify documents with confidence below threshold via LLM.
+
+    Args:
+        classifications: Heuristic classification results.
+        threshold: Confidence threshold for re-classification.
+        config: LLM configuration.
+
+    Returns:
+        Updated list with LLM-refined classifications replacing
+        low-confidence entries.
+    """
+    results: list[DiaxisClassification] = []
+
+    for classification in classifications:
+        if classification.confidence >= threshold:
+            results.append(classification)
+            continue
+
+        refined = await _classify_with_llm(
+            classification.file_path, config=config
+        )
+        if refined is not None:
+            results.append(refined)
+        else:
+            results.append(classification)
+
+    return results
+
+
+async def _classify_with_llm(
+    file_path: Path,
+    *,
+    config: Optional[LLMConfig] = None,
+) -> Optional[DiaxisClassification]:
+    """Classify a single document via LLM.
+
+    Args:
+        file_path: Path to the document.
+        config: LLM configuration.
+
+    Returns:
+        DiaxisClassification or None if LLM call fails.
+    """
+    try:
+        text = file_path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        logger.warning("Cannot read %s: %s", file_path, exc)
+        return None
+
+    title = file_path.stem.replace("-", " ").replace("_", " ")
+    preview = text[:PREVIEW_CHARS]
+
+    prompt = PROMPT_DIATAXIS_CLASSIFY.format(
+        title=title,
+        preview=preview,
+    )
+
+    response = await generate(
+        prompt,
+        system_prompt=SYSTEM_DIATAXIS,
+        config=config,
+    )
+
+    if not response.success:
+        logger.warning(
+            "LLM classification failed for %s: %s",
+            file_path, response.error,
+        )
+        return None
+
+    return _parse_classification_response(response.content, file_path)
+
+
+def _parse_classification_response(
+    content: str | dict | list | None,
+    file_path: Path,
+) -> Optional[DiaxisClassification]:
+    """Parse LLM classification response.
+
+    Args:
+        content: LLM response content.
+        file_path: Path to the classified document.
+
+    Returns:
+        DiaxisClassification or None on parse failure.
+    """
+    if isinstance(content, str):
+        try:
+            content = json.loads(content)
+        except json.JSONDecodeError:
+            logger.warning("Failed to parse JSON from LLM response")
+            return None
+
+    if not isinstance(content, dict):
+        return None
+
+    quadrant_str = content.get("quadrant", "").lower().strip()
+    confidence = float(content.get("confidence", 0.5))
+
+    quadrant_map = {
+        "tutorial": DiaxisQuadrant.TUTORIAL,
+        "guide": DiaxisQuadrant.GUIDE,
+        "reference": DiaxisQuadrant.REFERENCE,
+        "explanation": DiaxisQuadrant.EXPLANATION,
+    }
+
+    quadrant = quadrant_map.get(quadrant_str)
+    if quadrant is None:
+        logger.warning(
+            "Unknown quadrant from LLM: %s", quadrant_str
+        )
+        return None
+
+    return DiaxisClassification(
+        file_path=file_path,
+        quadrant=quadrant,
+        confidence=round(confidence, 3),
+        triggers=[f"llm: {content.get('reasoning', '')}"],
+    )