ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/features_generator.py

@@ -0,0 +1,282 @@
+"""REF Features Generator.
+
+Infers FEATURES.md from codemap/ directory structure.
+Features are user-visible capabilities + modular structures.
+
+Usage:
+    from ref_agents.tools.features_generator import generate_features_file
+    result = generate_features_file("/path/to/project")
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# Keywords indicating user-facing features
+FEATURE_KEYWORDS = {
+    "api",
+    "auth",
+    "authentication",
+    "authorization",
+    "cache",
+    "client",
+    "config",
+    "connector",
+    "consumer",
+    "core",
+    "database",
+    "discovery",
+    "export",
+    "handler",
+    "import",
+    "index",
+    "indexing",
+    "integration",
+    "loader",
+    "manager",
+    "metrics",
+    "migration",
+    "notification",
+    "orchestrator",
+    "parser",
+    "pipeline",
+    "processor",
+    "provider",
+    "queue",
+    "report",
+    "scheduler",
+    "search",
+    "security",
+    "service",
+    "storage",
+    "sync",
+    "telemetry",
+    "transform",
+    "upload",
+    "validation",
+    "worker",
+}
+
+
+@dataclass
+class Feature:
+    """Inferred feature from codemap."""
+
+    name: str
+    module: str
+    description: str
+    category: str
+
+
+def _extract_purpose(content: str) -> str:
+    """Extract purpose section from codemap content.
+
+    Args:
+        content: Codemap markdown content.
+
+    Returns:
+        Purpose section text or empty string.
+    """
+    lines = content.splitlines()
+    in_purpose = False
+    purpose_lines: list[str] = []
+
+    for line in lines:
+        if line.startswith("## Purpose"):
+            in_purpose = True
+            continue
+        if in_purpose:
+            if line.startswith("## "):
+                break
+            if line.strip() and not line.startswith("<!--"):
+                purpose_lines.append(line.strip())
+
+    return " ".join(purpose_lines)
+
+
+def _infer_category(module_name: str) -> str:
+    """Infer feature category from module name.
+
+    Args:
+        module_name: Module/package name.
+
+    Returns:
+        Category string.
+    """
+    name_lower = module_name.lower()
+
+    if any(k in name_lower for k in ["api", "client", "connector"]):
+        return "Integration"
+    if any(k in name_lower for k in ["auth", "security"]):
+        return "Security"
+    if any(k in name_lower for k in ["index", "search", "discovery"]):
+        return "Data Processing"
+    if any(k in name_lower for k in ["config", "core", "shared"]):
+        return "Core"
+    if any(k in name_lower for k in ["metrics", "telemetry", "report"]):
+        return "Observability"
+    if any(k in name_lower for k in ["queue", "consumer", "worker"]):
+        return "Messaging"
+
+    return "Utility"
+
+
+def _format_feature_name(module_name: str) -> str:
+    """Convert module name to readable feature name.
+
+    Args:
+        module_name: Underscore-separated module name.
+
+    Returns:
+        Title-cased feature name.
+    """
+    # Remove common prefixes/suffixes
+    name = module_name.replace("_service", "").replace("_handler", "")
+    name = name.replace("_manager", "").replace("_processor", "")
+
+    # Title case
+    words = name.replace("_", " ").split()
+    return " ".join(word.capitalize() for word in words)
+
+
+def infer_features(codemap_dir: Path) -> list[Feature]:
+    """Infer features from codemap directory.
+
+    Args:
+        codemap_dir: Path to codemap/ directory.
+
+    Returns:
+        List of inferred features.
+    """
+    features: list[Feature] = []
+
+    if not codemap_dir.exists():
+        return features
+
+    for map_file in codemap_dir.glob("*.md"):
+        if map_file.name == "CODE_MAP.md":
+            continue  # Skip meta-map
+
+        try:
+            content = map_file.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+
+        # Module name from filename (e.g., app_discovery.md -> discovery)
+        module_name = map_file.stem
+        if "_" in module_name:
+            # Take last component (e.g., app_discovery -> discovery)
+            parts = module_name.split("_")
+            module_name = parts[-1] if len(parts) > 1 else module_name
+
+        # Check if module name suggests a feature
+        is_feature = any(kw in module_name.lower() for kw in FEATURE_KEYWORDS)
+
+        if is_feature:
+            purpose = _extract_purpose(content)
+            if not purpose or purpose.startswith("["):
+                purpose = (
+                    f"Handles {_format_feature_name(module_name).lower()} functionality"
+                )
+
+            features.append(
+                Feature(
+                    name=_format_feature_name(module_name),
+                    module=map_file.stem,
+                    description=purpose,
+                    category=_infer_category(module_name),
+                )
+            )
+
+    # Sort by category then name
+    return sorted(features, key=lambda f: (f.category, f.name))
+
+
+def generate_features_md(features: list[Feature]) -> str:
+    """Generate FEATURES.md content.
+
+    Args:
+        features: List of features.
+
+    Returns:
+        Markdown content.
+    """
+    content = """# FEATURES.md
+
+## Overview
+
+This document lists user-facing features inferred from the codebase structure.
+
+> **Note:** Features are identified by analyzing module names and purposes in `codemap/`.
+> Update module purposes to refine this list.
+
+## Features by Category
+
+"""
+
+    # Group by category
+    categories: dict[str, list[Feature]] = {}
+    for feature in features:
+        if feature.category not in categories:
+            categories[feature.category] = []
+        categories[feature.category].append(feature)
+
+    for category in sorted(categories.keys()):
+        content += f"### {category}\n\n"
+        content += "| Feature | Module | Description |\n"
+        content += "|---------|--------|-------------|\n"
+        for f in categories[category]:
+            content += f"| **{f.name}** | `{f.module}` | {f.description} |\n"
+        content += "\n"
+
+    content += f"""## Summary
+
+- **Total Features:** {len(features)}
+- **Categories:** {len(categories)}
+
+---
+*Auto-generated by discovery agent. Edit module purposes in `codemap/` to refine.*
+"""
+
+    return content
+
+
+def generate_features_file(directory: str) -> str:
+    """Generate FEATURES.md at project root.
+
+    Args:
+        directory: Path to project root.
+
+    Returns:
+        Status message.
+    """
+    root = Path(directory)
+    codemap_dir = root / "codemap"
+
+    if not codemap_dir.exists():
+        return "Error: codemap/ directory not found. Run generate_codemaps first."
+
+    features = infer_features(codemap_dir)
+
+    if not features:
+        return (
+            "No features inferred. Ensure codemap/ contains module maps with purposes."
+        )
+
+    content = generate_features_md(features)
+
+    features_path = root / "FEATURES.md"
+    features_path.write_text(content, encoding="utf-8")
+
+    logger.info(
+        "features_generated", path=str(features_path), feature_count=len(features)
+    )
+
+    return (
+        f"✅ FEATURES.md generated at `{features_path}`\n\n"
+        f"**Features found:** {len(features)}\n"
+        f"**Categories:** {len({f.category for f in features})}"
+    )

ref_agents/tools/flow_gap_detector.py

@@ -0,0 +1,373 @@
+"""Flow Gap Detector for REF Agents.
+
+Compares FLOWS.md definitions against REQ.md stories
+to detect missing steps, orphan stories, and broken links.
+"""
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import structlog
+
+from ref_agents.constants import Severity, Status
+from ref_agents.tools.flow_mapper import scan_requirements_for_flows
+
+logger = structlog.get_logger(__name__)
+
+# Regex for parsing FLOWS.md step tables
+# Format: | Step | Name | Description | Story ID | Status |
+STEP_TABLE_ROW = re.compile(
+    r"\|\s*(\d+)\s*\|\s*([^|]+)\s*\|\s*([^|]+)\s*\|\s*([^|]+)\s*\|\s*([^|]+)\s*\|"
+)
+FLOW_HEADER = re.compile(r"##\s*Flow:\s*(\S+)")
+PROCESS_LINE = re.compile(r"\*\*Process:\*\*\s*(.+)")
+
+
+@dataclass
+class FlowDefinition:
+    """Flow definition from FLOWS.md."""
+
+    name: str
+    process: str = ""
+    steps: dict[str, dict[str, str]] = field(default_factory=dict)
+
+    def add_step(  # noqa: PLR0913  # TECH_DEBT: refactor to TypedDict — STORY-046
+        self,
+        order: int,
+        name: str,
+        description: str,
+        story_id: str,
+        status: str,
+    ) -> None:
+        """Add a step to the flow definition."""
+        self.steps[name.strip()] = {
+            "order": str(order),
+            "name": name.strip(),
+            "description": description.strip(),
+            "story_id": story_id.strip(),
+            "status": status.strip(),
+        }
+
+
+@dataclass
+class Gap:
+    """Represents a detected gap."""
+
+    gap_type: str  # missing_story, orphan_story, broken_link, status_mismatch
+    severity: str  # high, medium, low
+    flow: str
+    step: str
+    description: str
+    story_id: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        """Convert to dictionary."""
+        return {
+            "type": self.gap_type,
+            "severity": self.severity,
+            "flow": self.flow,
+            "step": self.step,
+            "description": self.description,
+            "story_id": self.story_id,
+        }
+
+
+@dataclass
+class GapReport:
+    """Collection of detected gaps."""
+
+    gaps: list[Gap] = field(default_factory=list)
+    flows_checked: int = 0
+    steps_checked: int = 0
+    stories_checked: int = 0
+
+    def add_gap(self, gap: Gap) -> None:
+        """Add a gap to the report."""
+        self.gaps.append(gap)
+
+    @property
+    def high_severity_count(self) -> int:
+        """Count high severity gaps."""
+        return sum(1 for g in self.gaps if g.severity == "high")
+
+    @property
+    def medium_severity_count(self) -> int:
+        """Count medium severity gaps."""
+        return sum(1 for g in self.gaps if g.severity == "medium")
+
+    @property
+    def low_severity_count(self) -> int:
+        """Count low severity gaps."""
+        return sum(1 for g in self.gaps if g.severity == "low")
+
+    def to_markdown(self) -> str:
+        """Generate markdown report."""
+        lines = [
+            "# Flow Gap Analysis Report",
+            "",
+            "*Auto-generated by flow_gap_detector.py*",
+            "",
+            "## Summary",
+            "",
+            f"- Flows checked: {self.flows_checked}",
+            f"- Steps checked: {self.steps_checked}",
+            f"- Stories checked: {self.stories_checked}",
+            f"- **Total gaps: {len(self.gaps)}**",
+            f"  - {Severity.HIGH_ICON} High: {self.high_severity_count}",
+            f"  - {Severity.MEDIUM_ICON} Medium: {self.medium_severity_count}",
+            f"  - {Severity.LOW_ICON} Low: {self.low_severity_count}",
+            "",
+        ]
+
+        if not self.gaps:
+            lines.append(
+                f"{Status.SUCCESS} **No gaps detected. All flows are complete.**"
+            )
+            return "\n".join(lines)
+
+        # Group by severity
+        for severity, icon in [
+            ("high", Severity.HIGH_ICON),
+            ("medium", Severity.MEDIUM_ICON),
+            ("low", Severity.LOW_ICON),
+        ]:
+            severity_gaps = [g for g in self.gaps if g.severity == severity]
+            if severity_gaps:
+                lines.append(f"## {icon} {severity.title()} Severity Gaps")
+                lines.append("")
+                lines.append("| Flow | Step | Type | Description |")
+                lines.append("|------|------|------|-------------|")
+                for gap in severity_gaps:
+                    lines.append(
+                        f"| {gap.flow} | {gap.step} | {gap.gap_type} | {gap.description} |"
+                    )
+                lines.append("")
+
+        return "\n".join(lines)
+
+
+def parse_flows_md(file_path: Path) -> dict[str, FlowDefinition]:
+    """Parse FLOWS.md file to extract flow definitions.
+
+    Args:
+        file_path: Path to FLOWS.md file.
+
+    Returns:
+        Dict mapping flow name to FlowDefinition.
+    """
+    flows: dict[str, FlowDefinition] = {}
+
+    try:
+        content = file_path.read_text(encoding="utf-8")
+    except OSError as e:
+        logger.warning("flows_file_read_error", file=str(file_path), error=str(e))
+        return flows
+
+    current_flow: FlowDefinition | None = None
+    lines = content.split("\n")
+
+    for i, line in enumerate(lines):
+        # Check for flow header
+        flow_match = FLOW_HEADER.match(line)
+        if flow_match:
+            flow_name = flow_match.group(1)
+            current_flow = FlowDefinition(name=flow_name)
+            flows[flow_name] = current_flow
+            continue
+
+        # Check for process line
+        if current_flow:
+            process_match = PROCESS_LINE.match(line)
+            if process_match:
+                current_flow.process = process_match.group(1).strip()
+                continue
+
+        # Check for step table row
+        if current_flow:
+            step_match = STEP_TABLE_ROW.match(line)
+            if step_match:
+                order = int(step_match.group(1))
+                name = step_match.group(2)
+                description = step_match.group(3)
+                story_id = step_match.group(4)
+                status = step_match.group(5)
+
+                # Skip header row
+                if name.strip().lower() == "name":
+                    continue
+
+                current_flow.add_step(order, name, description, story_id, status)
+
+    logger.info("flows_md_parsed", flows_found=len(flows))
+    return flows
+
+
+def detect_gaps(
+    flows_file: str | Path,
+    req_directory: str | Path,
+) -> GapReport:
+    """Detect gaps between FLOWS.md and REQ.md files.
+
+    Args:
+        flows_file: Path to FLOWS.md file.
+        req_directory: Path to requirements directory.
+
+    Returns:
+        GapReport with all detected gaps.
+    """
+    report = GapReport()
+
+    # Parse FLOWS.md
+    flows_path = Path(flows_file)
+    if not flows_path.exists():
+        report.gaps.append(
+            Gap(
+                gap_type="config_error",
+                severity="high",
+                flow="-",
+                step="-",
+                description=f"FLOWS.md not found: {flows_file}",
+            )
+        )
+        return report
+
+    expected_flows = parse_flows_md(flows_path)
+    report.flows_checked = len(expected_flows)
+
+    # Scan REQ.md files for actual flow context
+    actual_flow_map = scan_requirements_for_flows(req_directory)
+    report.stories_checked = len(actual_flow_map.orphan_stories) + sum(
+        len(f.steps) for f in actual_flow_map.flows.values()
+    )
+
+    # Check each expected flow
+    for flow_name, expected_flow in expected_flows.items():
+        actual_flow = actual_flow_map.flows.get(flow_name)
+        report.steps_checked += len(expected_flow.steps)
+
+        for step_name, expected_step in expected_flow.steps.items():
+            story_id = expected_step["story_id"]
+
+            # Gap 1: Missing story (step defined but no story linked)
+            if story_id in ("-", "", "N/A", "Not Started"):
+                report.add_gap(
+                    Gap(
+                        gap_type="missing_story",
+                        severity="high",
+                        flow=flow_name,
+                        step=step_name,
+                        description=f"Flow step '{step_name}' has no linked story",
+                    )
+                )
+                continue
+
+            # Gap 2: Story exists in FLOWS.md but not in REQ.md
+            if actual_flow is None or step_name not in actual_flow.steps:
+                report.add_gap(
+                    Gap(
+                        gap_type="missing_implementation",
+                        severity="medium",
+                        flow=flow_name,
+                        step=step_name,
+                        story_id=story_id,
+                        description=f"Story {story_id} not found in requirements or missing flow context",
+                    )
+                )
+                continue
+
+            # Gap 3: Status mismatch
+            actual_step = actual_flow.steps[step_name]
+            expected_status = expected_step["status"].lower()
+            actual_status = actual_step.status.lower()
+
+            if expected_status != actual_status:
+                report.add_gap(
+                    Gap(
+                        gap_type="status_mismatch",
+                        severity="low",
+                        flow=flow_name,
+                        step=step_name,
+                        story_id=story_id,
+                        description=f"Status mismatch: FLOWS.md says '{expected_step['status']}', story says '{actual_step.status}'",
+                    )
+                )
+
+    # Check for orphan flow steps (in REQ.md but not in FLOWS.md)
+    for flow_name, actual_flow in actual_flow_map.flows.items():
+        expected_flow = expected_flows.get(flow_name)
+
+        if expected_flow is None:
+            # Entire flow not in FLOWS.md
+            for step_name, step in actual_flow.steps.items():
+                report.add_gap(
+                    Gap(
+                        gap_type="orphan_flow",
+                        severity="medium",
+                        flow=flow_name,
+                        step=step_name,
+                        story_id=step.story_id,
+                        description=f"Flow '{flow_name}' not defined in FLOWS.md",
+                    )
+                )
+        else:
+            # Check individual steps
+            for step_name, step in actual_flow.steps.items():
+                if step_name not in expected_flow.steps:
+                    report.add_gap(
+                        Gap(
+                            gap_type="orphan_step",
+                            severity="low",
+                            flow=flow_name,
+                            step=step_name,
+                            story_id=step.story_id,
+                            description=f"Step '{step_name}' in story but not in FLOWS.md",
+                        )
+                    )
+
+    logger.info(
+        "gap_detection_complete",
+        gaps_found=len(report.gaps),
+        high=report.high_severity_count,
+        medium=report.medium_severity_count,
+        low=report.low_severity_count,
+    )
+
+    return report
+
+
+def detect_flow_gaps(flows_file: str, req_directory: str, summary: bool = False) -> str:
+    """Detect and report flow gaps.
+
+    Args:
+        flows_file: Path to FLOWS.md file.
+        req_directory: Path to requirements directory.
+        summary: If True, return brief summary. If False, return full report.
+
+    Returns:
+        Markdown report or brief summary of detected gaps.
+    """
+    report = detect_gaps(flows_file, req_directory)
+
+    # Summary mode
+    if summary:
+        if not report.gaps:
+            return f"{Status.SUCCESS} No flow gaps detected."
+
+        return (
+            f"{Status.WARNING} {len(report.gaps)} gaps found: "
+            f"{Severity.HIGH_ICON} {report.high_severity_count} high, "
+            f"{Severity.MEDIUM_ICON} {report.medium_severity_count} medium, "
+            f"{Severity.LOW_ICON} {report.low_severity_count} low"
+        )
+
+    # Full report mode
+    markdown = report.to_markdown()
+
+    # Write artifact
+    output_path = Path(req_directory) / "FLOW_GAP_REPORT.md"
+    try:
+        output_path.write_text(markdown, encoding="utf-8")
+        return f"Generated: `{output_path}`\n\n{markdown}"
+    except OSError as e:
+        return f"Failed to write artifact: {e}\n\n{markdown}"