ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/cross_repo_linker.py

@@ -0,0 +1,393 @@
+"""Cross-Repo Linker for REF Agents.
+
+Compares EXTERNAL_APIS.md registry against detected dependencies
+to find unregistered deps and stale registry entries.
+"""
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import structlog
+
+from ref_agents.tools.external_detector import scan_directory
+
+logger = structlog.get_logger(__name__)
+
+# Regex patterns for parsing EXTERNAL_APIS.md tables
+ENV_VAR_ROW = re.compile(r"\|\s*`([A-Z_]+)`\s*\|")
+LIBRARY_ROW = re.compile(r"\|\s*`([a-z0-9_-]+)`\s*\|")
+DOMAIN_ROW = re.compile(r"\|\s*`?([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})`?\s*\|")
+
+# Section headers
+ENV_SECTION = re.compile(r"###?\s*Environment Variables", re.IGNORECASE)
+LIB_SECTION = re.compile(r"###?\s*API Clients|Libraries", re.IGNORECASE)
+URL_SECTION = re.compile(r"###?\s*External URLs", re.IGNORECASE)
+
+
+@dataclass
+class RegisteredDependency:
+    """A dependency registered in EXTERNAL_APIS.md."""
+
+    name: str
+    dep_type: str  # env_var, api_client, url
+    purpose: str = ""
+    contract: str = ""
+
+
+@dataclass
+class DependencyGap:
+    """A gap between registry and detected dependencies."""
+
+    gap_type: str  # unregistered, stale
+    dep_type: str  # env_var, api_client, url
+    name: str
+    location: str = ""  # file:line for unregistered
+    notes: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        """Convert to dictionary."""
+        return {
+            "gap_type": self.gap_type,
+            "dep_type": self.dep_type,
+            "name": self.name,
+            "location": self.location,
+            "notes": self.notes,
+        }
+
+
+@dataclass
+class LinkReport:
+    """Result of cross-repo linking analysis."""
+
+    matched: list[str] = field(default_factory=list)
+    unregistered: list[DependencyGap] = field(default_factory=list)
+    stale: list[DependencyGap] = field(default_factory=list)
+    registry_count: int = 0
+    detected_count: int = 0
+
+    @property
+    def total_gaps(self) -> int:
+        """Total number of gaps."""
+        return len(self.unregistered) + len(self.stale)
+
+    def to_markdown(self) -> str:
+        """Generate markdown report."""
+        lines = [
+            "# External Dependency Link Report",
+            "",
+            "*Auto-generated by cross_repo_linker.py*",
+            "",
+            "## Summary",
+            "",
+            f"- Registered in EXTERNAL_APIS.md: {self.registry_count}",
+            f"- Detected in codebase: {self.detected_count}",
+            f"- Matched: {len(self.matched)}",
+            f"- **Unregistered: {len(self.unregistered)}**",
+            f"- **Stale entries: {len(self.stale)}**",
+            "",
+        ]
+
+        if not self.unregistered and not self.stale:
+            lines.append("✅ **All dependencies are registered and accounted for.**")
+            return "\n".join(lines)
+
+        if self.unregistered:
+            lines.append("## ⚠️ Unregistered Dependencies")
+            lines.append("")
+            lines.append("These were detected in code but not in EXTERNAL_APIS.md:")
+            lines.append("")
+            lines.append("| Type | Name | Location | Action |")
+            lines.append("|------|------|----------|--------|")
+            for gap in self.unregistered:
+                lines.append(
+                    f"| {gap.dep_type} | `{gap.name}` | {gap.location} | Add to registry |"
+                )
+            lines.append("")
+
+        if self.stale:
+            lines.append("## 🗑️ Stale Registry Entries")
+            lines.append("")
+            lines.append("These are in EXTERNAL_APIS.md but not detected in code:")
+            lines.append("")
+            lines.append("| Type | Name | Action |")
+            lines.append("|------|------|--------|")
+            for gap in self.stale:
+                lines.append(
+                    f"| {gap.dep_type} | `{gap.name}` | Remove or verify needed |"
+                )
+            lines.append("")
+
+        if self.matched:
+            lines.append("## ✅ Matched Dependencies")
+            lines.append("")
+            lines.append("These are registered and detected:")
+            lines.append("")
+            for name in self.matched:
+                lines.append(f"- `{name}`")
+            lines.append("")
+
+        return "\n".join(lines)
+
+
+def _detect_section(line: str) -> str | None:
+    """Detect if line marks a new section."""
+    if ENV_SECTION.search(line):
+        return "env_vars"
+    if LIB_SECTION.search(line):
+        return "api_clients"
+    if URL_SECTION.search(line):
+        return "urls"
+    return None
+
+
+def _extract_registry_entry(
+    line: str, current_section: str, registry: dict[str, set[str]]
+) -> None:
+    """Extract entry from table line into registry."""
+    # Skip non-table lines
+    if not line.strip().startswith("|"):
+        return
+
+    # Skip header/separator rows
+    if "---" in line or "Env Var" in line or "Library" in line or "Domain" in line:
+        return
+
+    if current_section == "env_vars":
+        match = ENV_VAR_ROW.search(line)
+        if match:
+            registry["env_vars"].add(match.group(1))
+    elif current_section == "api_clients":
+        match = LIBRARY_ROW.search(line)
+        if match:
+            registry["api_clients"].add(match.group(1))
+    elif current_section == "urls":
+        match = DOMAIN_ROW.search(line)
+        if match:
+            registry["urls"].add(match.group(1).lower())
+
+
+def parse_registry(file_path: Path) -> dict[str, set[str]]:
+    """Parse EXTERNAL_APIS.md to extract registered dependencies.
+
+    Args:
+        file_path: Path to EXTERNAL_APIS.md file.
+
+    Returns:
+        Dict with keys 'env_vars', 'api_clients', 'urls' mapping to sets of names.
+    """
+    registry: dict[str, set[str]] = {
+        "env_vars": set(),
+        "api_clients": set(),
+        "urls": set(),
+    }
+
+    try:
+        content = file_path.read_text(encoding="utf-8")
+    except OSError as e:
+        logger.warning("registry_read_error", file=str(file_path), error=str(e))
+        return registry
+
+    lines = content.split("\n")
+    current_section = None
+
+    for line in lines:
+        # Detect section changes
+        new_section = _detect_section(line)
+        if new_section:
+            current_section = new_section
+            continue
+
+        if current_section:
+            _extract_registry_entry(line, current_section, registry)
+
+    logger.info(
+        "registry_parsed",
+        env_vars=len(registry["env_vars"]),
+        api_clients=len(registry["api_clients"]),
+        urls=len(registry["urls"]),
+    )
+
+    return registry
+
+
+def _match_dependency_group(
+    registry_set: set[str],
+    detected_map: dict[str, str],
+    dep_type: str,
+    report: LinkReport,
+) -> None:
+    """Match a group of dependencies against registry.
+
+    Args:
+        registry_set: Set of registered names.
+        detected_map: Map of detected names to locations.
+        dep_type: Type of dependency (e.g., 'env_var').
+        report: Report to update.
+    """
+    # Match registered items
+    for name in registry_set:
+        if name in detected_map:
+            report.matched.append(name)
+        else:
+            report.stale.append(
+                DependencyGap(
+                    gap_type="stale",
+                    dep_type=dep_type,
+                    name=name,
+                    notes="In registry but not detected in code",
+                )
+            )
+
+    # Find unregistered items
+    for name, location in detected_map.items():
+        if name not in registry_set:
+            report.unregistered.append(
+                DependencyGap(
+                    gap_type="unregistered",
+                    dep_type=dep_type,
+                    name=name,
+                    location=location,
+                )
+            )
+
+
+def link_dependencies(
+    registry_file: str | Path,
+    code_directory: str | Path,
+) -> LinkReport:
+    """Compare registry against detected dependencies.
+
+    Args:
+        registry_file: Path to EXTERNAL_APIS.md.
+        code_directory: Directory to scan for dependencies.
+
+    Returns:
+        LinkReport with matched, unregistered, and stale entries.
+    """
+    report = LinkReport()
+
+    # Parse registry
+    registry_path = Path(registry_file)
+    if not registry_path.exists():
+        logger.warning("registry_not_found", file=str(registry_file))
+        # Scan anyway to report all as unregistered
+        registry: dict[str, set[str]] = {
+            "env_vars": set(),
+            "api_clients": set(),
+            "urls": set(),
+        }
+    else:
+        registry = parse_registry(registry_path)
+
+    report.registry_count = sum(len(v) for v in registry.values())
+
+    # Scan codebase
+    scan_result = scan_directory(code_directory)
+    report.detected_count = len(scan_result.dependencies)
+
+    # Build sets of detected items
+    detected_env_vars: dict[str, str] = {}
+    detected_clients: dict[str, str] = {}
+    detected_urls: dict[str, str] = {}
+
+    for dep in scan_result.dependencies:
+        location = f"{Path(dep.source_file).name}:{dep.line_number or '?'}"
+        if dep.dep_type == "env_var":
+            detected_env_vars[dep.name] = location
+        elif dep.dep_type == "api_client":
+            detected_clients[dep.name] = location
+        elif dep.dep_type == "url":
+            # Normalize URL to domain
+            detected_urls[dep.name.lower()] = location
+
+    # Match groups
+    _match_dependency_group(registry["env_vars"], detected_env_vars, "env_var", report)
+    _match_dependency_group(
+        registry["api_clients"], detected_clients, "api_client", report
+    )
+    _match_dependency_group(registry["urls"], detected_urls, "url", report)
+
+    logger.info(
+        "link_complete",
+        matched=len(report.matched),
+        unregistered=len(report.unregistered),
+        stale=len(report.stale),
+    )
+
+    return report
+
+
+def validate_external_deps(registry_file: str, code_directory: str) -> str:
+    """Compare EXTERNAL_APIS.md against detected dependencies.
+
+    Args:
+        registry_file: Path to EXTERNAL_APIS.md.
+        code_directory: Directory to scan.
+
+    Returns:
+        Markdown report with gaps.
+    """
+    report = link_dependencies(registry_file, code_directory)
+    markdown = report.to_markdown()
+
+    # Write artifact
+    output_path = Path(code_directory) / "EXTERNAL_DEPS_REPORT.md"
+    try:
+        output_path.write_text(markdown, encoding="utf-8")
+        return f"Generated: `{output_path}`\n\n{markdown}"
+    except OSError as e:
+        return f"Failed to write artifact: {e}\n\n{markdown}"
+
+
+def get_unregistered_deps(code_directory: str) -> str:
+    """Find external dependencies not in any registry.
+
+    Scans code without requiring EXTERNAL_APIS.md.
+
+    Args:
+        code_directory: Directory to scan.
+
+    Returns:
+        Summary of detected dependencies.
+    """
+    scan_result = scan_directory(code_directory)
+
+    if not scan_result.dependencies:
+        return "No external dependencies detected."
+
+    lines = [
+        "# Detected External Dependencies",
+        "",
+        f"*Scanned {scan_result.files_scanned} files*",
+        "",
+    ]
+
+    if scan_result.env_vars:
+        lines.append("## Environment Variables")
+        lines.append("")
+        for dep in scan_result.env_vars:
+            lines.append(
+                f"- `{dep.name}` ({Path(dep.source_file).name}:{dep.line_number})"
+            )
+        lines.append("")
+
+    if scan_result.api_clients:
+        lines.append("## API Clients")
+        lines.append("")
+        for dep in scan_result.api_clients:
+            lines.append(f"- `{dep.name}` — {dep.description}")
+        lines.append("")
+
+    if scan_result.urls:
+        lines.append("## External URLs")
+        lines.append("")
+        for dep in scan_result.urls:
+            lines.append(f"- `{dep.name}` ({Path(dep.source_file).name})")
+        lines.append("")
+
+    lines.append("---")
+    lines.append("")
+    lines.append("To register these, add them to `docs/templates/EXTERNAL_APIS.md`.")
+
+    return "\n".join(lines)