skill-seekers 2.7.3__py3-none-any.whl

skill_seekers/cli/quality_checker.py

@@ -0,0 +1,519 @@
+#!/usr/bin/env python3
+"""
+Quality Checker for Claude Skills
+Validates skill quality, checks links, and generates quality reports.
+
+Usage:
+    python3 quality_checker.py output/react/
+    python3 quality_checker.py output/godot/ --verbose
+"""
+
+import re
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class QualityIssue:
+    """Represents a quality issue found during validation."""
+
+    level: str  # 'error', 'warning', 'info'
+    category: str  # 'enhancement', 'content', 'links', 'structure'
+    message: str
+    file: str | None = None
+    line: int | None = None
+
+
+@dataclass
+class QualityReport:
+    """Complete quality report for a skill."""
+
+    skill_name: str
+    skill_path: Path
+    errors: list[QualityIssue] = field(default_factory=list)
+    warnings: list[QualityIssue] = field(default_factory=list)
+    info: list[QualityIssue] = field(default_factory=list)
+
+    def add_error(self, category: str, message: str, file: str = None, line: int = None):
+        """Add an error to the report."""
+        self.errors.append(QualityIssue("error", category, message, file, line))
+
+    def add_warning(self, category: str, message: str, file: str = None, line: int = None):
+        """Add a warning to the report."""
+        self.warnings.append(QualityIssue("warning", category, message, file, line))
+
+    def add_info(self, category: str, message: str, file: str = None, line: int = None):
+        """Add info to the report."""
+        self.info.append(QualityIssue("info", category, message, file, line))
+
+    @property
+    def has_errors(self) -> bool:
+        """Check if there are any errors."""
+        return len(self.errors) > 0
+
+    @property
+    def has_warnings(self) -> bool:
+        """Check if there are any warnings."""
+        return len(self.warnings) > 0
+
+    @property
+    def is_excellent(self) -> bool:
+        """Check if quality is excellent (no errors, no warnings)."""
+        return not self.has_errors and not self.has_warnings
+
+    @property
+    def quality_score(self) -> float:
+        """Calculate quality score (0-100)."""
+        # Start with perfect score
+        score = 100.0
+
+        # Deduct points for issues
+        score -= len(self.errors) * 15  # -15 per error
+        score -= len(self.warnings) * 5  # -5 per warning
+
+        # Never go below 0
+        return max(0.0, score)
+
+    @property
+    def quality_grade(self) -> str:
+        """Get quality grade (A-F)."""
+        score = self.quality_score
+        if score >= 90:
+            return "A"
+        elif score >= 80:
+            return "B"
+        elif score >= 70:
+            return "C"
+        elif score >= 60:
+            return "D"
+        else:
+            return "F"
+
+
+class SkillQualityChecker:
+    """Validates skill quality and generates reports."""
+
+    def __init__(self, skill_dir: Path):
+        """Initialize quality checker.
+
+        Args:
+            skill_dir: Path to skill directory
+        """
+        self.skill_dir = Path(skill_dir)
+        self.skill_md_path = self.skill_dir / "SKILL.md"
+        self.references_dir = self.skill_dir / "references"
+        self.report = QualityReport(skill_name=self.skill_dir.name, skill_path=self.skill_dir)
+
+    def check_all(self) -> QualityReport:
+        """Run all quality checks and return report.
+
+        Returns:
+            QualityReport: Complete quality report
+        """
+        # Basic structure checks
+        self._check_skill_structure()
+
+        # Enhancement verification
+        self._check_enhancement_quality()
+
+        # Content quality checks
+        self._check_content_quality()
+
+        # Link validation
+        self._check_links()
+
+        # Completeness checks
+        self._check_skill_completeness()
+
+        return self.report
+
+    def _check_skill_structure(self):
+        """Check basic skill structure."""
+        # Check SKILL.md exists
+        if not self.skill_md_path.exists():
+            self.report.add_error("structure", "SKILL.md file not found", str(self.skill_md_path))
+            return
+
+        # Check references directory exists
+        if not self.references_dir.exists():
+            self.report.add_warning(
+                "structure",
+                "references/ directory not found - skill may be incomplete",
+                str(self.references_dir),
+            )
+        elif not list(self.references_dir.rglob("*.md")):
+            self.report.add_warning(
+                "structure",
+                "references/ directory is empty - no reference documentation found",
+                str(self.references_dir),
+            )
+
+    def _check_enhancement_quality(self):
+        """Check if SKILL.md was properly enhanced."""
+        if not self.skill_md_path.exists():
+            return
+
+        content = self.skill_md_path.read_text(encoding="utf-8")
+
+        # Check for template indicators (signs it wasn't enhanced)
+        template_indicators = [
+            "TODO:",
+            "[Add description]",
+            "[Framework specific tips]",
+            "coming soon",
+        ]
+
+        for indicator in template_indicators:
+            if indicator.lower() in content.lower():
+                self.report.add_warning(
+                    "enhancement",
+                    f'Found template placeholder: "{indicator}" - SKILL.md may not be enhanced',
+                    "SKILL.md",
+                )
+
+        # Check for good signs of enhancement
+        enhancement_indicators = {
+            "code_examples": re.compile(r"```[\w-]+\n", re.MULTILINE),
+            "real_examples": re.compile(r"Example:", re.IGNORECASE),
+            "sections": re.compile(r"^## .+", re.MULTILINE),
+        }
+
+        code_blocks = len(enhancement_indicators["code_examples"].findall(content))
+        _real_examples = len(enhancement_indicators["real_examples"].findall(content))
+        sections = len(enhancement_indicators["sections"].findall(content))
+
+        # Quality thresholds
+        if code_blocks == 0:
+            self.report.add_warning(
+                "enhancement", "No code examples found in SKILL.md - consider enhancing", "SKILL.md"
+            )
+        elif code_blocks < 3:
+            self.report.add_info(
+                "enhancement",
+                f"Only {code_blocks} code examples found - more examples would improve quality",
+                "SKILL.md",
+            )
+        else:
+            self.report.add_info("enhancement", f"✓ Found {code_blocks} code examples", "SKILL.md")
+
+        if sections < 4:
+            self.report.add_warning(
+                "enhancement",
+                f"Only {sections} sections found - SKILL.md may be too basic",
+                "SKILL.md",
+            )
+        else:
+            self.report.add_info("enhancement", f"✓ Found {sections} sections", "SKILL.md")
+
+    def _check_content_quality(self):
+        """Check content quality."""
+        if not self.skill_md_path.exists():
+            return
+
+        content = self.skill_md_path.read_text(encoding="utf-8")
+
+        # Check YAML frontmatter
+        if not content.startswith("---"):
+            self.report.add_error(
+                "content", "Missing YAML frontmatter - SKILL.md must start with ---", "SKILL.md", 1
+            )
+        else:
+            # Extract frontmatter
+            try:
+                frontmatter_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
+                if frontmatter_match:
+                    frontmatter = frontmatter_match.group(1)
+
+                    # Check for required fields
+                    if "name:" not in frontmatter:
+                        self.report.add_error(
+                            "content", 'Missing "name:" field in YAML frontmatter', "SKILL.md", 2
+                        )
+
+                    # Check for description
+                    if "description:" in frontmatter:
+                        self.report.add_info(
+                            "content", "✓ YAML frontmatter includes description", "SKILL.md"
+                        )
+                else:
+                    self.report.add_error(
+                        "content", "Invalid YAML frontmatter format", "SKILL.md", 1
+                    )
+            except Exception as e:
+                self.report.add_error(
+                    "content", f"Error parsing YAML frontmatter: {e}", "SKILL.md", 1
+                )
+
+        # Check code block language tags
+        code_blocks_without_lang = re.findall(r"```\n[^`]", content)
+        if code_blocks_without_lang:
+            self.report.add_warning(
+                "content",
+                f"Found {len(code_blocks_without_lang)} code blocks without language tags",
+                "SKILL.md",
+            )
+
+        # Check for "When to Use" section
+        if "when to use" not in content.lower():
+            self.report.add_warning(
+                "content", 'Missing "When to Use This Skill" section', "SKILL.md"
+            )
+        else:
+            self.report.add_info("content", '✓ Found "When to Use" section', "SKILL.md")
+
+        # Check reference files
+        if self.references_dir.exists():
+            ref_files = list(self.references_dir.rglob("*.md"))
+            if ref_files:
+                self.report.add_info(
+                    "content", f"✓ Found {len(ref_files)} reference files", "references/"
+                )
+
+                # Check if references are mentioned in SKILL.md
+                mentioned_refs = 0
+                for ref_file in ref_files:
+                    if ref_file.name in content:
+                        mentioned_refs += 1
+
+                if mentioned_refs == 0:
+                    self.report.add_warning(
+                        "content",
+                        "Reference files exist but none are mentioned in SKILL.md",
+                        "SKILL.md",
+                    )
+
+    def _check_links(self):
+        """Check internal markdown links."""
+        if not self.skill_md_path.exists():
+            return
+
+        content = self.skill_md_path.read_text(encoding="utf-8")
+
+        # Find all markdown links [text](path)
+        link_pattern = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
+        links = link_pattern.findall(content)
+
+        broken_links = []
+
+        for text, link in links:
+            # Skip external links (http/https)
+            if link.startswith("http://") or link.startswith("https://"):
+                continue
+
+            # Skip anchor links
+            if link.startswith("#"):
+                continue
+
+            # Check if file exists (relative to SKILL.md)
+            link_path = self.skill_dir / link
+            if not link_path.exists():
+                broken_links.append((text, link))
+
+        if broken_links:
+            for text, link in broken_links:
+                self.report.add_warning("links", f"Broken link: [{text}]({link})", "SKILL.md")
+        else:
+            if links:
+                internal_links = [link for t, link in links if not link.startswith("http")]
+                if internal_links:
+                    self.report.add_info(
+                        "links", f"✓ All {len(internal_links)} internal links are valid", "SKILL.md"
+                    )
+
+    def _check_skill_completeness(self):
+        """Check skill completeness based on best practices.
+
+        Validates that skills include verification/prerequisites sections,
+        error handling guidance, and clear workflow steps.
+        """
+        if not self.skill_md_path.exists():
+            return
+
+        content = self.skill_md_path.read_text(encoding="utf-8")
+
+        # Check for grounding/verification section (prerequisites)
+        grounding_patterns = [
+            r"before\s+(executing|running|proceeding|you\s+start)",
+            r"verify\s+that",
+            r"prerequisites?",
+            r"requirements?:",
+            r"make\s+sure\s+you\s+have",
+        ]
+        has_grounding = any(
+            re.search(pattern, content, re.IGNORECASE) for pattern in grounding_patterns
+        )
+        if has_grounding:
+            self.report.add_info(
+                "completeness", "✓ Found verification/prerequisites section", "SKILL.md"
+            )
+        else:
+            self.report.add_info(
+                "completeness",
+                "Consider adding prerequisites section - helps Claude verify conditions first",
+                "SKILL.md",
+            )
+
+        # Check for error handling/troubleshooting guidance
+        error_patterns = [
+            r"if\s+.*\s+(fails?|errors?)",
+            r"troubleshoot",
+            r"common\s+(issues?|problems?)",
+            r"error\s+handling",
+            r"when\s+things\s+go\s+wrong",
+        ]
+        has_error_handling = any(
+            re.search(pattern, content, re.IGNORECASE) for pattern in error_patterns
+        )
+        if has_error_handling:
+            self.report.add_info(
+                "completeness", "✓ Found error handling/troubleshooting guidance", "SKILL.md"
+            )
+        else:
+            self.report.add_info(
+                "completeness",
+                "Consider adding troubleshooting section for common issues",
+                "SKILL.md",
+            )
+
+        # Check for workflow steps (numbered or sequential indicators)
+        step_patterns = [
+            r"step\s+\d",
+            r"##\s+\d\.",
+            r"first,?\s+",
+            r"then,?\s+",
+            r"finally,?\s+",
+            r"next,?\s+",
+        ]
+        steps_found = sum(
+            1 for pattern in step_patterns if re.search(pattern, content, re.IGNORECASE)
+        )
+        if steps_found >= 3:
+            self.report.add_info(
+                "completeness",
+                f"✓ Found clear workflow indicators ({steps_found} step markers)",
+                "SKILL.md",
+            )
+        elif steps_found > 0:
+            self.report.add_info(
+                "completeness",
+                f"Some workflow guidance found ({steps_found} markers) - consider adding numbered steps for clarity",
+                "SKILL.md",
+            )
+
+
+def print_report(report: QualityReport, verbose: bool = False):
+    """Print quality report to console.
+
+    Args:
+        report: Quality report to print
+        verbose: Show all info messages
+    """
+    print("\n" + "=" * 60)
+    print(f"QUALITY REPORT: {report.skill_name}")
+    print("=" * 60)
+    print()
+
+    # Quality score
+    print(f"Quality Score: {report.quality_score:.1f}/100 (Grade: {report.quality_grade})")
+    print()
+
+    # Errors
+    if report.errors:
+        print(f"❌ ERRORS ({len(report.errors)}):")
+        for issue in report.errors:
+            location = (
+                f" ({issue.file}:{issue.line})"
+                if issue.file and issue.line
+                else f" ({issue.file})"
+                if issue.file
+                else ""
+            )
+            print(f"   [{issue.category}] {issue.message}{location}")
+        print()
+
+    # Warnings
+    if report.warnings:
+        print(f"⚠️  WARNINGS ({len(report.warnings)}):")
+        for issue in report.warnings:
+            location = (
+                f" ({issue.file}:{issue.line})"
+                if issue.file and issue.line
+                else f" ({issue.file})"
+                if issue.file
+                else ""
+            )
+            print(f"   [{issue.category}] {issue.message}{location}")
+        print()
+
+    # Info (only in verbose mode)
+    if verbose and report.info:
+        print(f"ℹ️  INFO ({len(report.info)}):")
+        for issue in report.info:
+            location = f" ({issue.file})" if issue.file else ""
+            print(f"   [{issue.category}] {issue.message}{location}")
+        print()
+
+    # Summary
+    if report.is_excellent:
+        print("✅ EXCELLENT! No issues found.")
+    elif not report.has_errors:
+        print("✓ GOOD! No errors, but some warnings to review.")
+    else:
+        print("❌ NEEDS IMPROVEMENT! Please fix errors before packaging.")
+
+    print()
+
+
+def main():
+    """Main entry point."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Check skill quality and generate report",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Basic quality check
+  python3 quality_checker.py output/react/
+
+  # Verbose mode (show all info)
+  python3 quality_checker.py output/godot/ --verbose
+
+  # Exit with error code if issues found
+  python3 quality_checker.py output/django/ --strict
+""",
+    )
+
+    parser.add_argument("skill_directory", help="Path to skill directory (e.g., output/react/)")
+
+    parser.add_argument("--verbose", "-v", action="store_true", help="Show all info messages")
+
+    parser.add_argument(
+        "--strict", action="store_true", help="Exit with error code if any warnings or errors found"
+    )
+
+    args = parser.parse_args()
+
+    # Check if directory exists
+    skill_dir = Path(args.skill_directory)
+    if not skill_dir.exists():
+        print(f"❌ Directory not found: {skill_dir}")
+        sys.exit(1)
+
+    # Run quality checks
+    checker = SkillQualityChecker(skill_dir)
+    report = checker.check_all()
+
+    # Print report
+    print_report(report, verbose=args.verbose)
+
+    # Exit code
+    if args.strict and (report.has_errors or report.has_warnings) or report.has_errors:
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()