skill-seekers 2.7.3__py3-none-any.whl

skill_seekers/cli/main.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python3
+"""
+Skill Seekers - Unified CLI Entry Point
+
+Provides a git-style unified command-line interface for all Skill Seekers tools.
+
+Usage:
+    skill-seekers <command> [options]
+
+Commands:
+    config               Configure GitHub tokens, API keys, and settings
+    scrape               Scrape documentation website
+    github               Scrape GitHub repository
+    pdf                  Extract from PDF file
+    unified              Multi-source scraping (docs + GitHub + PDF)
+    enhance              AI-powered enhancement (local, no API key)
+    enhance-status       Check enhancement status (for background/daemon modes)
+    package              Package skill into .zip file
+    upload               Upload skill to Claude
+    estimate             Estimate page count before scraping
+    extract-test-examples Extract usage examples from test files
+    install-agent        Install skill to AI agent directories
+    resume               Resume interrupted scraping job
+
+Examples:
+    skill-seekers scrape --config configs/react.json
+    skill-seekers github --repo microsoft/TypeScript
+    skill-seekers unified --config configs/react_unified.json
+    skill-seekers extract-test-examples tests/ --language python
+    skill-seekers package output/react/
+    skill-seekers install-agent output/react/ --agent cursor
+"""
+
+import argparse
+import sys
+
+from skill_seekers.cli import __version__
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create the main argument parser with subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="skill-seekers",
+        description="Convert documentation, GitHub repos, and PDFs into Claude AI skills",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Scrape documentation
+  skill-seekers scrape --config configs/react.json
+
+  # Scrape GitHub repository
+  skill-seekers github --repo microsoft/TypeScript --name typescript
+
+  # Multi-source scraping (unified)
+  skill-seekers unified --config configs/react_unified.json
+
+  # AI-powered enhancement
+  skill-seekers enhance output/react/
+
+  # Package and upload
+  skill-seekers package output/react/
+  skill-seekers upload output/react.zip
+
+For more information: https://github.com/yusufkaraaslan/Skill_Seekers
+        """,
+    )
+
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+
+    subparsers = parser.add_subparsers(
+        dest="command",
+        title="commands",
+        description="Available Skill Seekers commands",
+        help="Command to run",
+    )
+
+    # === config subcommand ===
+    config_parser = subparsers.add_parser(
+        "config",
+        help="Configure GitHub tokens, API keys, and settings",
+        description="Interactive configuration wizard",
+    )
+    config_parser.add_argument(
+        "--github", action="store_true", help="Go directly to GitHub token setup"
+    )
+    config_parser.add_argument(
+        "--api-keys", action="store_true", help="Go directly to API keys setup"
+    )
+    config_parser.add_argument(
+        "--show", action="store_true", help="Show current configuration and exit"
+    )
+    config_parser.add_argument("--test", action="store_true", help="Test connections and exit")
+
+    # === scrape subcommand ===
+    scrape_parser = subparsers.add_parser(
+        "scrape",
+        help="Scrape documentation website",
+        description="Scrape documentation website and generate skill",
+    )
+    scrape_parser.add_argument("url", nargs="?", help="Documentation URL (positional argument)")
+    scrape_parser.add_argument("--config", help="Config JSON file")
+    scrape_parser.add_argument("--name", help="Skill name")
+    scrape_parser.add_argument("--description", help="Skill description")
+    scrape_parser.add_argument("--max-pages", type=int, dest="max_pages", help="Maximum pages to scrape (override config)")
+    scrape_parser.add_argument(
+        "--skip-scrape", action="store_true", help="Skip scraping, use cached data"
+    )
+    scrape_parser.add_argument("--enhance", action="store_true", help="AI enhancement (API)")
+    scrape_parser.add_argument(
+        "--enhance-local", action="store_true", help="AI enhancement (local)"
+    )
+    scrape_parser.add_argument("--dry-run", action="store_true", help="Dry run mode")
+    scrape_parser.add_argument(
+        "--async", dest="async_mode", action="store_true", help="Use async scraping"
+    )
+    scrape_parser.add_argument("--workers", type=int, help="Number of async workers")
+
+    # === github subcommand ===
+    github_parser = subparsers.add_parser(
+        "github",
+        help="Scrape GitHub repository",
+        description="Scrape GitHub repository and generate skill",
+    )
+    github_parser.add_argument("--config", help="Config JSON file")
+    github_parser.add_argument("--repo", help="GitHub repo (owner/repo)")
+    github_parser.add_argument("--name", help="Skill name")
+    github_parser.add_argument("--description", help="Skill description")
+    github_parser.add_argument("--enhance", action="store_true", help="AI enhancement (API)")
+    github_parser.add_argument(
+        "--enhance-local", action="store_true", help="AI enhancement (local)"
+    )
+    github_parser.add_argument("--api-key", type=str, help="Anthropic API key for --enhance")
+    github_parser.add_argument(
+        "--non-interactive",
+        action="store_true",
+        help="Non-interactive mode (fail fast on rate limits)",
+    )
+    github_parser.add_argument("--profile", type=str, help="GitHub profile name from config")
+
+    # === pdf subcommand ===
+    pdf_parser = subparsers.add_parser(
+        "pdf",
+        help="Extract from PDF file",
+        description="Extract content from PDF and generate skill",
+    )
+    pdf_parser.add_argument("--config", help="Config JSON file")
+    pdf_parser.add_argument("--pdf", help="PDF file path")
+    pdf_parser.add_argument("--name", help="Skill name")
+    pdf_parser.add_argument("--description", help="Skill description")
+    pdf_parser.add_argument("--from-json", help="Build from extracted JSON")
+
+    # === unified subcommand ===
+    unified_parser = subparsers.add_parser(
+        "unified",
+        help="Multi-source scraping (docs + GitHub + PDF)",
+        description="Combine multiple sources into one skill",
+    )
+    unified_parser.add_argument("--config", required=True, help="Unified config JSON file")
+    unified_parser.add_argument("--merge-mode", help="Merge mode (rule-based, claude-enhanced)")
+    unified_parser.add_argument("--fresh", action="store_true", help="Clear existing data and start fresh")
+    unified_parser.add_argument("--dry-run", action="store_true", help="Dry run mode")
+
+    # === enhance subcommand ===
+    enhance_parser = subparsers.add_parser(
+        "enhance",
+        help="AI-powered enhancement (local, no API key)",
+        description="Enhance SKILL.md using Claude Code (local)",
+    )
+    enhance_parser.add_argument("skill_directory", help="Skill directory path")
+    enhance_parser.add_argument("--background", action="store_true", help="Run in background")
+    enhance_parser.add_argument("--daemon", action="store_true", help="Run as daemon")
+    enhance_parser.add_argument(
+        "--no-force", action="store_true", help="Disable force mode (enable confirmations)"
+    )
+    enhance_parser.add_argument("--timeout", type=int, default=600, help="Timeout in seconds")
+
+    # === enhance-status subcommand ===
+    enhance_status_parser = subparsers.add_parser(
+        "enhance-status",
+        help="Check enhancement status (for background/daemon modes)",
+        description="Monitor background enhancement processes",
+    )
+    enhance_status_parser.add_argument("skill_directory", help="Skill directory path")
+    enhance_status_parser.add_argument(
+        "--watch", "-w", action="store_true", help="Watch in real-time"
+    )
+    enhance_status_parser.add_argument("--json", action="store_true", help="JSON output")
+    enhance_status_parser.add_argument(
+        "--interval", type=int, default=2, help="Watch interval in seconds"
+    )
+
+    # === package subcommand ===
+    package_parser = subparsers.add_parser(
+        "package",
+        help="Package skill into .zip file",
+        description="Package skill directory into uploadable .zip",
+    )
+    package_parser.add_argument("skill_directory", help="Skill directory path")
+    package_parser.add_argument("--no-open", action="store_true", help="Don't open output folder")
+    package_parser.add_argument("--upload", action="store_true", help="Auto-upload after packaging")
+
+    # === upload subcommand ===
+    upload_parser = subparsers.add_parser(
+        "upload",
+        help="Upload skill to Claude",
+        description="Upload .zip file to Claude via Anthropic API",
+    )
+    upload_parser.add_argument("zip_file", help=".zip file to upload")
+    upload_parser.add_argument("--api-key", help="Anthropic API key")
+
+    # === estimate subcommand ===
+    estimate_parser = subparsers.add_parser(
+        "estimate",
+        help="Estimate page count before scraping",
+        description="Estimate total pages for documentation scraping",
+    )
+    estimate_parser.add_argument("config", nargs="?", help="Config JSON file")
+    estimate_parser.add_argument("--all", action="store_true", help="List all available configs")
+    estimate_parser.add_argument("--max-discovery", type=int, help="Max pages to discover")
+
+    # === extract-test-examples subcommand ===
+    test_examples_parser = subparsers.add_parser(
+        "extract-test-examples",
+        help="Extract usage examples from test files",
+        description="Analyze test files to extract real API usage patterns",
+    )
+    test_examples_parser.add_argument(
+        "directory", nargs="?", help="Directory containing test files"
+    )
+    test_examples_parser.add_argument("--file", help="Single test file to analyze")
+    test_examples_parser.add_argument(
+        "--language", help="Filter by programming language (python, javascript, etc.)"
+    )
+    test_examples_parser.add_argument(
+        "--min-confidence",
+        type=float,
+        default=0.5,
+        help="Minimum confidence threshold (0.0-1.0, default: 0.5)",
+    )
+    test_examples_parser.add_argument(
+        "--max-per-file", type=int, default=10, help="Maximum examples per file (default: 10)"
+    )
+    test_examples_parser.add_argument("--json", action="store_true", help="Output JSON format")
+    test_examples_parser.add_argument(
+        "--markdown", action="store_true", help="Output Markdown format"
+    )
+
+    # === install-agent subcommand ===
+    install_agent_parser = subparsers.add_parser(
+        "install-agent",
+        help="Install skill to AI agent directories",
+        description="Copy skill to agent-specific installation directories",
+    )
+    install_agent_parser.add_argument(
+        "skill_directory", help="Skill directory path (e.g., output/react/)"
+    )
+    install_agent_parser.add_argument(
+        "--agent",
+        required=True,
+        help="Agent name (claude, cursor, vscode, amp, goose, opencode, all)",
+    )
+    install_agent_parser.add_argument(
+        "--force", action="store_true", help="Overwrite existing installation without asking"
+    )
+    install_agent_parser.add_argument(
+        "--dry-run", action="store_true", help="Preview installation without making changes"
+    )
+
+    # === install subcommand ===
+    install_parser = subparsers.add_parser(
+        "install",
+        help="Complete workflow: fetch → scrape → enhance → package → upload",
+        description="One-command skill installation (AI enhancement MANDATORY)",
+    )
+    install_parser.add_argument(
+        "--config",
+        required=True,
+        help="Config name (e.g., 'react') or path (e.g., 'configs/custom.json')",
+    )
+    install_parser.add_argument(
+        "--destination", default="output", help="Output directory (default: output/)"
+    )
+    install_parser.add_argument(
+        "--no-upload", action="store_true", help="Skip automatic upload to Claude"
+    )
+    install_parser.add_argument(
+        "--unlimited", action="store_true", help="Remove page limits during scraping"
+    )
+    install_parser.add_argument(
+        "--dry-run", action="store_true", help="Preview workflow without executing"
+    )
+
+    # === resume subcommand ===
+    resume_parser = subparsers.add_parser(
+        "resume",
+        help="Resume interrupted scraping job",
+        description="Continue from saved progress checkpoint",
+    )
+    resume_parser.add_argument(
+        "job_id", nargs="?", help="Job ID to resume (or use --list to see available jobs)"
+    )
+    resume_parser.add_argument("--list", action="store_true", help="List all resumable jobs")
+    resume_parser.add_argument("--clean", action="store_true", help="Clean up old progress files")
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point for the unified CLI.
+
+    Args:
+        argv: Command-line arguments (defaults to sys.argv)
+
+    Returns:
+        Exit code (0 for success, non-zero for error)
+    """
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    if not args.command:
+        parser.print_help()
+        return 1
+
+    # Delegate to the appropriate tool
+    try:
+        if args.command == "config":
+            from skill_seekers.cli.config_command import main as config_main
+
+            sys.argv = ["config_command.py"]
+            if args.github:
+                sys.argv.append("--github")
+            if args.api_keys:
+                sys.argv.append("--api-keys")
+            if args.show:
+                sys.argv.append("--show")
+            if args.test:
+                sys.argv.append("--test")
+            return config_main() or 0
+
+        elif args.command == "scrape":
+            from skill_seekers.cli.doc_scraper import main as scrape_main
+
+            # Convert args namespace to sys.argv format for doc_scraper
+            sys.argv = ["doc_scraper.py"]
+            # Add positional URL if provided (positional arg has priority)
+            if hasattr(args, 'url') and args.url:
+                sys.argv.append(args.url)
+            if args.config:
+                sys.argv.extend(["--config", args.config])
+            if args.name:
+                sys.argv.extend(["--name", args.name])
+            if args.description:
+                sys.argv.extend(["--description", args.description])
+            if hasattr(args, 'max_pages') and args.max_pages:
+                sys.argv.extend(["--max-pages", str(args.max_pages)])
+            if args.skip_scrape:
+                sys.argv.append("--skip-scrape")
+            if args.enhance:
+                sys.argv.append("--enhance")
+            if args.enhance_local:
+                sys.argv.append("--enhance-local")
+            if args.dry_run:
+                sys.argv.append("--dry-run")
+            if args.async_mode:
+                sys.argv.append("--async")
+            if args.workers:
+                sys.argv.extend(["--workers", str(args.workers)])
+            return scrape_main() or 0
+
+        elif args.command == "github":
+            from skill_seekers.cli.github_scraper import main as github_main
+
+            sys.argv = ["github_scraper.py"]
+            if args.config:
+                sys.argv.extend(["--config", args.config])
+            if args.repo:
+                sys.argv.extend(["--repo", args.repo])
+            if args.name:
+                sys.argv.extend(["--name", args.name])
+            if args.description:
+                sys.argv.extend(["--description", args.description])
+            if args.enhance:
+                sys.argv.append("--enhance")
+            if args.enhance_local:
+                sys.argv.append("--enhance-local")
+            if args.api_key:
+                sys.argv.extend(["--api-key", args.api_key])
+            if args.non_interactive:
+                sys.argv.append("--non-interactive")
+            if args.profile:
+                sys.argv.extend(["--profile", args.profile])
+            return github_main() or 0
+
+        elif args.command == "pdf":
+            from skill_seekers.cli.pdf_scraper import main as pdf_main
+
+            sys.argv = ["pdf_scraper.py"]
+            if args.config:
+                sys.argv.extend(["--config", args.config])
+            if args.pdf:
+                sys.argv.extend(["--pdf", args.pdf])
+            if args.name:
+                sys.argv.extend(["--name", args.name])
+            if args.description:
+                sys.argv.extend(["--description", args.description])
+            if args.from_json:
+                sys.argv.extend(["--from-json", args.from_json])
+            return pdf_main() or 0
+
+        elif args.command == "unified":
+            from skill_seekers.cli.unified_scraper import main as unified_main
+
+            sys.argv = ["unified_scraper.py", "--config", args.config]
+            if args.merge_mode:
+                sys.argv.extend(["--merge-mode", args.merge_mode])
+            if args.fresh:
+                sys.argv.append("--fresh")
+            if args.dry_run:
+                sys.argv.append("--dry-run")
+            return unified_main() or 0
+
+        elif args.command == "enhance":
+            from skill_seekers.cli.enhance_skill_local import main as enhance_main
+
+            sys.argv = ["enhance_skill_local.py", args.skill_directory]
+            if args.background:
+                sys.argv.append("--background")
+            if args.daemon:
+                sys.argv.append("--daemon")
+            if args.no_force:
+                sys.argv.append("--no-force")
+            if args.timeout:
+                sys.argv.extend(["--timeout", str(args.timeout)])
+            return enhance_main() or 0
+
+        elif args.command == "enhance-status":
+            from skill_seekers.cli.enhance_status import main as enhance_status_main
+
+            sys.argv = ["enhance_status.py", args.skill_directory]
+            if args.watch:
+                sys.argv.append("--watch")
+            if args.json:
+                sys.argv.append("--json")
+            if args.interval:
+                sys.argv.extend(["--interval", str(args.interval)])
+            return enhance_status_main() or 0
+
+        elif args.command == "package":
+            from skill_seekers.cli.package_skill import main as package_main
+
+            sys.argv = ["package_skill.py", args.skill_directory]
+            if args.no_open:
+                sys.argv.append("--no-open")
+            if args.upload:
+                sys.argv.append("--upload")
+            return package_main() or 0
+
+        elif args.command == "upload":
+            from skill_seekers.cli.upload_skill import main as upload_main
+
+            sys.argv = ["upload_skill.py", args.zip_file]
+            if args.api_key:
+                sys.argv.extend(["--api-key", args.api_key])
+            return upload_main() or 0
+
+        elif args.command == "estimate":
+            from skill_seekers.cli.estimate_pages import main as estimate_main
+
+            sys.argv = ["estimate_pages.py"]
+            if args.all:
+                sys.argv.append("--all")
+            elif args.config:
+                sys.argv.append(args.config)
+            if args.max_discovery:
+                sys.argv.extend(["--max-discovery", str(args.max_discovery)])
+            return estimate_main() or 0
+
+        elif args.command == "extract-test-examples":
+            from skill_seekers.cli.test_example_extractor import main as test_examples_main
+
+            sys.argv = ["test_example_extractor.py"]
+            if args.directory:
+                sys.argv.append(args.directory)
+            if args.file:
+                sys.argv.extend(["--file", args.file])
+            if args.language:
+                sys.argv.extend(["--language", args.language])
+            if args.min_confidence:
+                sys.argv.extend(["--min-confidence", str(args.min_confidence)])
+            if args.max_per_file:
+                sys.argv.extend(["--max-per-file", str(args.max_per_file)])
+            if args.json:
+                sys.argv.append("--json")
+            if args.markdown:
+                sys.argv.append("--markdown")
+            return test_examples_main() or 0
+
+        elif args.command == "install-agent":
+            from skill_seekers.cli.install_agent import main as install_agent_main
+
+            sys.argv = ["install_agent.py", args.skill_directory, "--agent", args.agent]
+            if args.force:
+                sys.argv.append("--force")
+            if args.dry_run:
+                sys.argv.append("--dry-run")
+            return install_agent_main() or 0
+
+        elif args.command == "install":
+            from skill_seekers.cli.install_skill import main as install_main
+
+            sys.argv = ["install_skill.py"]
+            if args.config:
+                sys.argv.extend(["--config", args.config])
+            if args.destination:
+                sys.argv.extend(["--destination", args.destination])
+            if args.no_upload:
+                sys.argv.append("--no-upload")
+            if args.unlimited:
+                sys.argv.append("--unlimited")
+            if args.dry_run:
+                sys.argv.append("--dry-run")
+            return install_main() or 0
+
+        elif args.command == "resume":
+            from skill_seekers.cli.resume_command import main as resume_main
+
+            sys.argv = ["resume_command.py"]
+            if args.job_id:
+                sys.argv.append(args.job_id)
+            if args.list:
+                sys.argv.append("--list")
+            if args.clean:
+                sys.argv.append("--clean")
+            return resume_main() or 0
+
+        else:
+            print(f"Error: Unknown command '{args.command}'", file=sys.stderr)
+            parser.print_help()
+            return 1
+
+    except KeyboardInterrupt:
+        print("\n\nInterrupted by user", file=sys.stderr)
+        return 130
+    except Exception as e:
+        # Provide helpful error message
+        error_msg = str(e) if str(e) else f"{type(e).__name__} occurred"
+        print(f"Error: {error_msg}", file=sys.stderr)
+
+        # Show traceback in verbose mode (if -v flag exists in args)
+        import traceback
+        if hasattr(args, 'verbose') and getattr(args, 'verbose', False):
+            traceback.print_exc()
+
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skill_seekers/cli/markdown_cleaner.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Markdown Cleaner Utility
+
+Removes HTML tags and bloat from markdown content while preserving structure.
+Used to clean README files and other documentation for skill generation.
+"""
+
+import re
+
+
+class MarkdownCleaner:
+    """Clean HTML from markdown while preserving structure"""
+
+    @staticmethod
+    def remove_html_tags(text: str) -> str:
+        """
+        Remove HTML tags while preserving text content.
+
+        Args:
+            text: Markdown text possibly containing HTML
+
+        Returns:
+            Cleaned markdown with HTML tags removed
+        """
+        # Remove HTML comments
+        text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+
+        # Remove HTML tags but keep content
+        text = re.sub(r"<[^>]+>", "", text)
+
+        # Remove empty lines created by HTML removal
+        text = re.sub(r"\n\s*\n\s*\n+", "\n\n", text)
+
+        return text.strip()
+
+    @staticmethod
+    def extract_first_section(text: str, max_chars: int = 500) -> str:
+        """
+        Extract first meaningful content, respecting markdown structure.
+
+        Captures content including section headings up to max_chars.
+        For short READMEs, includes everything. For longer ones, extracts
+        intro + first few sections (e.g., installation, quick start).
+
+        Args:
+            text: Full markdown text
+            max_chars: Maximum characters to extract
+
+        Returns:
+            First section content (cleaned, including headings)
+        """
+        # Remove HTML first
+        text = MarkdownCleaner.remove_html_tags(text)
+
+        # If text is short, return it all
+        if len(text) <= max_chars:
+            return text.strip()
+
+        # For longer text, extract smartly
+        lines = text.split("\n")
+        content_lines = []
+        char_count = 0
+        section_count = 0
+        in_code_block = False  # Track code fence state to avoid truncating mid-block
+
+        for line in lines:
+            # Check for code fence (```)
+            if line.strip().startswith("```"):
+                in_code_block = not in_code_block
+
+            # Check for any heading (H1-H6)
+            is_heading = re.match(r"^#{1,6}\s+", line)
+
+            if is_heading:
+                section_count += 1
+                # Include first 4 sections (title + 3 sections like Installation, Quick Start, Features)
+                if section_count <= 4:
+                    content_lines.append(line)
+                    char_count += len(line)
+                else:
+                    # Stop after 4 sections (but not if in code block)
+                    if not in_code_block:
+                        break
+            else:
+                # Include content
+                content_lines.append(line)
+                char_count += len(line)
+
+            # Stop if we have enough content (but not if in code block)
+            if char_count >= max_chars and not in_code_block:
+                break
+
+        result = "\n".join(content_lines).strip()
+
+        # If we truncated, ensure we don't break markdown (only if not in code block)
+        if char_count >= max_chars and not in_code_block:
+            # Find last complete sentence
+            result = MarkdownCleaner._truncate_at_sentence(result, max_chars)
+
+        return result
+
+    @staticmethod
+    def _truncate_at_sentence(text: str, max_chars: int) -> str:
+        """
+        Truncate at last complete sentence before max_chars.
+
+        Args:
+            text: Text to truncate
+            max_chars: Maximum character count
+
+        Returns:
+            Truncated text ending at sentence boundary
+        """
+        if len(text) <= max_chars:
+            return text
+
+        # Find last sentence boundary before max_chars
+        truncated = text[:max_chars]
+
+        # Look for last period, exclamation, or question mark
+        last_sentence = max(truncated.rfind(". "), truncated.rfind("! "), truncated.rfind("? "))
+
+        if last_sentence > max_chars // 2:  # At least half the content
+            return truncated[: last_sentence + 1]
+
+        # Fall back to word boundary
+        last_space = truncated.rfind(" ")
+        if last_space > 0:
+            return truncated[:last_space] + "..."
+
+        return truncated + "..."