greenmining 0.1.4__py3-none-any.whl

greenmining/__init__.py

@@ -0,0 +1,20 @@
+"""Green Microservices Mining - GSF Pattern Analysis Tool."""
+
+from greenmining.config import Config
+from greenmining.gsf_patterns import (
+    GREEN_KEYWORDS,
+    GSF_PATTERNS,
+    get_pattern_by_keywords,
+    is_green_aware,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Config",
+    "GSF_PATTERNS",
+    "GREEN_KEYWORDS",
+    "is_green_aware",
+    "get_pattern_by_keywords",
+    "__version__",
+]

greenmining/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running greenmining as a module: python -m greenmining"""
+
+from greenmining.cli import cli
+
+if __name__ == "__main__":
+    cli()

greenmining/__version__.py

@@ -0,0 +1,3 @@
+"""Version information for greenmining."""
+
+__version__ = "0.1.4"

greenmining/cli.py

@@ -0,0 +1,370 @@
+"""Green Microservices Mining CLI"""
+
+import click
+
+from greenmining.config import Config
+from greenmining.controllers.repository_controller import RepositoryController
+from greenmining.presenters.console_presenter import ConsolePresenter
+from greenmining.utils import colored_print, load_json_file
+
+# Initialize configuration
+config = Config()
+
+# Initialize presenter
+presenter = ConsolePresenter()
+
+
+@click.group()
+@click.option("--config-file", default=".env", help="Path to configuration file")
+@click.option("--verbose", is_flag=True, help="Enable verbose output")
+def cli(config_file, verbose):
+    """Green Microservices Mining"""
+    if verbose:
+        config.VERBOSE = True
+
+
+@cli.command()
+@click.option("--max-repos", default=100, type=int, help="Maximum repositories to fetch")
+@click.option("--min-stars", default=100, type=int, help="Minimum stars required")
+@click.option(
+    "--languages", default="Python,Java,Go,JavaScript,TypeScript", help="Comma-separated languages"
+)
+def fetch(max_repos, min_stars, languages):
+    """Fetch top microservice repositories from GitHub."""
+    presenter.show_banner()
+    colored_print(f"\n🎯 Target: {max_repos} repositories\n", "cyan")
+
+    controller = RepositoryController(config)
+    lang_list = [lang.strip() for lang in languages.split(",")]
+
+    try:
+        repositories = controller.fetch_repositories(
+            max_repos=max_repos, min_stars=min_stars, languages=lang_list
+        )
+
+        # Show results
+        repo_dicts = [r.to_dict() for r in repositories]
+        presenter.show_repositories(repo_dicts, limit=10)
+
+        stats = controller.get_repository_stats(repositories)
+        colored_print(f"\n📊 Total Stars: {stats.get('total_stars', 0):,}", "green")
+        colored_print(f"📈 Average Stars: {stats.get('avg_stars', 0):.0f}", "green")
+
+        presenter.show_success(f"Fetched {len(repositories)} repositories successfully!")
+
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+@cli.command()
+@click.option("--max-commits", default=50, type=int, help="Max commits per repository")
+@click.option("--skip-merges", is_flag=True, default=True, help="Skip merge commits")
+@click.option("--days-back", default=730, type=int, help="Days to look back (default: 2 years)")
+def extract(max_commits, skip_merges, days_back):
+    """Extract commits from fetched repositories."""
+    presenter.show_banner()
+
+    from services.commit_extractor import CommitExtractor
+
+    try:
+        # Load repositories
+        controller = RepositoryController(config)
+        repositories = controller.load_repositories()
+
+        colored_print(f"\n📝 Extracting commits from {len(repositories)} repositories...\n", "cyan")
+        colored_print(
+            f"   Settings: max={max_commits}/repo, skip_merges={skip_merges}, days_back={days_back}\n",
+            "cyan",
+        )
+
+        # Extract commits
+        extractor = CommitExtractor(
+            max_commits=max_commits, skip_merges=skip_merges, days_back=days_back
+        )
+        commits = extractor.extract_from_repositories(
+            repositories=[r.to_dict() for r in repositories]
+        )
+
+        # Save commits
+        from utils import save_json_file
+
+        save_json_file(commits, config.COMMITS_FILE)
+        colored_print(f"   Saved to: {config.COMMITS_FILE}", "cyan")
+
+        # Show stats
+        stats = {
+            "total_commits": len(commits),
+            "total_repos": len(repositories),
+            "avg_per_repo": len(commits) / len(repositories) if repositories else 0,
+        }
+
+        presenter.show_commit_stats(stats)
+        presenter.show_success(f"Extracted {len(commits)} commits successfully!")
+
+    except FileNotFoundError as e:
+        presenter.show_error(str(e))
+        colored_print("💡 Run 'fetch' command first to get repositories", "yellow")
+        raise click.Abort() from e
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+@cli.command()
+@click.option("--batch-size", default=10, type=int, help="Batch size for processing")
+def analyze(batch_size):
+    """Analyze commits for green software patterns."""
+    presenter.show_banner()
+
+    from services.data_analyzer import DataAnalyzer
+    from utils import save_json_file
+
+    try:
+        # Load commits
+        if not config.COMMITS_FILE.exists():
+            raise FileNotFoundError("No commits file found. Run 'extract' first.")
+
+        commits = load_json_file(config.COMMITS_FILE)
+        colored_print(f"\n🔬 Analyzing {len(commits)} commits for green patterns...\n", "cyan")
+        colored_print("   Method: Keyword-based heuristic analysis\n", "cyan")
+        colored_print(f"   Batch size: {batch_size}\n", "cyan")
+
+        # Analyze
+        analyzer = DataAnalyzer()
+        results = analyzer.analyze_commits_batch(commits, batch_size=batch_size)
+
+        # Save results
+        save_json_file(results, config.ANALYSIS_FILE)
+
+        # Show results
+        green_count = sum(1 for r in results if r.get("green_aware", False))
+        green_rate = (green_count / len(results)) if results else 0
+
+        results_dict = {
+            "summary": {
+                "total_commits": len(results),
+                "green_commits": green_count,
+                "green_commit_rate": green_rate,
+            },
+            "known_patterns": {},
+        }
+
+        presenter.show_analysis_results(results_dict)
+        presenter.show_success(f"Analysis complete! Results saved to {config.ANALYSIS_FILE}")
+
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+@cli.command()
+def aggregate():
+    """Aggregate analysis results and generate statistics."""
+    presenter.show_banner()
+
+    from services.data_aggregator import DataAggregator
+    from utils import save_json_file
+
+    try:
+        # Load data
+        if not config.ANALYSIS_FILE.exists():
+            raise FileNotFoundError("No analysis file found. Run 'analyze' first.")
+
+        results = load_json_file(config.ANALYSIS_FILE)
+        repos = load_json_file(config.REPOS_FILE) if config.REPOS_FILE.exists() else []
+
+        colored_print(f"\n📊 Aggregating results from {len(results)} commits...\n", "cyan")
+
+        # Aggregate
+        aggregator = DataAggregator()
+        aggregated = aggregator.aggregate(results, repos)
+
+        # Save
+        save_json_file(aggregated, config.AGGREGATED_FILE)
+
+        # Show results
+        presenter.show_analysis_results(aggregated)
+
+        if aggregated.get("known_patterns"):
+            presenter.show_pattern_distribution(aggregated["known_patterns"], limit=10)
+
+        presenter.show_success(f"Aggregation complete! Results saved to {config.AGGREGATED_FILE}")
+
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+@cli.command()
+@click.option("--output", default="green_microservices_analysis.md", help="Output filename")
+def report(output):
+    """Generate comprehensive markdown report."""
+    presenter.show_banner()
+
+    from services.reports import ReportGenerator
+
+    try:
+        # Load aggregated data
+        if not config.AGGREGATED_FILE.exists():
+            raise FileNotFoundError("No aggregated data found. Run 'aggregate' first.")
+
+        aggregated = load_json_file(config.AGGREGATED_FILE)
+
+        colored_print("\n📄 Generating comprehensive report...\n", "cyan")
+
+        # Generate report
+        generator = ReportGenerator()
+        report_path = generator.generate_report(aggregated, output)
+
+        presenter.show_success(f"Report generated: {report_path}")
+        colored_print("\n📖 The report includes:", "cyan")
+        colored_print("   • Executive Summary", "white")
+        colored_print("   • Methodology", "white")
+        colored_print("   • Results & Statistics", "white")
+        colored_print("   • Pattern Analysis", "white")
+        colored_print("   • Per-Repository Breakdown", "white")
+        colored_print("   • Discussion & Conclusions", "white")
+
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+@cli.command()
+def status():
+    """Show current pipeline status."""
+    presenter.show_banner()
+
+    phases = {
+        "1. Fetch Repositories": {
+            "file": str(config.REPOS_FILE),
+            "completed": config.REPOS_FILE.exists(),
+            "size": (
+                f"{config.REPOS_FILE.stat().st_size / 1024:.1f} KB"
+                if config.REPOS_FILE.exists()
+                else "N/A"
+            ),
+        },
+        "2. Extract Commits": {
+            "file": str(config.COMMITS_FILE),
+            "completed": config.COMMITS_FILE.exists(),
+            "size": (
+                f"{config.COMMITS_FILE.stat().st_size / 1024:.1f} KB"
+                if config.COMMITS_FILE.exists()
+                else "N/A"
+            ),
+        },
+        "3. Analyze Commits": {
+            "file": str(config.ANALYSIS_FILE),
+            "completed": config.ANALYSIS_FILE.exists(),
+            "size": (
+                f"{config.ANALYSIS_FILE.stat().st_size / 1024:.1f} KB"
+                if config.ANALYSIS_FILE.exists()
+                else "N/A"
+            ),
+        },
+        "4. Aggregate Results": {
+            "file": str(config.AGGREGATED_FILE),
+            "completed": config.AGGREGATED_FILE.exists(),
+            "size": (
+                f"{config.AGGREGATED_FILE.stat().st_size / 1024:.1f} KB"
+                if config.AGGREGATED_FILE.exists()
+                else "N/A"
+            ),
+        },
+        "5. Generate Report": {
+            "file": str(config.REPORT_FILE),
+            "completed": config.REPORT_FILE.exists(),
+            "size": (
+                f"{config.REPORT_FILE.stat().st_size / 1024:.1f} KB"
+                if config.REPORT_FILE.exists()
+                else "N/A"
+            ),
+        },
+    }
+
+    presenter.show_pipeline_status(phases)
+
+    # Show next step
+    for phase_name, info in phases.items():
+        if not info["completed"]:
+            colored_print(f"\n💡 Next step: {phase_name}", "yellow")
+            break
+    else:
+        colored_print("\n✅ All phases complete!", "green")
+
+
+@cli.command()
+@click.option("--max-repos", default=100, type=int, help="Maximum repositories to analyze")
+@click.option("--skip-fetch", is_flag=True, help="Skip fetch phase if data exists")
+def pipeline(max_repos, skip_fetch):
+    """Run full pipeline: fetch → extract → analyze → aggregate → report."""
+    presenter.show_banner()
+
+    colored_print("\n🚀 Starting Full Pipeline...\n", "green")
+    colored_print(f"   Target: {max_repos} repositories", "cyan")
+    colored_print("   Phases: fetch → extract → analyze → aggregate → report\n", "cyan")
+
+    try:
+        # Phase 1: Fetch
+        if not skip_fetch or not config.REPOS_FILE.exists():
+            colored_print("\n[1/5] 🔍 Fetching repositories...", "cyan")
+            controller = RepositoryController(config)
+            controller.fetch_repositories(max_repos=max_repos)
+        else:
+            colored_print("\n[1/5] ⏭️  Skipping fetch (using existing data)", "yellow")
+
+        # Phase 2: Extract
+        colored_print("\n[2/5] 📝 Extracting commits...", "cyan")
+        from services.commit_extractor import CommitExtractor
+        from utils import save_json_file
+
+        controller = RepositoryController(config)
+        repos = controller.load_repositories()
+        extractor = CommitExtractor()
+        commits = extractor.extract_from_repositories([r.to_dict() for r in repos])
+        save_json_file(commits, config.COMMITS_FILE)
+        colored_print(f"   Saved {len(commits)} commits to: {config.COMMITS_FILE}", "green")
+
+        # Phase 3: Analyze
+        colored_print("\n[3/5] 🔬 Analyzing commits...", "cyan")
+        from services.data_analyzer import DataAnalyzer
+
+        commits = load_json_file(config.COMMITS_FILE)
+        analyzer = DataAnalyzer()
+        results = analyzer.analyze_commits_batch(commits)
+        save_json_file(results, config.ANALYSIS_FILE)
+        colored_print(
+            f"   Analyzed {len(results)} commits, saved to: {config.ANALYSIS_FILE}", "green"
+        )
+
+        # Phase 4: Aggregate
+        colored_print("\n[4/5] 📊 Aggregating results...", "cyan")
+        from services.data_aggregator import DataAggregator
+
+        aggregator = DataAggregator()
+        aggregated = aggregator.aggregate(results, [r.to_dict() for r in repos])
+        save_json_file(aggregated, config.AGGREGATED_FILE)
+
+        # Phase 5: Report
+        colored_print("\n[5/5] 📄 Generating report...", "cyan")
+        from services.reports import ReportGenerator
+
+        generator = ReportGenerator()
+        generator.generate_report(aggregated)
+
+        colored_print("\n" + "=" * 60, "green")
+        colored_print("✅ Pipeline Complete!", "green")
+        colored_print("=" * 60, "green")
+
+        presenter.show_success(f"All results saved to {config.OUTPUT_DIR}")
+        colored_print(f"\n📖 View report: {config.REPORT_FILE}", "cyan")
+
+    except Exception as e:
+        presenter.show_error(str(e))
+        raise click.Abort() from e
+
+
+if __name__ == "__main__":
+    cli()

greenmining/config.py

@@ -0,0 +1,120 @@
+"""Configuration management for green microservices mining CLI."""
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+
+class Config:
+    """Configuration class for loading and validating environment variables."""
+
+    def __init__(self, env_file: str = ".env"):
+        """Initialize configuration from environment file.
+
+        Args:
+            env_file: Path to .env file
+        """
+        # Load environment variables
+        env_path = Path(env_file)
+        if env_path.exists():
+            load_dotenv(env_path)
+        else:
+            load_dotenv()  # Load from system environment
+
+        # GitHub API Configuration
+        self.GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
+        if not self.GITHUB_TOKEN or self.GITHUB_TOKEN == "your_github_pat_here":
+            raise ValueError("GITHUB_TOKEN not set. Please set it in .env file or environment.")
+
+        # Analysis Type - Using GitHub Copilot for AI-powered analysis
+        self.ANALYSIS_TYPE = "keyword_heuristic"
+
+        # Search and Processing Configuration
+        self.GITHUB_SEARCH_KEYWORDS = ["microservices", "microservice-architecture", "cloud-native"]
+
+        self.SUPPORTED_LANGUAGES = [
+            "Java",
+            "Python",
+            "Go",
+            "JavaScript",
+            "TypeScript",
+            "C#",
+            "Rust",
+        ]
+
+        # Repository and Commit Limits
+        self.MIN_STARS = int(os.getenv("MIN_STARS", "100"))
+        self.MAX_REPOS = int(os.getenv("MAX_REPOS", "100"))
+        self.COMMITS_PER_REPO = int(os.getenv("COMMITS_PER_REPO", "50"))
+        self.DAYS_BACK = int(os.getenv("DAYS_BACK", "730"))  # 2 years
+
+        # Analysis Configuration
+        self.BATCH_SIZE = int(os.getenv("BATCH_SIZE", "10"))
+
+        # Processing Configuration
+        self.TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", "30"))
+        self.MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
+        self.RETRY_DELAY = 2  # seconds
+        self.EXPONENTIAL_BACKOFF = True
+
+        # Output Configuration
+        self.OUTPUT_DIR = Path(os.getenv("OUTPUT_DIR", "./data"))
+        self.OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+        # File Paths
+        self.REPOS_FILE = self.OUTPUT_DIR / "repositories.json"
+        self.COMMITS_FILE = self.OUTPUT_DIR / "commits.json"
+        self.ANALYSIS_FILE = self.OUTPUT_DIR / "analysis_results.json"
+        self.AGGREGATED_FILE = self.OUTPUT_DIR / "aggregated_statistics.json"
+        self.CSV_FILE = self.OUTPUT_DIR / "green_analysis_results.csv"
+        self.REPORT_FILE = self.OUTPUT_DIR / "green_microservices_analysis.md"
+        self.CHECKPOINT_FILE = self.OUTPUT_DIR / "checkpoint.json"
+
+        # Logging
+        self.VERBOSE = os.getenv("VERBOSE", "false").lower() == "true"
+        self.LOG_FILE = self.OUTPUT_DIR / "mining.log"
+
+    def validate(self) -> bool:
+        """Validate that all required configuration is present.
+
+        Returns:
+            True if configuration is valid
+        """
+        required_attrs = ["GITHUB_TOKEN", "CLAUDE_API_KEY", "MAX_REPOS", "COMMITS_PER_REPO"]
+
+        for attr in required_attrs:
+            if not getattr(self, attr, None):
+                raise ValueError(f"Missing required configuration: {attr}")
+
+        return True
+
+    def __repr__(self) -> str:
+        """String representation of configuration (hiding sensitive data)."""
+        return (
+            f"Config("
+            f"MAX_REPOS={self.MAX_REPOS}, "
+            f"COMMITS_PER_REPO={self.COMMITS_PER_REPO}, "
+            f"BATCH_SIZE={self.BATCH_SIZE}, "
+            f"OUTPUT_DIR={self.OUTPUT_DIR}"
+            f")"
+        )
+
+
+# Global config instance
+_config_instance = None
+
+
+def get_config(env_file: str = ".env") -> Config:
+    """Get or create global configuration instance.
+
+    Args:
+        env_file: Path to .env file
+
+    Returns:
+        Config instance
+    """
+    global _config_instance
+    if _config_instance is None:
+        _config_instance = Config(env_file)
+    return _config_instance

greenmining/controllers/__init__.py

@@ -0,0 +1,11 @@
+"""
+Controllers Package - Business logic and orchestration for mining operations.
+
+Controllers coordinate between models, services, and presenters following MCP architecture.
+"""
+
+from .repository_controller import RepositoryController
+
+__all__ = [
+    "RepositoryController",
+]

greenmining/controllers/repository_controller.py

@@ -0,0 +1,117 @@
+"""Repository Controller - Handles repository fetching operations."""
+
+from github import Github, GithubException
+from tqdm import tqdm
+
+from greenmining.config import Config
+from greenmining.models.repository import Repository
+from greenmining.utils import colored_print, load_json_file, save_json_file
+
+
+class RepositoryController:
+    """Controller for GitHub repository operations."""
+
+    def __init__(self, config: Config):
+        """Initialize controller with configuration."""
+        self.config = config
+        self.github = Github(config.GITHUB_TOKEN)
+
+    def fetch_repositories(
+        self, max_repos: int = None, min_stars: int = None, languages: list[str] = None
+    ) -> list[Repository]:
+        """Fetch repositories from GitHub.
+
+        Args:
+            max_repos: Maximum number of repositories to fetch
+            min_stars: Minimum stars filter
+            languages: List of programming languages to filter
+
+        Returns:
+            List of Repository model instances
+        """
+        max_repos = max_repos or self.config.MAX_REPOS
+        min_stars = min_stars or self.config.MIN_STARS
+        languages = languages or self.config.SUPPORTED_LANGUAGES
+
+        colored_print(f"🔍 Fetching up to {max_repos} repositories...", "cyan")
+        colored_print(f"   Filters: min_stars={min_stars}", "cyan")
+
+        # Build search query - simpler approach
+        query = f"microservices stars:>={min_stars}"
+
+        try:
+            # Execute search
+            search_results = self.github.search_repositories(
+                query=query, sort="stars", order="desc"
+            )
+
+            total_found = search_results.totalCount
+            colored_print(f"   Found {total_found} repositories", "green")
+
+            # Fetch repositories
+            repositories = []
+            with tqdm(total=min(max_repos, total_found), desc="Fetching", unit="repo") as pbar:
+                for idx, repo in enumerate(search_results):
+                    if idx >= max_repos:
+                        break
+
+                    try:
+                        repo_model = Repository.from_github_repo(repo, idx + 1)
+                        repositories.append(repo_model)
+                        pbar.update(1)
+                    except GithubException as e:
+                        colored_print(f"   Error: {repo.full_name}: {e}", "yellow")
+                        continue
+
+            # Save to file
+            repo_dicts = [r.to_dict() for r in repositories]
+            save_json_file(repo_dicts, self.config.REPOS_FILE)
+
+            colored_print(f"✅ Fetched {len(repositories)} repositories", "green")
+            colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
+
+            return repositories
+
+        except Exception as e:
+            colored_print(f"❌ Error fetching repositories: {e}", "red")
+            raise
+
+    def load_repositories(self) -> list[Repository]:
+        """Load repositories from file.
+
+        Returns:
+            List of Repository model instances
+        """
+        if not self.config.REPOS_FILE.exists():
+            raise FileNotFoundError(f"No repositories file found at {self.config.REPOS_FILE}")
+
+        repo_dicts = load_json_file(self.config.REPOS_FILE)
+        return [Repository.from_dict(r) for r in repo_dicts]
+
+    def get_repository_stats(self, repositories: list[Repository]) -> dict:
+        """Get statistics about fetched repositories.
+
+        Args:
+            repositories: List of Repository instances
+
+        Returns:
+            Dictionary with statistics
+        """
+        if not repositories:
+            return {}
+
+        return {
+            "total": len(repositories),
+            "by_language": self._count_by_language(repositories),
+            "total_stars": sum(r.stars for r in repositories),
+            "avg_stars": sum(r.stars for r in repositories) / len(repositories),
+            "top_repo": max(repositories, key=lambda r: r.stars).full_name,
+        }
+
+    def _count_by_language(self, repositories: list[Repository]) -> dict:
+        """Count repositories by language."""
+        counts = {}
+        for repo in repositories:
+            lang = repo.language or "Unknown"
+            counts[lang] = counts.get(lang, 0) + 1
+        return counts