greenmining 0.1.4__py3-none-any.whl

greenmining/services/data_analyzer.py

@@ -0,0 +1,333 @@
+"""Data analyzer for green microservices commits using GSF patterns."""
+
+import json
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any, Optional
+
+import click
+from tqdm import tqdm
+
+from greenmining.config import get_config
+from greenmining.gsf_patterns import (
+    GREEN_KEYWORDS,
+    GSF_PATTERNS,
+    get_pattern_by_keywords,
+    is_green_aware,
+)
+from greenmining.utils import (
+    colored_print,
+    create_checkpoint,
+    format_timestamp,
+    load_checkpoint,
+    load_json_file,
+    print_banner,
+    save_json_file,
+)
+
+
+class DataAnalyzer:
+    """Analyzes commits for green software patterns using GSF (Green Software Foundation) patterns."""
+
+    def __init__(self, batch_size: int = 10):
+        """Initialize analyzer with GSF patterns.
+
+        Args:
+            batch_size: Number of commits to process in each batch
+        """
+        # Use GSF patterns from gsf_patterns.py
+        self.gsf_patterns = GSF_PATTERNS
+        self.green_keywords = GREEN_KEYWORDS
+        self.batch_size = batch_size
+
+    def analyze_commits(
+        self, commits: list[dict[str, Any]], resume_from: int = 0
+    ) -> list[dict[str, Any]]:
+        """Analyze commits for green software practices.
+
+        Args:
+            commits: List of commit dictionaries
+            resume_from: Index to resume from
+
+        Returns:
+            List of analysis results
+        """
+        results = []
+
+        colored_print(f"\nAnalyzing {len(commits)} commits for green practices...", "cyan")
+
+        with tqdm(
+            total=len(commits), initial=resume_from, desc="Analyzing commits", unit="commit"
+        ) as pbar:
+            for _idx, commit in enumerate(commits[resume_from:], start=resume_from):
+                try:
+                    analysis = self._analyze_commit(commit)
+                    results.append(analysis)
+                    pbar.update(1)
+                except Exception as e:
+                    colored_print(
+                        f"\nError analyzing commit {commit.get('commit_id', 'unknown')}: {e}",
+                        "yellow",
+                    )
+                    pbar.update(1)
+
+        return results
+
+    def _analyze_commit(self, commit: dict[str, Any]) -> dict[str, Any]:
+        """Analyze a single commit using GSF patterns.
+
+        Args:
+            commit: Commit dictionary
+
+        Returns:
+            Analysis result with GSF pattern matching
+        """
+        message = commit.get("message", "")
+
+        # Q1: GREEN AWARENESS - Check using GSF keywords
+        green_aware = is_green_aware(message)
+
+        # Q2: KNOWN GSF PATTERNS - Match against Green Software Foundation patterns
+        matched_patterns = get_pattern_by_keywords(message)
+
+        # Get detailed pattern info
+        pattern_details = []
+        for _pattern_id, pattern in self.gsf_patterns.items():
+            if pattern["name"] in matched_patterns:
+                pattern_details.append(
+                    {
+                        "name": pattern["name"],
+                        "category": pattern["category"],
+                        "description": pattern["description"],
+                        "sci_impact": pattern["sci_impact"],
+                    }
+                )
+
+        # Calculate confidence based on number of patterns matched
+        confidence = (
+            "high"
+            if len(matched_patterns) >= 2
+            else "medium" if len(matched_patterns) == 1 else "low"
+        )
+
+        return {
+            "commit_hash": commit.get("hash", commit.get("commit_id", "unknown")),
+            "repository": commit.get("repository", commit.get("repo_name", "unknown")),
+            "author": commit.get("author_name", "unknown"),
+            "date": commit.get("author_date", commit.get("date", "unknown")),
+            "message": message,
+            # Research Question 1: Green awareness
+            "green_aware": green_aware,
+            # Research Question 2: Known GSF patterns
+            "gsf_patterns_matched": matched_patterns,
+            "pattern_count": len(matched_patterns),
+            "pattern_details": pattern_details,
+            "confidence": confidence,
+            # Additional metadata
+            "files_modified": commit.get("modified_files", commit.get("files_changed", [])),
+            "insertions": commit.get("insertions", commit.get("lines_added", 0)),
+            "deletions": commit.get("deletions", commit.get("lines_deleted", 0)),
+            "lines_deleted": commit.get("lines_deleted", 0),
+        }
+
+    def _check_green_awareness(self, message: str, files: list[str]) -> tuple[bool, Optional[str]]:
+        """Check if commit explicitly mentions green/energy concerns.
+
+        Args:
+            message: Commit message (lowercase)
+            files: List of changed files (lowercase)
+
+        Returns:
+            Tuple of (is_green_aware, evidence_text)
+        """
+        # Check message for green keywords
+        for keyword in self.GREEN_KEYWORDS:
+            if keyword in message:
+                # Extract context around keyword
+                pattern = rf".{{0,30}}{re.escape(keyword)}.{{0,30}}"
+                match = re.search(pattern, message, re.IGNORECASE)
+                if match:
+                    evidence = match.group(0).strip()
+                    return True, f"Keyword '{keyword}': {evidence}"
+
+        # Check file names for patterns
+        cache_files = [f for f in files if "cache" in f or "redis" in f]
+        if cache_files:
+            return True, f"Modified cache-related file: {cache_files[0]}"
+
+        perf_files = [f for f in files if "performance" in f or "optimization" in f]
+        if perf_files:
+            return True, f"Modified performance file: {perf_files[0]}"
+
+        return False, None
+
+    def _detect_known_pattern(self, message: str, files: list[str]) -> tuple[Optional[str], str]:
+        """Detect known green software pattern.
+
+        Args:
+            message: Commit message (lowercase)
+            files: List of changed files (lowercase)
+
+        Returns:
+            Tuple of (pattern_name, confidence_level)
+        """
+        matches = []
+
+        # Check each pattern
+        for pattern_name, keywords in self.GREEN_PATTERNS.items():
+            for keyword in keywords:
+                if keyword in message:
+                    # Calculate confidence based on specificity
+                    confidence = "HIGH" if len(keyword) > 10 else "MEDIUM"
+                    matches.append((pattern_name, confidence, len(keyword)))
+
+        # Check file names for pattern hints
+        all_files = " ".join(files)
+        for pattern_name, keywords in self.GREEN_PATTERNS.items():
+            for keyword in keywords:
+                if keyword in all_files:
+                    matches.append((pattern_name, "MEDIUM", len(keyword)))
+
+        if not matches:
+            return "NONE DETECTED", "NONE"
+
+        # Return most specific match (longest keyword)
+        matches.sort(key=lambda x: x[2], reverse=True)
+        return matches[0][0], matches[0][1]
+
+    def save_results(self, results: list[dict[str, Any]], output_file: Path):
+        """Save analysis results to JSON file.
+
+        Args:
+            results: List of analysis results
+            output_file: Output file path
+        """
+        # Calculate summary statistics
+        green_aware_count = sum(1 for r in results if r["green_aware"])
+        pattern_counts = Counter(
+            r["known_pattern"] for r in results if r["known_pattern"] != "NONE DETECTED"
+        )
+
+        data = {
+            "metadata": {
+                "analyzed_at": format_timestamp(),
+                "total_commits_analyzed": len(results),
+                "green_aware_commits": green_aware_count,
+                "green_aware_percentage": (
+                    round(green_aware_count / len(results) * 100, 2) if results else 0
+                ),
+                "analyzer_type": "keyword_heuristic",
+                "note": "This analysis uses keyword and heuristic matching. For AI-powered analysis, use Claude API.",
+            },
+            "results": results,
+        }
+
+        save_json_file(data, output_file)
+        colored_print(f"Saved analysis for {len(results)} commits to {output_file}", "green")
+
+        # Display summary
+        colored_print("\n📊 Analysis Summary:", "cyan")
+        colored_print(
+            f"  Green-aware commits: {green_aware_count} ({data['metadata']['green_aware_percentage']}%)",
+            "white",
+        )
+        if pattern_counts:
+            colored_print("\n  Top patterns detected:", "cyan")
+            for pattern, count in pattern_counts.most_common(5):
+                colored_print(f"    - {pattern}: {count}", "white")
+
+
+@click.command()
+@click.option("--batch-size", default=10, help="Batch size for processing")
+@click.option("--resume", is_flag=True, help="Resume from checkpoint")
+@click.option(
+    "--commits-file", default=None, help="Input commits file (default: data/commits.json)"
+)
+@click.option(
+    "--output", default=None, help="Output file path (default: data/analysis_results.json)"
+)
+@click.option("--config-file", default=".env", help="Path to .env configuration file")
+def analyze(
+    batch_size: int,
+    resume: bool,
+    commits_file: Optional[str],
+    output: Optional[str],
+    config_file: str,
+):
+    """Analyze commits for green software practices."""
+    print_banner("Data Analyzer")
+
+    try:
+        # Load configuration
+        config = get_config(config_file)
+
+        # Determine input/output files
+        input_file = Path(commits_file) if commits_file else config.COMMITS_FILE
+        output_file = Path(output) if output else config.ANALYSIS_FILE
+
+        # Check if input file exists
+        if not input_file.exists():
+            colored_print(f"Input file not found: {input_file}", "red")
+            colored_print("Please run 'extract' command first to extract commits", "yellow")
+            exit(1)
+
+        # Load commits
+        colored_print(f"Loading commits from {input_file}...", "blue")
+        data = load_json_file(input_file)
+        commits = data.get("commits", [])
+
+        if not commits:
+            colored_print("No commits found in input file", "yellow")
+            exit(1)
+
+        colored_print(f"Loaded {len(commits)} commits", "green")
+
+        # Check for resume
+        resume_from = 0
+        if resume:
+            checkpoint_data = load_checkpoint(config.CHECKPOINT_FILE)
+            if checkpoint_data:
+                resume_from = checkpoint_data.get("processed_count", 0)
+                colored_print(
+                    f"Resuming from checkpoint: {resume_from} commits processed", "yellow"
+                )
+
+        # Initialize analyzer
+        analyzer = DataAnalyzer(batch_size=batch_size)
+
+        # Analyze commits
+        results = analyzer.analyze_commits(commits, resume_from=resume_from)
+
+        if not results:
+            colored_print("No analysis results generated", "yellow")
+            exit(1)
+
+        # Save results
+        analyzer.save_results(results, output_file)
+
+        # Save checkpoint
+        create_checkpoint(
+            config.CHECKPOINT_FILE,
+            {"processed_count": len(results), "timestamp": format_timestamp()},
+        )
+
+        colored_print(f"\n✓ Successfully analyzed {len(results)} commits", "green")
+        colored_print(f"Output saved to: {output_file}", "green")
+
+    except FileNotFoundError as e:
+        colored_print(f"File not found: {e}", "red")
+        exit(1)
+    except json.JSONDecodeError:
+        colored_print(f"Invalid JSON in input file: {input_file}", "red")
+        exit(1)
+    except Exception as e:
+        colored_print(f"Error: {e}", "red")
+        import traceback
+
+        traceback.print_exc()
+        exit(1)
+
+
+if __name__ == "__main__":
+    analyze()

greenmining/services/github_fetcher.py

@@ -0,0 +1,266 @@
+"""GitHub repository fetcher for green microservices mining."""
+
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional
+
+import click
+from github import Github, GithubException, RateLimitExceededException
+from tqdm import tqdm
+
+from greenmining.config import get_config
+from greenmining.utils import (
+    colored_print,
+    format_timestamp,
+    print_banner,
+    save_json_file,
+)
+
+
+class GitHubFetcher:
+    """Fetches microservice repositories from GitHub."""
+
+    def __init__(
+        self,
+        token: str,
+        max_repos: int = 100,
+        min_stars: int = 100,
+        languages: Optional[list[str]] = None,
+    ):
+        """Initialize GitHub fetcher.
+
+        Args:
+            token: GitHub personal access token
+            max_repos: Maximum number of repositories to fetch
+            min_stars: Minimum number of stars required
+            languages: List of programming languages to filter
+        """
+        self.github = Github(token)
+        self.max_repos = max_repos
+        self.min_stars = min_stars
+        self.languages = languages or [
+            "Java",
+            "Python",
+            "Go",
+            "JavaScript",
+            "TypeScript",
+            "C#",
+            "Rust",
+        ]
+
+    def search_repositories(self) -> list[dict[str, Any]]:
+        """Search for microservice repositories.
+
+        Returns:
+            List of repository metadata dictionaries
+        """
+        repositories = []
+        keywords = ["microservices", "microservice-architecture", "cloud-native"]
+
+        colored_print(f"Searching for repositories with keywords: {', '.join(keywords)}", "cyan")
+        colored_print(
+            f"Filters: min_stars={self.min_stars}, languages={', '.join(self.languages)}", "cyan"
+        )
+
+        # Build search query
+        keyword_query = " OR ".join(keywords)
+        language_query = " OR ".join([f"language:{lang}" for lang in self.languages])
+        query = f"({keyword_query}) ({language_query}) stars:>={self.min_stars}"
+
+        try:
+            # Execute search
+            search_results = self.github.search_repositories(
+                query=query, sort="stars", order="desc"
+            )
+
+            total_found = search_results.totalCount
+            colored_print(f"Found {total_found} repositories matching criteria", "green")
+
+            # Fetch repository details with progress bar
+            with tqdm(
+                total=min(self.max_repos, total_found), desc="Fetching repositories", unit="repo"
+            ) as pbar:
+                for idx, repo in enumerate(search_results):
+                    if idx >= self.max_repos:
+                        break
+
+                    try:
+                        repo_data = self._extract_repo_metadata(repo, idx + 1)
+                        repositories.append(repo_data)
+                        pbar.update(1)
+                    except GithubException as e:
+                        colored_print(f"Error fetching {repo.full_name}: {e}", "yellow")
+                        continue
+                    except RateLimitExceededException:
+                        colored_print("Rate limit exceeded. Waiting...", "red")
+                        self._handle_rate_limit()
+                        continue
+
+            return repositories
+
+        except GithubException as e:
+            colored_print(f"GitHub API error: {e}", "red")
+            raise
+        except Exception as e:
+            colored_print(f"Unexpected error: {e}", "red")
+            raise
+
+    def _extract_repo_metadata(self, repo, repo_id: int) -> dict[str, Any]:
+        """Extract metadata from repository object.
+
+        Args:
+            repo: GitHub repository object
+            repo_id: Sequential repository ID
+
+        Returns:
+            Dictionary with repository metadata
+        """
+        return {
+            "repo_id": repo_id,
+            "name": repo.name,
+            "owner": repo.owner.login,
+            "full_name": repo.full_name,
+            "url": repo.html_url,
+            "clone_url": repo.clone_url,
+            "language": repo.language,
+            "stars": repo.stargazers_count,
+            "forks": repo.forks_count,
+            "watchers": repo.watchers_count,
+            "open_issues": repo.open_issues_count,
+            "last_updated": repo.updated_at.isoformat() if repo.updated_at else None,
+            "created_at": repo.created_at.isoformat() if repo.created_at else None,
+            "description": repo.description or "",
+            "main_branch": repo.default_branch,
+            "topics": repo.get_topics() if hasattr(repo, "get_topics") else [],
+            "size": repo.size,
+            "has_issues": repo.has_issues,
+            "has_wiki": repo.has_wiki,
+            "archived": repo.archived,
+            "license": repo.license.name if repo.license else None,
+        }
+
+    def _handle_rate_limit(self):
+        """Handle GitHub API rate limiting."""
+        rate_limit = self.github.get_rate_limit()
+        reset_time = rate_limit.core.reset
+        wait_seconds = (reset_time - datetime.now()).total_seconds()
+
+        if wait_seconds > 0:
+            colored_print(f"Rate limit will reset in {wait_seconds:.0f} seconds", "yellow")
+            import time
+
+            time.sleep(min(wait_seconds + 10, 60))  # Wait with max 60 seconds
+
+    def save_results(self, repositories: list[dict[str, Any]], output_file: Path):
+        """Save fetched repositories to JSON file.
+
+        Args:
+            repositories: List of repository metadata
+            output_file: Output file path
+        """
+        data = {
+            "metadata": {
+                "fetched_at": format_timestamp(),
+                "total_repos": len(repositories),
+                "min_stars": self.min_stars,
+                "languages": self.languages,
+                "search_keywords": ["microservices", "microservice-architecture", "cloud-native"],
+            },
+            "repositories": repositories,
+        }
+
+        save_json_file(data, output_file)
+        colored_print(f"Saved {len(repositories)} repositories to {output_file}", "green")
+
+
+@click.command()
+@click.option("--max-repos", default=100, help="Maximum number of repositories to fetch")
+@click.option("--min-stars", default=100, help="Minimum stars required")
+@click.option(
+    "--languages",
+    default="java,python,go,javascript,typescript,csharp,rust",
+    help="Comma-separated list of languages",
+)
+@click.option("--output", default=None, help="Output file path (default: data/repositories.json)")
+@click.option("--config-file", default=".env", help="Path to .env configuration file")
+def fetch(max_repos: int, min_stars: int, languages: str, output: Optional[str], config_file: str):
+    """Fetch top microservice repositories from GitHub."""
+    print_banner("GitHub Repository Fetcher")
+
+    try:
+        # Load configuration
+        config = get_config(config_file)
+
+        # Parse languages
+        language_list = [lang.strip().title() for lang in languages.split(",")]
+
+        # Map common language names
+        language_map = {"Nodejs": "JavaScript", "Csharp": "C#", "Typescript": "TypeScript"}
+        language_list = [language_map.get(lang, lang) for lang in language_list]
+
+        # Determine output file
+        output_file = Path(output) if output else config.REPOS_FILE
+
+        colored_print(f"Fetching up to {max_repos} repositories...", "blue")
+
+        # Initialize fetcher
+        fetcher = GitHubFetcher(
+            token=config.GITHUB_TOKEN,
+            max_repos=max_repos,
+            min_stars=min_stars,
+            languages=language_list,
+        )
+
+        # Search and fetch repositories
+        repositories = fetcher.search_repositories()
+
+        if not repositories:
+            colored_print("No repositories found matching criteria", "yellow")
+            return
+
+        # Save results
+        fetcher.save_results(repositories, output_file)
+
+        # Display summary
+        colored_print(f"\n✓ Successfully fetched {len(repositories)} repositories", "green")
+        colored_print(f"Output saved to: {output_file}", "green")
+
+        # Show top 5 repos
+        colored_print("\nTop 5 repositories by stars:", "cyan")
+        from tabulate import tabulate
+
+        top_repos = sorted(repositories, key=lambda x: x["stars"], reverse=True)[:5]
+        table_data = [
+            [
+                repo["full_name"],
+                repo["language"],
+                f"{repo['stars']:,}",
+                repo["description"][:50] + "...",
+            ]
+            for repo in top_repos
+        ]
+        print(
+            tabulate(
+                table_data,
+                headers=["Repository", "Language", "Stars", "Description"],
+                tablefmt="simple",
+            )
+        )
+
+    except ValueError as e:
+        colored_print(f"Configuration error: {e}", "red")
+        colored_print("Please check your .env file and ensure GITHUB_TOKEN is set", "yellow")
+        exit(1)
+    except GithubException as e:
+        colored_print(f"GitHub API error: {e}", "red")
+        exit(1)
+    except Exception as e:
+        colored_print(f"Error: {e}", "red")
+        import traceback
+
+        traceback.print_exc()
+        exit(1)
+
+
+if __name__ == "__main__":
+    fetch()