greenmining 0.1.12__py3-none-any.whl → 1.0.2__py3-none-any.whl

greenmining/services/data_aggregator.py

@@ -1,14 +1,22 @@
 """Data aggregator for green microservices analysis results."""
 
+from __future__ import annotations
+
 import json
 from collections import defaultdict
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional
 
 import click
 import pandas as pd
 
+from greenmining.analyzers import (
+    EnhancedStatisticalAnalyzer,
+    TemporalAnalyzer,
+    QualitativeAnalyzer,
+)
 from greenmining.config import get_config
+from greenmining.models.repository import Repository
 from greenmining.utils import (
     colored_print,
     format_number,
@@ -23,9 +31,35 @@ from greenmining.utils import (
 class DataAggregator:
     """Aggregates analysis results and generates statistics."""
 
-    def __init__(self):
-        """Initialize aggregator."""
-        pass
+    def __init__(
+        self,
+        enable_enhanced_stats: bool = False,
+        enable_temporal: bool = False,
+        temporal_granularity: str = "quarter",
+    ):
+        """Initialize aggregator.
+
+        Args:
+            enable_enhanced_stats: Enable enhanced statistical analysis
+            enable_temporal: Enable temporal trend analysis
+            temporal_granularity: Granularity for temporal analysis (day/week/month/quarter/year)
+        """
+        self.enable_enhanced_stats = enable_enhanced_stats
+        self.enable_temporal = enable_temporal
+
+        if self.enable_enhanced_stats:
+            self.statistical_analyzer = EnhancedStatisticalAnalyzer()
+            colored_print("Enhanced statistical analysis enabled", "cyan")
+        else:
+            self.statistical_analyzer = None
+
+        if self.enable_temporal:
+            self.temporal_analyzer = TemporalAnalyzer(granularity=temporal_granularity)
+            colored_print(
+                f"Temporal analysis enabled (granularity: {temporal_granularity})", "cyan"
+            )
+        else:
+            self.temporal_analyzer = None
 
     def aggregate(
         self, analysis_results: list[dict[str, Any]], repositories: list[dict[str, Any]]
@@ -56,7 +90,38 @@ class DataAggregator:
         # Per-language statistics
         per_language_stats = self._generate_language_stats(analysis_results, repositories)
 
-        return {
+        # Enhanced statistical analysis (if enabled)
+        enhanced_stats = None
+        if self.enable_enhanced_stats and len(analysis_results) > 0:
+            try:
+                enhanced_stats = self._generate_enhanced_statistics(analysis_results)
+                colored_print("✅ Enhanced statistical analysis complete", "green")
+            except Exception as e:
+                colored_print(f"⚠️  Enhanced statistics failed: {e}", "yellow")
+                enhanced_stats = {"error": str(e)}
+
+        # Temporal trend analysis (if enabled)
+        temporal_analysis = None
+        if self.enable_temporal and len(analysis_results) > 0:
+            try:
+                # Convert analysis results to commits format for temporal analyzer
+                commits = [
+                    {
+                        "hash": r.get("commit_hash", "unknown"),
+                        "date": r.get("date"),
+                        "message": r.get("message", ""),
+                        "repository": r.get("repository", "unknown"),
+                    }
+                    for r in analysis_results
+                ]
+
+                temporal_analysis = self.temporal_analyzer.analyze_trends(commits, analysis_results)
+                colored_print("✅ Temporal trend analysis complete", "green")
+            except Exception as e:
+                colored_print(f"⚠️  Temporal analysis failed: {e}", "yellow")
+                temporal_analysis = {"error": str(e)}
+
+        result = {
             "summary": summary,
             "known_patterns": known_patterns,
             "emergent_patterns": emergent_patterns,
@@ -64,6 +129,14 @@ class DataAggregator:
             "per_language_stats": per_language_stats,
         }
 
+        if enhanced_stats:
+            result["enhanced_statistics"] = enhanced_stats
+
+        if temporal_analysis:
+            result["temporal_analysis"] = temporal_analysis
+
+        return result
+
     def _generate_summary(
         self, results: list[dict[str, Any]], repos: list[dict[str, Any]]
     ) -> dict[str, Any]:
@@ -198,8 +271,13 @@ class DataAggregator:
         self, results: list[dict[str, Any]], repos: list[dict[str, Any]]
     ) -> list[dict[str, Any]]:
         """Generate per-language statistics."""
-        # Create repo name to language mapping
-        repo_language_map = {repo["full_name"]: repo.get("language", "Unknown") for repo in repos}
+        # Create repo name to language mapping (handle both Repository objects and dicts)
+        repo_language_map = {}
+        for repo in repos:
+            if isinstance(repo, Repository):
+                repo_language_map[repo.full_name] = repo.language or "Unknown"
+            else:
+                repo_language_map[repo["full_name"]] = repo.get("language", "Unknown")
 
         # Group commits by language
         language_commits = defaultdict(list)
@@ -228,6 +306,92 @@ class DataAggregator:
 
         return language_stats
 
+    def _generate_enhanced_statistics(self, results: list[dict[str, Any]]) -> dict[str, Any]:
+        """Generate enhanced statistical analysis.
+
+        Args:
+            results: List of commit analysis results
+
+        Returns:
+            Dictionary with enhanced statistical analysis
+        """
+        # Prepare DataFrame
+        df = pd.DataFrame(results)
+
+        # Ensure required columns exist
+        if "date" not in df.columns or "green_aware" not in df.columns:
+            return {"error": "Missing required columns for enhanced statistics"}
+
+        enhanced_stats = {}
+
+        # 1. Temporal Trend Analysis
+        if len(df) >= 8:  # Need at least 8 data points
+            try:
+                df_copy = df.copy()
+                df_copy["commit_hash"] = df_copy.get("commit_hash", df_copy.index)
+                trends = self.statistical_analyzer.temporal_trend_analysis(df_copy)
+                enhanced_stats["temporal_trends"] = {
+                    "trend_direction": trends["trend"]["direction"],
+                    "correlation": float(trends["trend"]["correlation"]),
+                    "p_value": float(trends["trend"]["p_value"]),
+                    "significant": trends["trend"]["significant"],
+                    "monthly_data_points": len(trends.get("monthly_data", {})),
+                }
+            except Exception as e:
+                enhanced_stats["temporal_trends"] = {"error": str(e)}
+
+        # 2. Pattern Correlation Analysis (if pattern columns exist)
+        pattern_cols = [col for col in df.columns if col.startswith("pattern_")]
+        if pattern_cols and len(pattern_cols) >= 2:
+            try:
+                correlations = self.statistical_analyzer.analyze_pattern_correlations(df)
+                enhanced_stats["pattern_correlations"] = {
+                    "significant_pairs_count": len(correlations["significant_pairs"]),
+                    "significant_pairs": correlations["significant_pairs"][:5],  # Top 5
+                    "interpretation": correlations["interpretation"],
+                }
+            except Exception as e:
+                enhanced_stats["pattern_correlations"] = {"error": str(e)}
+
+        # 3. Effect Size Analysis by Repository
+        if "repository" in df.columns:
+            try:
+                # Group by repository
+                green_rates_by_repo = df.groupby("repository")["green_aware"].mean()
+                if len(green_rates_by_repo) >= 2:
+                    # Compare top vs bottom half
+                    sorted_rates = sorted(green_rates_by_repo.values)
+                    mid_point = len(sorted_rates) // 2
+                    group1 = sorted_rates[:mid_point]
+                    group2 = sorted_rates[mid_point:]
+
+                    if len(group1) > 0 and len(group2) > 0:
+                        effect = self.statistical_analyzer.effect_size_analysis(
+                            list(group1), list(group2)
+                        )
+                        enhanced_stats["effect_size"] = {
+                            "cohens_d": float(effect["cohens_d"]),
+                            "magnitude": effect["magnitude"],
+                            "mean_difference": float(effect["mean_difference"]),
+                            "significant": effect["significant"],
+                            "comparison": "high_green_vs_low_green_repos",
+                        }
+            except Exception as e:
+                enhanced_stats["effect_size"] = {"error": str(e)}
+
+        # 4. Basic descriptive statistics
+        enhanced_stats["descriptive"] = {
+            "total_commits": len(df),
+            "green_commits": int(df["green_aware"].sum()),
+            "green_rate_mean": float(df["green_aware"].mean()),
+            "green_rate_std": float(df["green_aware"].std()) if len(df) > 1 else 0.0,
+            "unique_repositories": (
+                int(df["repository"].nunique()) if "repository" in df.columns else 0
+            ),
+        }
+
+        return enhanced_stats
+
     def save_results(
         self,
         aggregated_data: dict[str, Any],

greenmining/services/data_analyzer.py

@@ -1,14 +1,21 @@
 """Data analyzer for green microservices commits using GSF patterns."""
 
+from __future__ import annotations
+
 import json
 import re
 from collections import Counter
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import click
 from tqdm import tqdm
 
+from greenmining.analyzers import (
+    CodeDiffAnalyzer,
+    NLPAnalyzer,
+    MLFeatureExtractor,
+)
 from greenmining.config import get_config
 from greenmining.gsf_patterns import (
     GREEN_KEYWORDS,
@@ -30,16 +37,49 @@ from greenmining.utils import (
 class DataAnalyzer:
     """Analyzes commits for green software patterns using GSF (Green Software Foundation) patterns."""
 
-    def __init__(self, batch_size: int = 10):
+    def __init__(
+        self,
+        batch_size: int = 10,
+        enable_diff_analysis: bool = False,
+        enable_nlp: bool = False,
+        enable_ml_features: bool = False,
+    ):
         """Initialize analyzer with GSF patterns.
 
         Args:
             batch_size: Number of commits to process in each batch
+            enable_diff_analysis: Enable code diff analysis (slower but more accurate)
+            enable_nlp: Enable NLP-enhanced pattern detection
+            enable_ml_features: Enable ML feature extraction
         """
         # Use GSF patterns from gsf_patterns.py
         self.gsf_patterns = GSF_PATTERNS
         self.green_keywords = GREEN_KEYWORDS
         self.batch_size = batch_size
+        self.enable_diff_analysis = enable_diff_analysis
+        self.enable_nlp = enable_nlp
+        self.enable_ml_features = enable_ml_features
+
+        # Initialize code diff analyzer if enabled
+        if self.enable_diff_analysis:
+            self.diff_analyzer = CodeDiffAnalyzer()
+            colored_print("Code diff analysis enabled (may increase processing time)", "cyan")
+        else:
+            self.diff_analyzer = None
+
+        # Initialize NLP analyzer if enabled
+        if self.enable_nlp:
+            self.nlp_analyzer = NLPAnalyzer(enable_stemming=True, enable_synonyms=True)
+            colored_print("NLP analysis enabled (morphological variants + synonyms)", "cyan")
+        else:
+            self.nlp_analyzer = None
+
+        # Initialize ML feature extractor if enabled
+        if self.enable_ml_features:
+            self.ml_extractor = MLFeatureExtractor(green_keywords=list(GREEN_KEYWORDS))
+            colored_print("ML feature extraction enabled", "cyan")
+        else:
+            self.ml_extractor = None
 
     def analyze_commits(
         self, commits: list[dict[str, Any]], resume_from: int = 0
@@ -91,6 +131,42 @@ class DataAnalyzer:
         # Q2: KNOWN GSF PATTERNS - Match against Green Software Foundation patterns
         matched_patterns = get_pattern_by_keywords(message)
 
+        # Enhanced NLP analysis (if enabled)
+        nlp_results = None
+        if self.nlp_analyzer:
+            nlp_results = self.nlp_analyzer.analyze_text(message, list(self.green_keywords))
+
+            # Check if NLP found additional matches not caught by keyword matching
+            has_nlp_matches, additional_terms = self.nlp_analyzer.enhance_pattern_detection(
+                message, matched_patterns
+            )
+
+            if has_nlp_matches:
+                # NLP enhancement found additional evidence
+                green_aware = True
+
+        # Q3: CODE DIFF ANALYSIS (if enabled and diff data available)
+        diff_analysis = None
+        if self.diff_analyzer and commit.get("diff_data"):
+            try:
+                # Note: This requires commit object from PyDriller
+                # For now, we'll store a placeholder for future integration
+                diff_analysis = {
+                    "enabled": True,
+                    "status": "requires_pydriller_commit_object",
+                    "patterns_detected": [],
+                    "confidence": "none",
+                    "evidence": {},
+                    "metrics": {},
+                }
+            except Exception as e:
+                diff_analysis = {
+                    "enabled": True,
+                    "status": f"error: {str(e)}",
+                    "patterns_detected": [],
+                    "confidence": "none",
+                }
+
         # Get detailed pattern info
         pattern_details = []
         for _pattern_id, pattern in self.gsf_patterns.items():
@@ -105,13 +181,14 @@ class DataAnalyzer:
                 )
 
         # Calculate confidence based on number of patterns matched
-        confidence = (
-            "high"
-            if len(matched_patterns) >= 2
-            else "medium" if len(matched_patterns) == 1 else "low"
-        )
+        # Boost confidence if diff analysis also detected patterns
+        pattern_count = len(matched_patterns)
+        if diff_analysis and diff_analysis.get("patterns_detected"):
+            pattern_count += len(diff_analysis["patterns_detected"])
+
+        confidence = "high" if pattern_count >= 2 else "medium" if pattern_count == 1 else "low"
 
-        return {
+        result = {
             "commit_hash": commit.get("hash", commit.get("commit_id", "unknown")),
             "repository": commit.get("repository", commit.get("repo_name", "unknown")),
             "author": commit.get("author", commit.get("author_name", "unknown")),
@@ -130,6 +207,32 @@ class DataAnalyzer:
             "deletions": commit.get("lines_deleted", commit.get("deletions", 0)),
         }
 
+        # Add diff analysis results if available
+        if diff_analysis:
+            result["diff_analysis"] = diff_analysis
+
+        # Add NLP analysis results if available
+        if nlp_results:
+            result["nlp_analysis"] = {
+                "total_matches": nlp_results["total_nlp_matches"],
+                "match_density": nlp_results["match_density"],
+                "morphological_count": len(nlp_results["morphological_matches"]),
+                "semantic_count": len(nlp_results["semantic_matches"]),
+                "phrase_count": len(nlp_results["phrase_matches"]),
+            }
+
+        # Add ML features if enabled
+        if self.enable_ml_features and self.ml_extractor:
+            # Note: Full feature extraction requires repository context
+            # For now, extract basic text features
+            text_features = self.ml_extractor.extract_text_features(message)
+            result["ml_features"] = {
+                "text": text_features,
+                "note": "Full ML features require repository and historical context",
+            }
+
+        return result
+
     def _check_green_awareness(self, message: str, files: list[str]) -> tuple[bool, Optional[str]]:
         """Check if commit explicitly mentions green/energy concerns.
 

greenmining/services/github_fetcher.py

@@ -1,8 +1,10 @@
 """GitHub repository fetcher for green microservices mining."""
 
+from __future__ import annotations
+
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional
 
 import click
 from github import Github, GithubException, RateLimitExceededException
@@ -26,6 +28,10 @@ class GitHubFetcher:
         max_repos: int = 100,
         min_stars: int = 100,
         languages: Optional[list[str]] = None,
+        created_after: Optional[str] = None,
+        created_before: Optional[str] = None,
+        pushed_after: Optional[str] = None,
+        pushed_before: Optional[str] = None,
     ):
         """Initialize GitHub fetcher.
 
@@ -34,6 +40,10 @@ class GitHubFetcher:
             max_repos: Maximum number of repositories to fetch
             min_stars: Minimum number of stars required
             languages: List of programming languages to filter
+            created_after: Repository created after date (YYYY-MM-DD)
+            created_before: Repository created before date (YYYY-MM-DD)
+            pushed_after: Repository pushed after date (YYYY-MM-DD)
+            pushed_before: Repository pushed before date (YYYY-MM-DD)
         """
         self.github = Github(token)
         self.max_repos = max_repos
@@ -47,6 +57,10 @@ class GitHubFetcher:
             "C#",
             "Rust",
         ]
+        self.created_after = created_after
+        self.created_before = created_before
+        self.pushed_after = pushed_after
+        self.pushed_before = pushed_before
 
     def search_repositories(self) -> list[dict[str, Any]]:
         """Search for microservice repositories.
@@ -62,10 +76,8 @@ class GitHubFetcher:
             f"Filters: min_stars={self.min_stars}, languages={', '.join(self.languages)}", "cyan"
         )
 
-        # Build search query
-        keyword_query = " OR ".join(keywords)
-        language_query = " OR ".join([f"language:{lang}" for lang in self.languages])
-        query = f"({keyword_query}) ({language_query}) stars:>={self.min_stars}"
+        # Build search query with temporal filters
+        query = self._build_temporal_query(keywords)
 
         try:
             # Execute search
@@ -139,6 +151,51 @@ class GitHubFetcher:
             "license": repo.license.name if repo.license else None,
         }
 
+    def _build_temporal_query(self, keywords: list[str]) -> str:
+        """
+        Build GitHub search query with temporal constraints.
+
+        Args:
+            keywords: List of search keywords
+
+        Returns:
+            Complete search query string
+        """
+        query_parts = []
+
+        # Keywords
+        keyword_query = " OR ".join(keywords)
+        query_parts.append(f"({keyword_query})")
+
+        # Languages
+        language_query = " OR ".join([f"language:{lang}" for lang in self.languages])
+        query_parts.append(f"({language_query})")
+
+        # Stars
+        query_parts.append(f"stars:>={self.min_stars}")
+
+        # Archived filter
+        query_parts.append("archived:false")
+
+        # Temporal filters
+        if self.created_after and self.created_before:
+            query_parts.append(f"created:{self.created_after}..{self.created_before}")
+        elif self.created_after:
+            query_parts.append(f"created:>={self.created_after}")
+        elif self.created_before:
+            query_parts.append(f"created:<={self.created_before}")
+
+        if self.pushed_after and self.pushed_before:
+            query_parts.append(f"pushed:{self.pushed_after}..{self.pushed_before}")
+        elif self.pushed_after:
+            query_parts.append(f"pushed:>={self.pushed_after}")
+        elif self.pushed_before:
+            query_parts.append(f"pushed:<={self.pushed_before}")
+
+        query = " ".join(query_parts)
+        colored_print(f"Query: {query}", "cyan")
+        return query
+
     def _handle_rate_limit(self):
         """Handle GitHub API rate limiting."""
         rate_limit = self.github.get_rate_limit()

greenmining/services/reports.py

@@ -1,9 +1,11 @@
-"""Report generator for green microservices analysis."""
+"""Report generation for green mining analysis."""
+
+from __future__ import annotations
 
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, Optional
 
 import click
 
@@ -180,6 +182,11 @@ Commits were analyzed using a keyword and heuristic-based classification framewo
         # 2.4 Per-Repository Analysis
         sections.append(self._generate_repo_analysis_section(data))
 
+        # 2.5 Enhanced Statistics (if available)
+        enhanced_section = self._generate_enhanced_statistics_section(data)
+        if enhanced_section:
+            sections.append(enhanced_section)
+
         return "### 2. Results\n\n" + "\n\n".join(sections)
 
     def _generate_green_awareness_section(self, data: dict[str, Any]) -> str:
@@ -300,6 +307,120 @@ No novel microservice-specific green practices were automatically detected. Manu
 
 **Repositories with No Green Mentions:** {no_green_count} out of {len(per_repo)} repositories had zero green-aware commits."""
 
+    def _generate_enhanced_statistics_section(self, data: dict[str, Any]) -> str:
+        """Generate enhanced statistical analysis subsection.
+
+        Args:
+            data: Aggregated data containing enhanced_statistics field
+
+        Returns:
+            Markdown section with enhanced statistics
+        """
+        enhanced_stats = data.get("enhanced_statistics")
+
+        if not enhanced_stats:
+            return ""
+
+        # Handle error case
+        if "error" in enhanced_stats:
+            return f"""#### 2.5 Enhanced Statistical Analysis
+
+**Note:** Enhanced statistical analysis encountered an error: {enhanced_stats['error']}
+"""
+
+        sections = []
+        sections.append("#### 2.5 Enhanced Statistical Analysis")
+        sections.append("")
+        sections.append(
+            "This section presents advanced statistical analyses of green software engineering patterns."
+        )
+        sections.append("")
+
+        # Temporal trends
+        temporal = enhanced_stats.get("temporal_trends", {})
+        if temporal and "error" not in temporal:
+            sections.append("##### Temporal Trends")
+            sections.append("")
+
+            if "overall_trend" in temporal:
+                trend_dir = temporal["overall_trend"].get("direction", "unknown")
+                trend_sig = temporal["overall_trend"].get("significant", False)
+                sections.append(f"**Overall Trend:** {trend_dir.capitalize()}")
+                if trend_sig:
+                    sections.append(" (statistically significant)")
+                sections.append("")
+
+            if "monthly_stats" in temporal and temporal["monthly_stats"]:
+                sections.append("**Monthly Pattern Statistics:**")
+                sections.append("")
+                monthly = temporal["monthly_stats"]
+                sections.append(f"- Mean commits/month: {format_number(monthly.get('mean', 0))}")
+                sections.append(
+                    f"- Median commits/month: {format_number(monthly.get('median', 0))}"
+                )
+                sections.append(f"- Std deviation: {format_number(monthly.get('std', 0))}")
+                sections.append("")
+
+        # Pattern correlations
+        correlations = enhanced_stats.get("pattern_correlations", {})
+        if correlations and "error" not in correlations:
+            sections.append("##### Pattern Correlations")
+            sections.append("")
+
+            top_corr = correlations.get("top_positive_correlations", [])
+            if top_corr:
+                sections.append("**Top Positive Correlations (|r| > 0.5):**")
+                sections.append("")
+                sections.append("| Pattern 1 | Pattern 2 | Correlation (r) |")
+                sections.append("|-----------|-----------|-----------------|")
+                for corr in top_corr[:5]:
+                    sections.append(
+                        f"| {corr['pattern1']} | {corr['pattern2']} | {corr['correlation']:.3f} |"
+                    )
+                sections.append("")
+            else:
+                sections.append("No strong pattern correlations detected (|r| > 0.5).")
+                sections.append("")
+
+        # Effect sizes
+        effect_sizes = enhanced_stats.get("effect_size", {})
+        if effect_sizes and "error" not in effect_sizes:
+            sections.append("##### Effect Size Analysis")
+            sections.append("")
+
+            green_vs_nongreen = effect_sizes.get("green_vs_nongreen_patterns")
+            if green_vs_nongreen:
+                cohens_d = green_vs_nongreen.get("cohens_d", 0)
+                magnitude = green_vs_nongreen.get("magnitude", "negligible")
+                sections.append(f"**Green vs Non-Green Pattern Usage:**")
+                sections.append(f"- Cohen's d: {cohens_d:.3f}")
+                sections.append(f"- Effect magnitude: {magnitude.capitalize()}")
+                sections.append("")
+
+        # Descriptive statistics
+        descriptive = enhanced_stats.get("descriptive", {})
+        if descriptive and "error" not in descriptive:
+            sections.append("##### Descriptive Statistics")
+            sections.append("")
+
+            patterns = descriptive.get("patterns_per_commit", {})
+            if patterns:
+                sections.append("**Patterns per Commit:**")
+                sections.append(f"- Mean: {patterns.get('mean', 0):.2f}")
+                sections.append(f"- Median: {patterns.get('median', 0):.2f}")
+                sections.append(f"- Standard deviation: {patterns.get('std', 0):.2f}")
+                sections.append("")
+
+            repos = descriptive.get("green_commits_per_repo", {})
+            if repos:
+                sections.append("**Green Commits per Repository:**")
+                sections.append(f"- Mean: {repos.get('mean', 0):.2f}")
+                sections.append(f"- Median: {repos.get('median', 0):.2f}")
+                sections.append(f"- Standard deviation: {repos.get('std', 0):.2f}")
+                sections.append("")
+
+        return "\n".join(sections)
+
     def _generate_discussion(self, data: dict[str, Any]) -> str:
         """Generate discussion section."""
         summary = data["summary"]