greenmining 1.0.3__py3-none-any.whl → 1.0.4__py3-none-any.whl

greenmining/__init__.py

@@ -1,4 +1,4 @@
-"""Green Microservices Mining - GSF Pattern Analysis Tool."""
+# Green Microservices Mining - GSF Pattern Analysis Tool.
 
 from greenmining.config import Config
 from greenmining.controllers.repository_controller import RepositoryController
@@ -9,44 +9,26 @@ from greenmining.gsf_patterns import (
     is_green_aware,
 )
 
-__version__ = "1.0.3"
+__version__ = "1.0.4"
 
 
 def fetch_repositories(
     github_token: str,
-    max_repos: int = 100,
-    min_stars: int = 100,
+    max_repos: int = None,
+    min_stars: int = None,
     languages: list = None,
-    keywords: str = "microservices",
+    keywords: str = None,
 ):
-    """Fetch repositories from GitHub with custom search keywords.
-
-    Args:
-        github_token: GitHub personal access token
-        max_repos: Maximum number of repositories to fetch (default: 100)
-        min_stars: Minimum GitHub stars required (default: 100)
-        languages: List of programming languages to filter (default: ["Python", "Java", "Go", "JavaScript", "TypeScript"])
-        keywords: Search keywords (default: "microservices")
-
-    Returns:
-        List of Repository model instances
-
-    Example:
-        >>> from greenmining import fetch_repositories
-        >>> repos = fetch_repositories(
-        ...     github_token="your_token",
-        ...     max_repos=50,
-        ...     keywords="kubernetes cloud-native",
-        ...     min_stars=500
-        ... )
-        >>> print(f"Found {len(repos)} repositories")
-    """
+    # Fetch repositories from GitHub with custom search keywords.
     config = Config()
     config.GITHUB_TOKEN = github_token
     controller = RepositoryController(config)
 
     return controller.fetch_repositories(
-        max_repos=max_repos, min_stars=min_stars, languages=languages, keywords=keywords
+        max_repos=max_repos,
+        min_stars=min_stars,
+        languages=languages,
+        keywords=keywords,
     )
 
 
@@ -58,4 +40,4 @@ __all__ = [
     "get_pattern_by_keywords",
     "fetch_repositories",
     "__version__",
-]
+]

greenmining/__main__.py

@@ -1,6 +1,12 @@
-"""Allow running greenmining as a module: python -m greenmining"""
+# Allow running greenmining as a module: python -m greenmining
+# This is a library - use Python API for programmatic access.
 
-from greenmining.cli import cli
+from greenmining import __version__
 
 if __name__ == "__main__":
-    cli()
+    print(f"greenmining v{__version__}")
+    print("This is a Python library for analyzing green software patterns.")
+    print("\nUsage:")
+    print("  from greenmining import GSF_PATTERNS, is_green_aware, get_pattern_by_keywords")
+    print("  from greenmining.services import GitHubFetcher, CommitExtractor, DataAnalyzer")
+    print("\nDocumentation: https://github.com/adam-bouafia/greenmining")

greenmining/__version__.py

@@ -1,3 +1,3 @@
-"""Version information for greenmining."""
+# Version information for greenmining.
 
-__version__ = "1.0.2"
+__version__ = "1.0.4"

greenmining/analyzers/__init__.py

@@ -1,17 +1,13 @@
-"""Analyzers for GreenMining framework."""
+# Analyzers for GreenMining framework.
 
 from .code_diff_analyzer import CodeDiffAnalyzer
-from .statistical_analyzer import EnhancedStatisticalAnalyzer
-from .nlp_analyzer import NLPAnalyzer
+from .statistical_analyzer import StatisticalAnalyzer
 from .temporal_analyzer import TemporalAnalyzer
 from .qualitative_analyzer import QualitativeAnalyzer
-from .ml_feature_extractor import MLFeatureExtractor
 
 __all__ = [
     "CodeDiffAnalyzer",
-    "EnhancedStatisticalAnalyzer",
-    "NLPAnalyzer",
+    "StatisticalAnalyzer",
     "TemporalAnalyzer",
     "QualitativeAnalyzer",
-    "MLFeatureExtractor",
 ]

greenmining/analyzers/code_diff_analyzer.py

@@ -1,4 +1,4 @@
-"""Code diff analyzer for detecting green software patterns in code changes."""
+# Code diff analyzer for detecting green software patterns in code changes.
 
 import re
 from typing import Any, Dict, List
@@ -7,10 +7,7 @@ from pydriller import Commit, ModifiedFile
 
 
 class CodeDiffAnalyzer:
-    """
-    Analyze code diffs to detect green software patterns
-    beyond commit message keywords.
-    """
+    # Analyze code diffs to detect green software patterns
 
     # Pattern indicators in code changes
     PATTERN_SIGNATURES = {
@@ -64,22 +61,154 @@ class CodeDiffAnalyzer:
             "keywords": [r"lazy", r"defer", r"\.only\(", r"select_related"],
             "patterns": [r"@lazy", r"LazyLoader", r"dynamic.*import"],
         },
+        # NEW: Serverless computing patterns
+        "serverless_computing": {
+            "providers": [
+                r"aws.*lambda",
+                r"@app\.route",
+                r"functions\.https",
+                r"azure.*function",
+            ],
+            "frameworks": [r"serverless", r"chalice", r"zappa", r"claudia"],
+            "keywords": [r"lambda_handler", r"cloud.*function", r"function.*app"],
+        },
+        # NEW: CDN and edge computing
+        "cdn_edge": {
+            "providers": [
+                r"cloudflare",
+                r"cloudfront",
+                r"fastly",
+                r"akamai",
+                r"cdn\.js",
+            ],
+            "keywords": [
+                r"edge.*cache",
+                r"cdn",
+                r"\.distribute\(",
+                r"edge.*function",
+            ],
+        },
+        # NEW: Compression patterns
+        "compression": {
+            "algorithms": [r"gzip", r"brotli", r"deflate", r"zstd", r"lz4"],
+            "keywords": [
+                r"compress",
+                r"decompress",
+                r"\.gz\b",
+                r"Content-Encoding",
+            ],
+            "libraries": [r"import gzip", r"import zlib", r"import brotli"],
+        },
+        # NEW: ML model optimization
+        "model_optimization": {
+            "techniques": [
+                r"quantize",
+                r"quantization",
+                r"prune",
+                r"pruning",
+                r"distill",
+            ],
+            "formats": [r"onnx", r"tensorrt", r"tflite", r"coreml"],
+            "keywords": [
+                r"int8",
+                r"fp16",
+                r"mixed.*precision",
+                r"model\.optimize",
+            ],
+        },
+        # NEW: Efficient protocols (HTTP/2, gRPC)
+        "efficient_protocols": {
+            "http2": [r"http2", r"http/2", r"h2", r"alpn"],
+            "grpc": [r"grpc", r"protobuf", r"\.proto\b"],
+            "keywords": [
+                r"stream",
+                r"multiplexing",
+                r"server.*push",
+                r"binary.*protocol",
+            ],
+        },
+        # NEW: Container optimization
+        "container_optimization": {
+            "base_images": [
+                r"FROM.*alpine",
+                r"FROM.*scratch",
+                r"FROM.*distroless",
+            ],
+            "techniques": [
+                r"multi-stage",
+                r"--no-install-recommends",
+                r"&&.*rm.*-rf",
+                r"\.dockerignore",
+            ],
+            "keywords": [r"layer.*cache", r"build.*cache", r"image.*size"],
+        },
+        # NEW: Green cloud regions
+        "green_regions": {
+            "regions": [
+                r"eu-west",
+                r"eu-north",
+                r"sweden",
+                r"norway",
+                r"canada",
+            ],
+            "keywords": [
+                r"renewable",
+                r"green.*region",
+                r"sustainable.*region",
+                r"carbon.*neutral",
+            ],
+        },
+        # NEW: Auto-scaling patterns
+        "auto_scaling": {
+            "kubernetes": [
+                r"HorizontalPodAutoscaler",
+                r"autoscaling/v",
+                r"hpa",
+                r"minReplicas",
+                r"maxReplicas",
+            ],
+            "cloud": [
+                r"auto.*scal",
+                r"scale.*to.*zero",
+                r"ScalingPolicy",
+                r"TargetTracking",
+            ],
+            "keywords": [
+                r"scale.*up",
+                r"scale.*down",
+                r"metrics.*server",
+                r"cpu.*utilization",
+            ],
+        },
+        # NEW: Code splitting and lazy loading (web)
+        "code_splitting": {
+            "webpack": [
+                r"dynamic.*import",
+                r"lazy.*load",
+                r"code.*split",
+                r"chunk",
+            ],
+            "react": [r"React\.lazy", r"Suspense", r"loadable"],
+            "keywords": [r"bundle", r"split.*chunk", r"async.*component"],
+        },
+        # NEW: Green ML training
+        "green_ml_training": {
+            "keywords": [
+                r"early.*stopping",
+                r"learning.*rate.*scheduler",
+                r"gradient.*checkpointing",
+                r"mixed.*precision",
+            ],
+            "frameworks": [
+                r"apex",
+                r"torch\.cuda\.amp",
+                r"tf\.keras\.mixed_precision",
+            ],
+        },
     }
 
     def analyze_commit_diff(self, commit: Commit) -> Dict[str, Any]:
-        """
-        Analyze code changes in a commit to detect green patterns.
-
-        Args:
-            commit: PyDriller Commit object
-
-        Returns:
-            Dictionary containing:
-            - patterns_detected: List of detected pattern names
-            - confidence: Confidence level (high/medium/low/none)
-            - evidence: Dictionary mapping patterns to evidence lines
-            - metrics: Code change metrics
-        """
+        # Analyze code changes in a commit to detect green patterns.
         patterns_detected = []
         evidence = {}
         metrics = self._calculate_metrics(commit)
@@ -116,15 +245,7 @@ class CodeDiffAnalyzer:
         }
 
     def _detect_patterns_in_line(self, code_line: str) -> List[str]:
-        """
-        Detect patterns in a single line of code.
-
-        Args:
-            code_line: Line of code to analyze
-
-        Returns:
-            List of detected pattern names
-        """
+        # Detect patterns in a single line of code.
         detected = []
 
         for pattern_name, signatures in self.PATTERN_SIGNATURES.items():
@@ -137,15 +258,7 @@ class CodeDiffAnalyzer:
         return detected
 
     def _calculate_metrics(self, commit: Commit) -> Dict[str, int]:
-        """
-        Calculate code change metrics.
-
-        Args:
-            commit: PyDriller Commit object
-
-        Returns:
-            Dictionary of metrics
-        """
+        # Calculate code change metrics.
         lines_added = sum(f.added_lines for f in commit.modified_files)
         lines_removed = sum(f.deleted_lines for f in commit.modified_files)
         files_changed = len(commit.modified_files)
@@ -165,22 +278,7 @@ class CodeDiffAnalyzer:
     def _calculate_diff_confidence(
         self, patterns: List[str], evidence: Dict[str, List[str]], metrics: Dict[str, int]
     ) -> str:
-        """
-        Calculate confidence level for diff-based detection.
-
-        Factors:
-        - Number of patterns detected
-        - Amount of evidence per pattern
-        - Code change magnitude
-
-        Args:
-            patterns: List of detected patterns
-            evidence: Dictionary mapping patterns to evidence
-            metrics: Code change metrics
-
-        Returns:
-            Confidence level: high/medium/low/none
-        """
+        # Calculate confidence level for diff-based detection.
         if not patterns:
             return "none"
 
@@ -194,15 +292,7 @@ class CodeDiffAnalyzer:
             return "low"
 
     def _is_code_file(self, modified_file: ModifiedFile) -> bool:
-        """
-        Check if file is a code file (not config, docs, etc.).
-
-        Args:
-            modified_file: PyDriller ModifiedFile object
-
-        Returns:
-            True if file is a code file
-        """
+        # Check if file is a code file (not config, docs, etc.).
         code_extensions = [
             ".py",
             ".java",

greenmining/analyzers/qualitative_analyzer.py

@@ -1,15 +1,4 @@
-"""
-Qualitative Analysis Framework for Pattern Validation
-
-Implements qualitative validation from Soliman et al. (2017):
-- Stratified random sampling for manual validation
-- Precision/recall calculation framework
-- Inter-rater reliability support
-- False positive/negative tracking
-
-Based on Soliman et al.: 42/151 studies used qualitative analysis
-Critical for: validating IR-based approaches, calculating accuracy metrics
-"""
+# Qualitative Analysis Framework for Pattern Validation
 
 from __future__ import annotations
 
@@ -22,7 +11,7 @@ import json
 
 @dataclass
 class ValidationSample:
-    """Represents a single validation sample"""
+    # Represents a single validation sample
 
     commit_sha: str
     commit_message: str
@@ -38,7 +27,7 @@ class ValidationSample:
 
 @dataclass
 class ValidationMetrics:
-    """Precision/recall metrics for validation"""
+    # Precision/recall metrics for validation
 
     true_positives: int
     false_positives: int
@@ -51,26 +40,10 @@ class ValidationMetrics:
 
 
 class QualitativeAnalyzer:
-    """
-    Framework for manual validation and qualitative analysis.
-
-    Implements:
-    1. Stratified sampling (ensure representation across categories)
-    2. Validation workflow (export → review → import → calculate metrics)
-    3. Precision/recall calculation
-    4. Inter-rater reliability (if multiple reviewers)
-
-    Based on Soliman et al.: "42 studies used qualitative analysis for validation"
-    """
+    # Framework for manual validation and qualitative analysis.
 
     def __init__(self, sample_size: int = 30, stratify_by: str = "pattern"):
-        """
-        Initialize qualitative analyzer.
-
-        Args:
-            sample_size: Number of commits to sample for validation
-            stratify_by: Stratification method ('pattern', 'repository', 'time', 'random')
-        """
+        # Initialize qualitative analyzer.
         self.sample_size = sample_size
         self.stratify_by = stratify_by
         self.samples: List[ValidationSample] = []
@@ -78,17 +51,7 @@ class QualitativeAnalyzer:
     def generate_validation_samples(
         self, commits: List[Dict], analysis_results: List[Dict], include_negatives: bool = True
     ) -> List[ValidationSample]:
-        """
-        Generate stratified validation samples.
-
-        Args:
-            commits: All commits
-            analysis_results: Pattern detection results
-            include_negatives: Include non-green commits for false negative detection
-
-        Returns:
-            List of ValidationSample objects
-        """
+        # Generate stratified validation samples.
         # Build commit lookup
         commit_lookup = {c.get("hash", c.get("sha")): c for c in commits}
 
@@ -141,7 +104,7 @@ class QualitativeAnalyzer:
         return samples
 
     def _stratified_sample_by_pattern(self, results: List[Dict], sample_size: int) -> List[Dict]:
-        """Stratified sampling ensuring each pattern category is represented."""
+        # Stratified sampling ensuring each pattern category is represented.
         # Group by dominant pattern
         pattern_groups = defaultdict(list)
         for result in results:
@@ -172,7 +135,7 @@ class QualitativeAnalyzer:
     def _stratified_sample_by_repo(
         self, results: List[Dict], commit_lookup: Dict, sample_size: int
     ) -> List[Dict]:
-        """Stratified sampling ensuring each repository is represented."""
+        # Stratified sampling ensuring each repository is represented.
         # Group by repository
         repo_groups = defaultdict(list)
         for result in results:
@@ -194,12 +157,7 @@ class QualitativeAnalyzer:
         return samples[:sample_size]
 
     def export_samples_for_review(self, output_path: str) -> None:
-        """
-        Export validation samples to JSON for manual review.
-
-        Args:
-            output_path: Path to output JSON file
-        """
+        # Export validation samples to JSON for manual review.
         samples_data = []
         for i, sample in enumerate(self.samples, 1):
             samples_data.append(
@@ -223,12 +181,7 @@ class QualitativeAnalyzer:
             json.dump(samples_data, f, indent=2)
 
     def import_validated_samples(self, input_path: str) -> None:
-        """
-        Import manually validated samples from JSON.
-
-        Args:
-            input_path: Path to JSON file with validated samples
-        """
+        # Import manually validated samples from JSON.
         with open(input_path, "r") as f:
             samples_data = json.load(f)
 
@@ -248,12 +201,7 @@ class QualitativeAnalyzer:
                     break
 
     def calculate_metrics(self) -> ValidationMetrics:
-        """
-        Calculate precision, recall, F1, and accuracy.
-
-        Returns:
-            ValidationMetrics object
-        """
+        # Calculate precision, recall, F1, and accuracy.
         # Count outcomes
         tp = 0  # True positive: detected as green, truly green
         fp = 0  # False positive: detected as green, not green
@@ -295,12 +243,7 @@ class QualitativeAnalyzer:
         )
 
     def get_validation_report(self) -> Dict:
-        """
-        Generate comprehensive validation report.
-
-        Returns:
-            Dictionary with validation statistics and metrics
-        """
+        # Generate comprehensive validation report.
         validated_count = sum(1 for s in self.samples if s.validation_status == "validated")
         pending_count = sum(1 for s in self.samples if s.validation_status == "pending")
 
@@ -360,7 +303,7 @@ class QualitativeAnalyzer:
         }
 
     def _analyze_pattern_accuracy(self) -> Dict:
-        """Analyze accuracy per pattern category."""
+        # Analyze accuracy per pattern category.
         pattern_stats = defaultdict(lambda: {"tp": 0, "fp": 0})
 
         for sample in self.samples:
@@ -391,16 +334,7 @@ class QualitativeAnalyzer:
         samples_from_reviewer_a: List[ValidationSample],
         samples_from_reviewer_b: List[ValidationSample],
     ) -> Dict:
-        """
-        Calculate inter-rater reliability (Cohen's Kappa).
-
-        Args:
-            samples_from_reviewer_a: Samples validated by reviewer A
-            samples_from_reviewer_b: Samples validated by reviewer B (same commits)
-
-        Returns:
-            Dictionary with Cohen's Kappa and agreement statistics
-        """
+        # Calculate inter-rater reliability (Cohen's Kappa).
         # Match samples by commit_sha
         matched_samples = []
         for sample_a in samples_from_reviewer_a:
@@ -445,7 +379,7 @@ class QualitativeAnalyzer:
         }
 
     def _interpret_kappa(self, kappa: float) -> str:
-        """Interpret Cohen's Kappa value."""
+        # Interpret Cohen's Kappa value.
         if kappa < 0:
             return "Poor (less than chance)"
         elif kappa < 0.20:

greenmining/analyzers/statistical_analyzer.py

@@ -1,4 +1,4 @@
-"""Enhanced statistical analyzer for green software patterns."""
+# Statistical analyzer for green software patterns.
 
 from __future__ import annotations
 
@@ -9,24 +9,11 @@ import pandas as pd
 from scipy import stats
 
 
-class EnhancedStatisticalAnalyzer:
-    """
-    Advanced statistical analyses for green software patterns.
-    Based on Soliman et al. quantitative validation techniques.
-    """
+class StatisticalAnalyzer:
+    # Advanced statistical analyses for green software patterns.
 
     def analyze_pattern_correlations(self, commit_data: pd.DataFrame) -> Dict[str, Any]:
-        """
-        Analyze correlations between patterns.
-
-        Question: Do repositories that adopt caching also adopt resource limits?
-
-        Args:
-            commit_data: DataFrame with pattern columns
-
-        Returns:
-            Dictionary containing correlation matrix and significant pairs
-        """
+        # Analyze correlations between patterns.
         # Create pattern co-occurrence matrix
         pattern_columns = [col for col in commit_data.columns if col.startswith("pattern_")]
 
@@ -61,20 +48,7 @@ class EnhancedStatisticalAnalyzer:
         }
 
     def temporal_trend_analysis(self, commits_df: pd.DataFrame) -> Dict[str, Any]:
-        """
-        Analyze temporal trends in green awareness.
-
-        Techniques:
-        - Mann-Kendall trend test (monotonic trend detection)
-        - Seasonal decomposition (identify cyclical patterns)
-        - Change point detection (identify sudden shifts)
-
-        Args:
-            commits_df: DataFrame with date and green_aware columns
-
-        Returns:
-            Dictionary containing trend analysis results
-        """
+        # Analyze temporal trends in green awareness.
         # Prepare time series data
         commits_df["date"] = pd.to_datetime(commits_df["date"])
         commits_df = commits_df.sort_values("date")
@@ -127,21 +101,7 @@ class EnhancedStatisticalAnalyzer:
         }
 
     def effect_size_analysis(self, group1: List[float], group2: List[float]) -> Dict[str, Any]:
-        """
-        Calculate effect size between two groups.
-
-        Use case: Compare green awareness between:
-        - Different programming languages
-        - Different time periods
-        - Different repository sizes
-
-        Args:
-            group1: First group values
-            group2: Second group values
-
-        Returns:
-            Dictionary containing effect size metrics
-        """
+        # Calculate effect size between two groups.
         # Cohen's d (effect size)
         mean1, mean2 = np.mean(group1), np.mean(group2)
         std1, std2 = np.std(group1, ddof=1), np.std(group2, ddof=1)
@@ -175,20 +135,7 @@ class EnhancedStatisticalAnalyzer:
         }
 
     def pattern_adoption_rate_analysis(self, commits_df: pd.DataFrame) -> Dict[str, Any]:
-        """
-        Analyze pattern adoption rates over repository lifetime.
-
-        Metrics:
-        - Time to first adoption (TTFA)
-        - Adoption acceleration
-        - Pattern stickiness (continued use after adoption)
-
-        Args:
-            commits_df: DataFrame with pattern and date columns
-
-        Returns:
-            Dictionary mapping patterns to adoption metrics
-        """
+        # Analyze pattern adoption rates over repository lifetime.
         results = {}
 
         for pattern in commits_df["pattern"].unique():
@@ -220,15 +167,7 @@ class EnhancedStatisticalAnalyzer:
         return results
 
     def _interpret_correlations(self, significant_pairs: List[Dict[str, Any]]) -> str:
-        """
-        Generate interpretation of correlation results.
-
-        Args:
-            significant_pairs: List of significant correlation pairs
-
-        Returns:
-            Interpretation string
-        """
+        # Generate interpretation of correlation results.
         if not significant_pairs:
             return "No significant correlations found between patterns."