greenmining 1.1.9__py3-none-any.whl → 1.2.0__py3-none-any.whl

greenmining/__init__.py

@@ -1,6 +1,5 @@
-# Green Microservices Mining - GSF Pattern Analysis Tool.
+# GreenMining - MSR library for Green IT research.
 
-from greenmining.config import Config
 from greenmining.controllers.repository_controller import RepositoryController
 from greenmining.gsf_patterns import (
     GREEN_KEYWORDS,
@@ -9,24 +8,23 @@ from greenmining.gsf_patterns import (
     is_green_aware,
 )
 
-__version__ = "1.1.9"
+__version__ = "1.2.0"
 
 
 def fetch_repositories(
     github_token: str,
-    max_repos: int = None,
-    min_stars: int = None,
+    max_repos: int = 100,
+    min_stars: int = 100,
     languages: list = None,
     keywords: str = None,
     created_after: str = None,
     created_before: str = None,
     pushed_after: str = None,
     pushed_before: str = None,
+    output_dir: str = "./data",
 ):
-    # Fetch repositories from GitHub with custom search keywords.
-    config = Config()
-    config.GITHUB_TOKEN = github_token
-    controller = RepositoryController(config)
+    # Fetch repositories from GitHub via GraphQL search.
+    controller = RepositoryController(github_token, output_dir=output_dir)
 
     return controller.fetch_repositories(
         max_repos=max_repos,
@@ -40,6 +38,27 @@ def fetch_repositories(
     )
 
 
+def clone_repositories(
+    repositories: list,
+    github_token: str = None,
+    output_dir: str = "./data",
+    cleanup_existing: bool = False,
+):
+    # Clone repositories into ./greenmining_repos with sanitized directory names.
+    # Args:
+    #   repositories: List of Repository objects (from fetch_repositories)
+    #   github_token: GitHub token (required for controller init)
+    #   output_dir: Output directory for metadata files
+    #   cleanup_existing: Remove existing greenmining_repos/ before cloning
+    token = github_token or "unused"
+    controller = RepositoryController(token, output_dir=output_dir)
+
+    return controller.clone_repositories(
+        repositories=repositories,
+        cleanup_existing=cleanup_existing,
+    )
+
+
 def analyze_repositories(
     urls: list,
     max_commits: int = 500,
@@ -99,12 +118,12 @@ def analyze_repositories(
 
 
 __all__ = [
-    "Config",
     "GSF_PATTERNS",
     "GREEN_KEYWORDS",
     "is_green_aware",
     "get_pattern_by_keywords",
     "fetch_repositories",
+    "clone_repositories",
     "analyze_repositories",
     "__version__",
 ]

greenmining/analyzers/__init__.py

@@ -3,20 +3,12 @@
 from .code_diff_analyzer import CodeDiffAnalyzer
 from .statistical_analyzer import StatisticalAnalyzer
 from .temporal_analyzer import TemporalAnalyzer
-from .qualitative_analyzer import QualitativeAnalyzer
-from .power_regression import PowerRegressionDetector, PowerRegression
 from .metrics_power_correlator import MetricsPowerCorrelator, CorrelationResult
-from .version_power_analyzer import VersionPowerAnalyzer, VersionPowerReport
 
 __all__ = [
     "CodeDiffAnalyzer",
     "StatisticalAnalyzer",
     "TemporalAnalyzer",
-    "QualitativeAnalyzer",
-    "PowerRegressionDetector",
-    "PowerRegression",
     "MetricsPowerCorrelator",
     "CorrelationResult",
-    "VersionPowerAnalyzer",
-    "VersionPowerReport",
 ]

greenmining/controllers/repository_controller.py

@@ -1,11 +1,9 @@
 # Repository Controller - Handles repository fetching + cloning operations.
-import os
+
 import re
 import shutil
 from pathlib import Path
-from typing import List, Dict
 
-from greenmining.config import Config
 from greenmining.models.repository import Repository
 from greenmining.services.github_graphql_fetcher import GitHubGraphQLFetcher
 from greenmining.utils import colored_print, load_json_file, save_json_file
@@ -14,86 +12,26 @@ from greenmining.utils import colored_print, load_json_file, save_json_file
 class RepositoryController:
     # Controller for GitHub repository operations using GraphQL API.
 
-    def __init__(self, config: Config):
-        # Initialize controller with configuration.
-        self.config = config
-        self.graphql_fetcher = GitHubGraphQLFetcher(config.GITHUB_TOKEN)
+    def __init__(self, github_token: str, output_dir: str = "./data"):
+        # Initialize controller with GitHub token and output directory.
+        self.graphql_fetcher = GitHubGraphQLFetcher(github_token)
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.repos_file = self.output_dir / "repositories.json"
         self.repos_dir = Path.cwd() / "greenmining_repos"
 
-    def _sanitize_repo_name(self, repo: Repository, index: int = 0) -> str:
-        """Safe unique dir name: owner_repo[_index]. Handles case collisions."""
-        base = re.sub(r'[^a-z0-9-]', '_', repo.full_name.replace('/', '_').lower())
-        name = f"{base}_{index}" if index else base
-        path = self.repos_dir / name
-        counter = 1
-        while path.exists():
-            name = f"{base}_{counter}"
-            path = self.repos_dir / name
-            counter += 1
-        return name
-
-    def clone_repositories(
-        self, 
-        repositories: List[Repository], 
-        github_token: str = None,
-        cleanup: bool = True,
-        depth: int = 1  # Shallow clone
-    ) -> List[Dict]:
-        """Clone repos to ./greenmining_repos/ with unique sanitized names."""
-        self.repos_dir.mkdir(exist_ok=True)
-        if cleanup:
-            shutil.rmtree(self.repos_dir, ignore_errors=True)
-            self.repos_dir.mkdir(exist_ok=True)
-            colored_print(f"Cleaned {self.repos_dir}", "yellow")
-
-        results = []
-        for i, repo in enumerate(repositories, 1):
-            safe_name = self._sanitize_repo_name(repo, i)
-            clone_path = self.repos_dir / safe_name
-            
-            colored_print(f"[{i}/{len(repositories)}] Cloning {repo.full_name} → {safe_name}", "cyan")
-            
-            url = f"https://{github_token}@github.com/{repo.full_name}.git" if github_token else repo.url
-            cmd = ["git", "clone", f"--depth={depth}", "-v", url, str(clone_path)]
-            
-            import subprocess
-            try:
-                subprocess.check_call(cmd, cwd=self.repos_dir.parent)
-                colored_print(f"{safe_name}", "green")
-                results.append({
-                    "full_name": repo.full_name,
-                    "local_path": str(clone_path),
-                    "success": True
-                })
-            except subprocess.CalledProcessError as e:
-                colored_print(f"{safe_name}: {e}", "red")
-                results.append({
-                    "full_name": repo.full_name,
-                    "local_path": str(clone_path),
-                    "success": False,
-                    "error": str(e)
-                })
-
-        # Save map for analyze_repositories
-        save_json_file(results, self.repos_dir / "clone_results.json")
-        success_rate = sum(1 for r in results if r["success"]) / len(results) * 100
-        colored_print(f"Cloned: {success_rate:.1f}% ({self.repos_dir}/clone_results.json)", "green")
-        return results
-
-
-
-
-
-    def fetch_repositories(self, max_repos: int = None, min_stars: int = None,
-                          languages: list[str] = None, keywords: str = None,
-                          created_after: str = None, created_before: str = None,
-                          pushed_after: str = None, pushed_before: str = None) -> list[Repository]:
+    def fetch_repositories(
+        self,
+        max_repos: int = 100,
+        min_stars: int = 100,
+        languages: list[str] = None,
+        keywords: str = None,
+        created_after: str = None,
+        created_before: str = None,
+        pushed_after: str = None,
+        pushed_before: str = None,
+    ) -> list[Repository]:
         # Fetch repositories from GitHub using GraphQL API.
-        max_repos = max_repos or self.config.MAX_REPOS
-        min_stars = min_stars or self.config.MIN_STARS
-        languages = languages or self.config.SUPPORTED_LANGUAGES
-        keywords = keywords 
-
         colored_print(f"Fetching up to {max_repos} repositories...", "cyan")
         colored_print(f"   Keywords: {keywords}", "cyan")
         colored_print(f"   Filters: min_stars={min_stars}", "cyan")
@@ -103,10 +41,11 @@ class RepositoryController:
                 f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
             )
         if pushed_after or pushed_before:
-            colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
+            colored_print(
+                f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan"
+            )
 
         try:
-            # Execute GraphQL search
             repositories = self.graphql_fetcher.search_repositories(
                 keywords=keywords,
                 max_repos=max_repos,
@@ -118,12 +57,11 @@ class RepositoryController:
                 pushed_before=pushed_before,
             )
 
-            # Save to file
             repo_dicts = [r.to_dict() for r in repositories]
-            save_json_file(repo_dicts, self.config.REPOS_FILE)
+            save_json_file(repo_dicts, self.repos_file)
 
             colored_print(f"Fetched {len(repositories)} repositories", "green")
-            colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
+            colored_print(f"   Saved to: {self.repos_file}", "cyan")
 
             return repositories
 
@@ -131,12 +69,69 @@ class RepositoryController:
             colored_print(f"Error fetching repositories: {e}", "red")
             raise
 
+    def clone_repositories(
+        self,
+        repositories: list[Repository],
+        cleanup_existing: bool = False,
+    ) -> list[Path]:
+        # Clone repositories into ./greenmining_repos with sanitized directory names.
+        self.repos_dir.mkdir(parents=True, exist_ok=True)
+
+        if cleanup_existing and self.repos_dir.exists():
+            shutil.rmtree(self.repos_dir)
+            self.repos_dir.mkdir(parents=True, exist_ok=True)
+
+        cloned_paths = []
+        colored_print(f"\nCloning {len(repositories)} repositories into {self.repos_dir}", "cyan")
+
+        for repo in repositories:
+            safe_name = self._sanitize_repo_name(repo)
+            local_path = self.repos_dir / safe_name
+
+            if local_path.exists():
+                colored_print(f"   Already exists: {safe_name}", "yellow")
+                cloned_paths.append(local_path)
+                continue
+
+            try:
+                url = repo.url if hasattr(repo, "url") else f"https://github.com/{repo.full_name}"
+                colored_print(f"   Cloning {repo.full_name} -> {safe_name}", "cyan")
+
+                import subprocess
+
+                subprocess.run(
+                    ["git", "clone", "--depth", "1", url, str(local_path)],
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                    timeout=120,
+                )
+                cloned_paths.append(local_path)
+                colored_print(f"   Cloned: {safe_name}", "green")
+
+            except Exception as e:
+                colored_print(f"   Failed to clone {repo.full_name}: {e}", "yellow")
+
+        colored_print(f"Cloned {len(cloned_paths)}/{len(repositories)} repositories", "green")
+        return cloned_paths
+
+    def _sanitize_repo_name(self, repo: Repository) -> str:
+        # Safe unique directory name: owner_repo. Handles case collisions.
+        base = re.sub(r"[^a-z0-9-]", "_", repo.full_name.replace("/", "_").lower())
+        path = self.repos_dir / base
+        if not path.exists():
+            return base
+        counter = 1
+        while (self.repos_dir / f"{base}_{counter}").exists():
+            counter += 1
+        return f"{base}_{counter}"
+
     def load_repositories(self) -> list[Repository]:
         # Load repositories from file.
-        if not self.config.REPOS_FILE.exists():
-            raise FileNotFoundError(f"No repositories file found at {self.config.REPOS_FILE}")
+        if not self.repos_file.exists():
+            raise FileNotFoundError(f"No repositories file found at {self.repos_file}")
 
-        repo_dicts = load_json_file(self.config.REPOS_FILE)
+        repo_dicts = load_json_file(self.repos_file)
         return [Repository.from_dict(r) for r in repo_dicts]
 
     def get_repository_stats(self, repositories: list[Repository]) -> dict:

greenmining/services/local_repo_analyzer.py

@@ -221,7 +221,7 @@ class LocalRepoAnalyzer:
         #   method_level_analysis: Extract per-method metrics via Lizard
         #   include_source_code: Include source code before/after in results
         #   process_metrics: "standard" or "full" PyDriller process metrics
-        self.clone_path = clone_path or Path(tempfile.gettempdir()) / "greenmining_repos"
+        self.clone_path = clone_path or Path.cwd() / "greenmining_repos"
         self.clone_path.mkdir(parents=True, exist_ok=True)
         self.max_commits = max_commits
         self.days_back = days_back

greenmining-1.2.0.dist-info/METADATA

@@ -0,0 +1,311 @@
+Metadata-Version: 2.4
+Name: greenmining
+Version: 1.2.0
+Summary: An empirical Python library for Mining Software Repositories (MSR) in Green IT research
+Author-email: Adam Bouafia <a.bouafia@student.vu.nl>
+License: MIT
+Project-URL: Homepage, https://github.com/adam-bouafia/greenmining
+Project-URL: Documentation, https://github.com/adam-bouafia/greenmining#readme
+Project-URL: Linkedin, https://www.linkedin.com/in/adam-bouafia/
+Project-URL: Repository, https://github.com/adam-bouafia/greenmining
+Project-URL: Issues, https://github.com/adam-bouafia/greenmining/issues
+Project-URL: Changelog, https://github.com/adam-bouafia/greenmining/blob/main/CHANGELOG.md
+Keywords: green-software,gsf,msr,mining-software-repositories,green-it,sustainability,carbon-footprint,energy-efficiency,repository-analysis,github-analysis,pydriller,empirical-software-engineering
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyGithub
+Requires-Dist: PyDriller
+Requires-Dist: pandas
+Requires-Dist: colorama
+Requires-Dist: tabulate
+Requires-Dist: tqdm
+Requires-Dist: matplotlib
+Requires-Dist: plotly
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Provides-Extra: energy
+Requires-Dist: psutil; extra == "energy"
+Requires-Dist: codecarbon; extra == "energy"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Dynamic: license-file
+
+# greenmining
+
+An empirical Python library for Mining Software Repositories (MSR) in Green IT research.
+
+[![PyPI](https://img.shields.io/pypi/v/greenmining)](https://pypi.org/project/greenmining/)
+[![Python](https://img.shields.io/pypi/pyversions/greenmining)](https://pypi.org/project/greenmining/)
+[![License](https://img.shields.io/github/license/adam-bouafia/greenmining)](LICENSE)
+[![Documentation](https://img.shields.io/badge/docs-readthedocs-blue)](https://greenmining.readthedocs.io/)
+
+## Overview
+
+`greenmining` is a research-grade Python library designed for **empirical Mining Software Repositories (MSR)** studies in **Green IT**. It enables researchers and practitioners to:
+
+- **Mine repositories at scale** - Search, fetch, and clone GitHub repositories via GraphQL API with configurable filters
+- **Classify green commits** - Detect 124 sustainability patterns from the Green Software Foundation (GSF) catalog using 332 keywords
+- **Analyze any repository by URL** - Direct Git-based analysis with support for private repositories
+- **Measure energy consumption** - RAPL, CodeCarbon, and CPU Energy Meter backends for power profiling
+- **Carbon footprint reporting** - CO2 emissions calculation with 20+ country profiles and cloud region support
+- **Method-level analysis** - Per-method complexity and metrics via Lizard integration
+- **Generate research datasets** - Statistical analysis, temporal trends, and publication-ready reports
+
+## Installation
+
+### Via pip
+
+```bash
+pip install greenmining
+```
+
+### With energy measurement
+
+```bash
+pip install greenmining[energy]
+```
+
+### From source
+
+```bash
+git clone https://github.com/adam-bouafia/greenmining.git
+cd greenmining
+pip install -e .
+```
+
+## Quick Start
+
+### Pattern Detection
+
+```python
+from greenmining import GSF_PATTERNS, is_green_aware, get_pattern_by_keywords
+
+print(f"Total patterns: {len(GSF_PATTERNS)}")  # 124 patterns across 15 categories
+
+commit_msg = "Optimize Redis caching to reduce energy consumption"
+if is_green_aware(commit_msg):
+    patterns = get_pattern_by_keywords(commit_msg)
+    print(f"Matched patterns: {patterns}")
+```
+
+### Fetch Repositories
+
+```python
+from greenmining import fetch_repositories
+
+repos = fetch_repositories(
+    github_token="your_token",
+    max_repos=50,
+    min_stars=500,
+    keywords="kubernetes cloud-native",
+    languages=["Python", "Go"],
+    created_after="2020-01-01",
+    pushed_after="2023-01-01",
+)
+
+for repo in repos[:5]:
+    print(f"- {repo.full_name} ({repo.stars} stars)")
+```
+
+### Clone Repositories
+
+```python
+from greenmining import fetch_repositories, clone_repositories
+
+repos = fetch_repositories(github_token="your_token", max_repos=10, keywords="android")
+
+# Clone into ./greenmining_repos/ with sanitized directory names
+paths = clone_repositories(repos)
+print(f"Cloned {len(paths)} repositories")
+```
+
+### Analyze Repositories by URL
+
+```python
+from greenmining import analyze_repositories
+
+results = analyze_repositories(
+    urls=[
+        "https://github.com/kubernetes/kubernetes",
+        "https://github.com/istio/istio",
+    ],
+    max_commits=100,
+    parallel_workers=2,
+    energy_tracking=True,
+    energy_backend="auto",
+    method_level_analysis=True,
+    include_source_code=True,
+    github_token="your_token",
+    since_date="2020-01-01",
+    to_date="2025-12-31",
+)
+
+for result in results:
+    print(f"{result.name}: {result.green_commit_rate:.1%} green")
+```
+
+### Access Pattern Data
+
+```python
+from greenmining import GSF_PATTERNS
+
+# Get patterns by category
+cloud = {k: v for k, v in GSF_PATTERNS.items() if v['category'] == 'cloud'}
+print(f"Cloud patterns: {len(cloud)}")
+
+# All categories
+categories = set(p['category'] for p in GSF_PATTERNS.values())
+print(f"Categories: {sorted(categories)}")
+```
+
+### Energy Measurement
+
+```python
+from greenmining.energy import get_energy_meter, CPUEnergyMeter
+
+# Auto-detect best backend
+meter = get_energy_meter("auto")
+meter.start()
+# ... your workload ...
+result = meter.stop()
+print(f"Energy: {result.joules:.2f} J, Power: {result.watts_avg:.2f} W")
+```
+
+### Statistical Analysis
+
+```python
+from greenmining.analyzers import StatisticalAnalyzer, TemporalAnalyzer
+
+stat = StatisticalAnalyzer()
+temporal = TemporalAnalyzer(granularity="quarter")
+
+# Pattern correlations, effect sizes, temporal trends
+# See experiment notebook for full usage
+```
+
+### Metrics-to-Power Correlation
+
+```python
+from greenmining.analyzers import MetricsPowerCorrelator
+
+correlator = MetricsPowerCorrelator()
+correlator.fit(
+    metrics=["complexity", "nloc", "code_churn"],
+    metrics_values={
+        "complexity": [10, 20, 30, 40],
+        "nloc": [100, 200, 300, 400],
+        "code_churn": [50, 100, 150, 200],
+    },
+    power_measurements=[5.0, 8.0, 12.0, 15.0],
+)
+print(f"Feature importance: {correlator.feature_importance}")
+```
+
+## Features
+
+### Core Capabilities
+
+- **Pattern Detection**: 124 sustainability patterns across 15 categories from the GSF catalog
+- **Keyword Analysis**: 332 green software detection keywords
+- **Repository Fetching**: GraphQL API with date, star, and language filters
+- **Repository Cloning**: Sanitized directory names in `./greenmining_repos/`
+- **URL-Based Analysis**: Direct Git-based analysis from GitHub URLs (HTTPS and SSH)
+- **Batch Processing**: Parallel analysis of multiple repositories
+- **Private Repository Support**: Authentication via SSH keys or GitHub tokens
+
+### Analysis & Measurement
+
+- **Energy Measurement**: RAPL, CodeCarbon, and CPU Energy Meter backends
+- **Carbon Footprint Reporting**: CO2 emissions with 20+ country profiles (AWS, GCP, Azure)
+- **Metrics-to-Power Correlation**: Pearson and Spearman analysis between code metrics and power
+- **Method-Level Analysis**: Per-method complexity metrics via Lizard integration
+- **Source Code Access**: Before/after source code for refactoring detection
+- **Process Metrics**: DMM size, complexity, interfacing via PyDriller
+- **Statistical Analysis**: Correlations, effect sizes, and temporal trends
+- **Multi-format Output**: JSON, CSV, pandas DataFrame
+
+### Energy Backends
+
+| Backend | Platform | Metrics | Requirements |
+|---------|----------|---------|--------------|
+| **RAPL** | Linux (Intel/AMD) | CPU/RAM energy (Joules) | `/sys/class/powercap/` access |
+| **CodeCarbon** | Cross-platform | Energy + Carbon emissions (gCO2) | `pip install codecarbon` |
+| **CPU Meter** | All platforms | Estimated CPU energy (Joules) | Optional: `pip install psutil` |
+| **Auto** | All platforms | Best available backend | Automatic detection |
+
+### GSF Pattern Categories
+
+**124 patterns across 15 categories:**
+
+| Category | Patterns | Examples |
+|----------|----------|----------|
+| Cloud | 42 | Auto-scaling, serverless, right-sizing, region selection |
+| Web | 17 | CDN, caching, lazy loading, compression |
+| AI/ML | 19 | Model pruning, quantization, edge inference |
+| Database | 5 | Indexing, query optimization, connection pooling |
+| Networking | 8 | Protocol optimization, HTTP/2, gRPC |
+| Network | 6 | Request batching, GraphQL, circuit breakers |
+| Microservices | 4 | Service decomposition, graceful shutdown |
+| Infrastructure | 4 | Alpine containers, IaC, renewable regions |
+| General | 8 | Feature flags, precomputation, background jobs |
+| Others | 11 | Caching, resource, data, async, code, monitoring |
+
+## Development
+
+```bash
+git clone https://github.com/adam-bouafia/greenmining.git
+cd greenmining
+pip install -e ".[dev]"
+
+pytest tests/
+black greenmining/ tests/
+ruff check greenmining/ tests/
+```
+
+## Requirements
+
+- Python 3.9+
+- PyGithub, PyDriller, pandas, colorama, tqdm
+
+**Optional:**
+
+```bash
+pip install greenmining[energy]      # psutil, codecarbon
+pip install greenmining[dev]         # pytest, black, ruff, mypy
+```
+
+## License
+
+MIT License - See [LICENSE](LICENSE) for details.
+
+## Links
+
+- **GitHub**: https://github.com/adam-bouafia/greenmining
+- **PyPI**: https://pypi.org/project/greenmining/
+- **Documentation**: https://greenmining.readthedocs.io/
+- **Docker Hub**: https://hub.docker.com/r/adambouafia/greenmining