greenmining 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

greenmining/controllers/repository_controller.py

@@ -1,20 +1,115 @@
-"""Repository Controller - Handles repository fetching operations."""
-
-from github import Github, GithubException
+# Repository Controller - Handles repository fetching operations.
+
+# ============================================================================
+# OLD REST API IMPLEMENTATION (DEADCODE - REPLACED WITH GRAPHQL)
+# ============================================================================
+# from github import Github, GithubException
+# from tqdm import tqdm
+#
+# from greenmining.config import Config
+# from greenmining.models.repository import Repository
+# from greenmining.utils import colored_print, load_json_file, save_json_file
+#
+#
+# class RepositoryController:
+#     # Controller for GitHub repository operations.
+#
+#     def __init__(self, config: Config):
+#         # Initialize controller with configuration.
+#         self.config = config
+#         self.github = Github(config.GITHUB_TOKEN)
+# ============================================================================
+
+# NEW GRAPHQL IMPLEMENTATION (5-10x faster)
 from tqdm import tqdm
 
 from greenmining.config import Config
 from greenmining.models.repository import Repository
+from greenmining.services.github_graphql_fetcher import GitHubGraphQLFetcher
 from greenmining.utils import colored_print, load_json_file, save_json_file
 
 
 class RepositoryController:
-    """Controller for GitHub repository operations."""
+    # Controller for GitHub repository operations using GraphQL API.
 
     def __init__(self, config: Config):
-        """Initialize controller with configuration."""
+        # Initialize controller with configuration.
         self.config = config
-        self.github = Github(config.GITHUB_TOKEN)
+        self.graphql_fetcher = GitHubGraphQLFetcher(config.GITHUB_TOKEN)
+
+    # ============================================================================
+    # OLD REST API METHOD (DEADCODE - 10x slower, high rate limit cost)
+    # ============================================================================
+    # def fetch_repositories(
+    #     self,
+    #     max_repos: int = None,
+    #     min_stars: int = None,
+    #     languages: list[str] = None,
+    #     keywords: str = None,
+    #     created_after: str = None,
+    #     created_before: str = None,
+    #     pushed_after: str = None,
+    #     pushed_before: str = None,
+    # ) -> list[Repository]:
+    #     # Fetch repositories from GitHub using REST API (slow).
+    #     max_repos = max_repos or self.config.MAX_REPOS
+    #     min_stars = min_stars or self.config.MIN_STARS
+    #     languages = languages or self.config.SUPPORTED_LANGUAGES
+    #     keywords = keywords or "microservices"
+    #
+    #     colored_print(f" Fetching up to {max_repos} repositories...", "cyan")
+    #     colored_print(f"   Keywords: {keywords}", "cyan")
+    #     colored_print(f"   Filters: min_stars={min_stars}", "cyan")
+    #
+    #     if created_after or created_before:
+    #         colored_print(
+    #             f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
+    #         )
+    #     if pushed_after or pushed_before:
+    #         colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
+    #
+    #     # Build search query with temporal filters
+    #     query = self._build_temporal_query(
+    #         keywords, min_stars, created_after, created_before, pushed_after, pushed_before
+    #     )
+    #
+    #     try:
+    #         # Execute search (REST API - many requests)
+    #         search_results = self.github.search_repositories(
+    #             query=query, sort="stars", order="desc"
+    #         )
+    #
+    #         total_found = search_results.totalCount
+    #         colored_print(f"   Found {total_found} repositories", "green")
+    #
+    #         # Fetch repositories (1 request per repo = slow)
+    #         repositories = []
+    #         with tqdm(total=min(max_repos, total_found), desc="Fetching", unit="repo") as pbar:
+    #             for idx, repo in enumerate(search_results):
+    #                 if idx >= max_repos:
+    #                     break
+    #
+    #                 try:
+    #                     repo_model = Repository.from_github_repo(repo, idx + 1)
+    #                     repositories.append(repo_model)
+    #                     pbar.update(1)
+    #                 except GithubException as e:
+    #                     colored_print(f"   Error: {repo.full_name}: {e}", "yellow")
+    #                     continue
+    #
+    #         # Save to file
+    #         repo_dicts = [r.to_dict() for r in repositories]
+    #         save_json_file(repo_dicts, self.config.REPOS_FILE)
+    #
+    #         colored_print(f" Fetched {len(repositories)} repositories", "green")
+    #         colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
+    #
+    #         return repositories
+    #
+    #     except Exception as e:
+    #         colored_print(f" Error fetching repositories: {e}", "red")
+    #         raise
+    # ============================================================================
 
     def fetch_repositories(
         self,
@@ -27,27 +122,13 @@ class RepositoryController:
         pushed_after: str = None,
         pushed_before: str = None,
     ) -> list[Repository]:
-        """Fetch repositories from GitHub.
-
-        Args:
-            max_repos: Maximum number of repositories to fetch
-            min_stars: Minimum stars filter
-            languages: List of programming languages to filter
-            keywords: Custom search keywords (default: "microservices")
-            created_after: Repository created after date (YYYY-MM-DD)
-            created_before: Repository created before date (YYYY-MM-DD)
-            pushed_after: Repository pushed after date (YYYY-MM-DD)
-            pushed_before: Repository pushed before date (YYYY-MM-DD)
-
-        Returns:
-            List of Repository model instances
-        """
+        # Fetch repositories from GitHub using GraphQL API (5-10x faster).
         max_repos = max_repos or self.config.MAX_REPOS
         min_stars = min_stars or self.config.MIN_STARS
         languages = languages or self.config.SUPPORTED_LANGUAGES
         keywords = keywords or "microservices"
 
-        colored_print(f"🔍 Fetching up to {max_repos} repositories...", "cyan")
+        colored_print(f"🚀 Fetching up to {max_repos} repositories (GraphQL API)...", "cyan")
         colored_print(f"   Keywords: {keywords}", "cyan")
         colored_print(f"   Filters: min_stars={min_stars}", "cyan")
 
@@ -58,85 +139,70 @@ class RepositoryController:
         if pushed_after or pushed_before:
             colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
 
-        # Build search query with temporal filters
-        query = self._build_temporal_query(
-            keywords, min_stars, created_after, created_before, pushed_after, pushed_before
-        )
-
         try:
-            # Execute search
-            search_results = self.github.search_repositories(
-                query=query, sort="stars", order="desc"
+            # Use GraphQL API (much faster!)
+            repositories = self.graphql_fetcher.search_repositories(
+                keywords=keywords,
+                max_repos=max_repos,
+                min_stars=min_stars,
+                languages=languages,
+                created_after=created_after,
+                created_before=created_before,
+                pushed_after=pushed_after,
+                pushed_before=pushed_before,
             )
 
-            total_found = search_results.totalCount
-            colored_print(f"   Found {total_found} repositories", "green")
-
-            # Fetch repositories
-            repositories = []
-            with tqdm(total=min(max_repos, total_found), desc="Fetching", unit="repo") as pbar:
-                for idx, repo in enumerate(search_results):
-                    if idx >= max_repos:
-                        break
-
-                    try:
-                        repo_model = Repository.from_github_repo(repo, idx + 1)
-                        repositories.append(repo_model)
-                        pbar.update(1)
-                    except GithubException as e:
-                        colored_print(f"   Error: {repo.full_name}: {e}", "yellow")
-                        continue
-
             # Save to file
             repo_dicts = [r.to_dict() for r in repositories]
             save_json_file(repo_dicts, self.config.REPOS_FILE)
 
-            colored_print(f"✅ Fetched {len(repositories)} repositories", "green")
+            colored_print(f"✓ Fetched {len(repositories)} repositories", "green")
             colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
+            colored_print(f"   API: GraphQL (5-10x faster than REST)", "green")
 
             return repositories
 
         except Exception as e:
-            colored_print(f"❌ Error fetching repositories: {e}", "red")
+            colored_print(f"✗ Error fetching repositories: {e}", "red")
             raise
 
-    def _build_temporal_query(
-        self,
-        keywords: str,
-        min_stars: int,
-        created_after: str = None,
-        created_before: str = None,
-        pushed_after: str = None,
-        pushed_before: str = None,
-    ) -> str:
-        """Build GitHub search query with temporal constraints."""
-        query_parts = [keywords, f"stars:>={min_stars}"]
-
-        # Temporal filters
-        if created_after and created_before:
-            query_parts.append(f"created:{created_after}..{created_before}")
-        elif created_after:
-            query_parts.append(f"created:>={created_after}")
-        elif created_before:
-            query_parts.append(f"created:<={created_before}")
-
-        if pushed_after and pushed_before:
-            query_parts.append(f"pushed:{pushed_after}..{pushed_before}")
-        elif pushed_after:
-            query_parts.append(f"pushed:>={pushed_after}")
-        elif pushed_before:
-            query_parts.append(f"pushed:<={pushed_before}")
-
-        query = " ".join(query_parts)
-        colored_print(f"   Query: {query}", "cyan")
-        return query
+    # ============================================================================
+    # OLD REST API HELPER (DEADCODE - handled by GraphQL fetcher now)
+    # ============================================================================
+    # def _build_temporal_query(
+    #     self,
+    #     keywords: str,
+    #     min_stars: int,
+    #     created_after: str = None,
+    #     created_before: str = None,
+    #     pushed_after: str = None,
+    #     pushed_before: str = None,
+    # ) -> str:
+    #     # Build GitHub search query with temporal constraints.
+    #     query_parts = [keywords, f"stars:>={min_stars}"]
+    #
+    #     # Temporal filters
+    #     if created_after and created_before:
+    #         query_parts.append(f"created:{created_after}..{created_before}")
+    #     elif created_after:
+    #         query_parts.append(f"created:>={created_after}")
+    #     elif created_before:
+    #         query_parts.append(f"created:<={created_before}")
+    #
+    #     if pushed_after and pushed_before:
+    #         query_parts.append(f"pushed:{pushed_after}..{pushed_before}")
+    #     elif pushed_after:
+    #         query_parts.append(f"pushed:>={pushed_after}")
+    #     elif pushed_before:
+    #         query_parts.append(f"pushed:<={pushed_before}")
+    #
+    #     query = " ".join(query_parts)
+    #     colored_print(f"   Query: {query}", "cyan")
+    #     return query
+    # ============================================================================
 
     def load_repositories(self) -> list[Repository]:
-        """Load repositories from file.
-
-        Returns:
-            List of Repository model instances
-        """
+        # Load repositories from file.
         if not self.config.REPOS_FILE.exists():
             raise FileNotFoundError(f"No repositories file found at {self.config.REPOS_FILE}")
 
@@ -144,14 +210,7 @@ class RepositoryController:
         return [Repository.from_dict(r) for r in repo_dicts]
 
     def get_repository_stats(self, repositories: list[Repository]) -> dict:
-        """Get statistics about fetched repositories.
-
-        Args:
-            repositories: List of Repository instances
-
-        Returns:
-            Dictionary with statistics
-        """
+        # Get statistics about fetched repositories.
         if not repositories:
             return {}
 
@@ -164,7 +223,7 @@ class RepositoryController:
         }
 
     def _count_by_language(self, repositories: list[Repository]) -> dict:
-        """Count repositories by language."""
+        # Count repositories by language.
         counts = {}
         for repo in repositories:
             lang = repo.language or "Unknown"

greenmining/energy/__init__.py

@@ -0,0 +1,13 @@
+# Energy measurement module for GreenMining.
+
+from .base import EnergyMeter, EnergyMetrics, EnergyBackend
+from .rapl import RAPLEnergyMeter
+from .codecarbon_meter import CodeCarbonMeter
+
+__all__ = [
+    "EnergyMeter",
+    "EnergyMetrics",
+    "EnergyBackend",
+    "RAPLEnergyMeter",
+    "CodeCarbonMeter",
+]

greenmining/energy/base.py

@@ -0,0 +1,165 @@
+# Base classes and interfaces for energy measurement.
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional, Callable
+import time
+
+
+class EnergyBackend(Enum):
+    # Supported energy measurement backends.
+    
+    RAPL = "rapl"           # Intel RAPL (Linux)
+    CODECARBON = "codecarbon"  # CodeCarbon (cross-platform)
+    CPU_METER = "cpu_meter"    # CPU Energy Meter
+
+
+@dataclass
+class EnergyMetrics:
+    # Energy measurement results from a profiling session.
+    
+    # Core energy metrics
+    joules: float = 0.0                    # Total energy consumed
+    watts_avg: float = 0.0                 # Average power draw
+    watts_peak: float = 0.0                # Peak power draw
+    duration_seconds: float = 0.0          # Measurement duration
+    
+    # Component-specific energy (if available)
+    cpu_energy_joules: float = 0.0         # CPU-specific energy
+    dram_energy_joules: float = 0.0        # Memory energy
+    gpu_energy_joules: Optional[float] = None  # GPU energy if available
+    
+    # Carbon footprint (if carbon tracking enabled)
+    carbon_grams: Optional[float] = None    # CO2 equivalent in grams
+    carbon_intensity: Optional[float] = None # gCO2/kWh of grid
+    
+    # Metadata
+    backend: str = ""
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        # Convert to dictionary.
+        return {
+            "joules": self.joules,
+            "watts_avg": self.watts_avg,
+            "watts_peak": self.watts_peak,
+            "duration_seconds": self.duration_seconds,
+            "cpu_energy_joules": self.cpu_energy_joules,
+            "dram_energy_joules": self.dram_energy_joules,
+            "gpu_energy_joules": self.gpu_energy_joules,
+            "carbon_grams": self.carbon_grams,
+            "carbon_intensity": self.carbon_intensity,
+            "backend": self.backend,
+            "start_time": self.start_time.isoformat() if self.start_time else None,
+            "end_time": self.end_time.isoformat() if self.end_time else None,
+        }
+
+
+@dataclass
+class CommitEnergyProfile:
+    # Energy profile for a specific commit.
+    
+    commit_hash: str
+    energy_before: Optional[EnergyMetrics] = None  # Parent commit energy
+    energy_after: Optional[EnergyMetrics] = None   # This commit energy
+    energy_delta: float = 0.0                      # Change in joules
+    energy_regression: bool = False                # True if energy increased
+    regression_percentage: float = 0.0             # % change
+    
+    def to_dict(self) -> Dict[str, Any]:
+        # Convert to dictionary.
+        return {
+            "commit_hash": self.commit_hash,
+            "energy_before": self.energy_before.to_dict() if self.energy_before else None,
+            "energy_after": self.energy_after.to_dict() if self.energy_after else None,
+            "energy_delta": self.energy_delta,
+            "energy_regression": self.energy_regression,
+            "regression_percentage": self.regression_percentage,
+        }
+
+
+class EnergyMeter(ABC):
+    # Abstract base class for energy measurement backends.
+    
+    def __init__(self, backend: EnergyBackend):
+        # Initialize the energy meter.
+        self.backend = backend
+        self._is_measuring = False
+        self._start_time: Optional[float] = None
+        self._measurements: List[float] = []
+    
+    @abstractmethod
+    def is_available(self) -> bool:
+        # Check if this energy measurement backend is available on the system.
+        pass
+    
+    @abstractmethod
+    def start(self) -> None:
+        # Start energy measurement.
+        pass
+    
+    @abstractmethod
+    def stop(self) -> EnergyMetrics:
+        # Stop energy measurement and return results.
+        pass
+    
+    def measure(self, func: Callable, *args, **kwargs) -> tuple[Any, EnergyMetrics]:
+        # Measure energy consumption of a function call.
+        self.start()
+        try:
+            result = func(*args, **kwargs)
+        finally:
+            metrics = self.stop()
+        return result, metrics
+    
+    def measure_command(self, command: str, timeout: Optional[int] = None) -> EnergyMetrics:
+        # Measure energy consumption of a shell command.
+        import subprocess
+        
+        self.start()
+        try:
+            subprocess.run(
+                command,
+                shell=True,
+                timeout=timeout,
+                capture_output=True,
+                text=True,
+            )
+        finally:
+            metrics = self.stop()
+        return metrics
+    
+    def __enter__(self):
+        # Context manager entry.
+        self.start()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Context manager exit.
+        self.stop()
+        return False
+
+
+def get_energy_meter(backend: str = "rapl") -> EnergyMeter:
+    # Factory function to get an energy meter instance.
+    from .rapl import RAPLEnergyMeter
+    from .codecarbon_meter import CodeCarbonMeter
+    
+    backend_lower = backend.lower()
+    
+    if backend_lower == "rapl":
+        meter = RAPLEnergyMeter()
+    elif backend_lower == "codecarbon":
+        meter = CodeCarbonMeter()
+    else:
+        raise ValueError(f"Unsupported energy backend: {backend}")
+    
+    if not meter.is_available():
+        raise ValueError(f"Energy backend '{backend}' is not available on this system")
+    
+    return meter

greenmining/energy/codecarbon_meter.py

@@ -0,0 +1,146 @@
+# CodeCarbon integration for carbon-aware energy measurement.
+
+from __future__ import annotations
+
+import time
+from datetime import datetime
+from typing import Optional
+
+from .base import EnergyMeter, EnergyMetrics, EnergyBackend
+
+
+class CodeCarbonMeter(EnergyMeter):
+    # Energy measurement using CodeCarbon library.
+    
+    def __init__(
+        self,
+        project_name: str = "greenmining",
+        output_dir: Optional[str] = None,
+        save_to_file: bool = False,
+    ):
+        # Initialize CodeCarbon energy meter.
+        super().__init__(EnergyBackend.CODECARBON)
+        self.project_name = project_name
+        self.output_dir = output_dir
+        self.save_to_file = save_to_file
+        self._tracker = None
+        self._start_time: Optional[float] = None
+        self._codecarbon_available = self._check_codecarbon()
+    
+    def _check_codecarbon(self) -> bool:
+        # Check if CodeCarbon is installed.
+        try:
+            from codecarbon import EmissionsTracker
+            return True
+        except ImportError:
+            return False
+    
+    def is_available(self) -> bool:
+        # Check if CodeCarbon is available.
+        return self._codecarbon_available
+    
+    def start(self) -> None:
+        # Start energy measurement.
+        if not self._codecarbon_available:
+            raise RuntimeError("CodeCarbon is not installed. Run: pip install codecarbon")
+        
+        if self._is_measuring:
+            raise RuntimeError("Already measuring energy")
+        
+        from codecarbon import EmissionsTracker
+        
+        self._is_measuring = True
+        self._start_time = time.time()
+        
+        # Create emissions tracker
+        tracker_kwargs = {
+            "project_name": self.project_name,
+            "measure_power_secs": 1,
+            "save_to_file": self.save_to_file,
+            "log_level": "error",  # Suppress verbose output
+        }
+        
+        if self.output_dir:
+            tracker_kwargs["output_dir"] = self.output_dir
+        
+        self._tracker = EmissionsTracker(**tracker_kwargs)
+        self._tracker.start()
+    
+    def stop(self) -> EnergyMetrics:
+        # Stop energy measurement and return results.
+        if not self._is_measuring:
+            raise RuntimeError("Not currently measuring energy")
+        
+        end_time = time.time()
+        self._is_measuring = False
+        
+        # Stop tracker and get emissions
+        emissions_kg = self._tracker.stop()
+        
+        # Get detailed data from tracker
+        duration = end_time - self._start_time
+        
+        # CodeCarbon stores data in tracker._total_energy (kWh)
+        # In v3.x it may return an Energy object, extract the value
+        energy_raw = getattr(self._tracker, "_total_energy", 0) or 0
+        if hasattr(energy_raw, "kWh"):
+            energy_kwh = float(energy_raw.kWh)
+        else:
+            energy_kwh = float(energy_raw) if energy_raw else 0.0
+        
+        # Convert kWh to joules (1 kWh = 3,600,000 J)
+        energy_joules = energy_kwh * 3_600_000
+        
+        # Calculate average power
+        watts_avg = (energy_joules / duration) if duration > 0 else 0
+        
+        # Get carbon intensity if available
+        carbon_intensity = None
+        try:
+            carbon_intensity = getattr(self._tracker, "_carbon_intensity", None)
+            if hasattr(carbon_intensity, "value"):
+                carbon_intensity = float(carbon_intensity.value)
+        except Exception:
+            pass
+        
+        # Convert emissions from kg to grams (handle Energy objects)
+        if hasattr(emissions_kg, "value"):
+            emissions_kg = float(emissions_kg.value)
+        carbon_grams = float(emissions_kg or 0) * 1000
+        
+        return EnergyMetrics(
+            joules=energy_joules,
+            watts_avg=watts_avg,
+            watts_peak=watts_avg,  # CodeCarbon doesn't provide peak
+            duration_seconds=duration,
+            cpu_energy_joules=energy_joules,  # CodeCarbon aggregates all sources
+            dram_energy_joules=0,
+            gpu_energy_joules=None,
+            carbon_grams=carbon_grams,
+            carbon_intensity=carbon_intensity,
+            backend="codecarbon",
+            start_time=datetime.fromtimestamp(self._start_time),
+            end_time=datetime.fromtimestamp(end_time),
+        )
+    
+    def get_carbon_intensity(self) -> Optional[float]:
+        # Get current carbon intensity for the configured region.
+        if not self._codecarbon_available:
+            return None
+        
+        try:
+            from codecarbon import EmissionsTracker
+            
+            # Create temporary tracker to get carbon intensity
+            tracker = EmissionsTracker(
+                project_name="carbon_check",
+                country_iso_code=self.country_iso_code,
+                save_to_file=False,
+                log_level="error",
+            )
+            tracker.start()
+            tracker.stop()
+            
+            return getattr(tracker, "_carbon_intensity", None)
+        except Exception:
+            return None