greenmining 1.0.7__tar.gz → 1.0.8__tar.gz

{greenmining-1.0.7/greenmining.egg-info → greenmining-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: greenmining
-Version: 1.0.7
+Version: 1.0.8
 Summary: An empirical Python library for Mining Software Repositories (MSR) in Green IT research
 Author-email: Adam Bouafia <a.bouafia@student.vu.nl>
 License: MIT

{greenmining-1.0.7 → greenmining-1.0.8}/greenmining/__init__.py

@@ -9,7 +9,7 @@ from greenmining.gsf_patterns import (
     is_green_aware,
 )
 
-__version__ = "1.0.7"
+__version__ = "1.0.8"
 
 
 def fetch_repositories(

greenmining-1.0.8/greenmining/controllers/repository_controller.py

@@ -0,0 +1,100 @@
+# Repository Controller - Handles repository fetching operations.
+
+from tqdm import tqdm
+
+from greenmining.config import Config
+from greenmining.models.repository import Repository
+from greenmining.services.github_graphql_fetcher import GitHubGraphQLFetcher
+from greenmining.utils import colored_print, load_json_file, save_json_file
+
+
+class RepositoryController:
+    # Controller for GitHub repository operations using GraphQL API.
+
+    def __init__(self, config: Config):
+        # Initialize controller with configuration.
+        self.config = config
+        self.graphql_fetcher = GitHubGraphQLFetcher(config.GITHUB_TOKEN)
+
+    def fetch_repositories(
+        self,
+        max_repos: int = None,
+        min_stars: int = None,
+        languages: list[str] = None,
+        keywords: str = None,
+        created_after: str = None,
+        created_before: str = None,
+        pushed_after: str = None,
+        pushed_before: str = None,
+    ) -> list[Repository]:
+        # Fetch repositories from GitHub using GraphQL API.
+        max_repos = max_repos or self.config.MAX_REPOS
+        min_stars = min_stars or self.config.MIN_STARS
+        languages = languages or self.config.SUPPORTED_LANGUAGES
+        keywords = keywords or "microservices"
+
+        colored_print(f"Fetching up to {max_repos} repositories...", "cyan")
+        colored_print(f"   Keywords: {keywords}", "cyan")
+        colored_print(f"   Filters: min_stars={min_stars}", "cyan")
+
+        if created_after or created_before:
+            colored_print(
+                f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
+            )
+        if pushed_after or pushed_before:
+            colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
+
+        try:
+            # Execute GraphQL search
+            repositories = self.graphql_fetcher.search_repositories(
+                keywords=keywords,
+                max_repos=max_repos,
+                min_stars=min_stars,
+                languages=languages,
+                created_after=created_after,
+                created_before=created_before,
+                pushed_after=pushed_after,
+                pushed_before=pushed_before,
+            )
+
+            # Save to file
+            repo_dicts = [r.to_dict() for r in repositories]
+            save_json_file(repo_dicts, self.config.REPOS_FILE)
+
+            colored_print(f"Fetched {len(repositories)} repositories", "green")
+            colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
+
+            return repositories
+
+        except Exception as e:
+            colored_print(f"Error fetching repositories: {e}", "red")
+            raise
+
+    def load_repositories(self) -> list[Repository]:
+        # Load repositories from file.
+        if not self.config.REPOS_FILE.exists():
+            raise FileNotFoundError(f"No repositories file found at {self.config.REPOS_FILE}")
+
+        repo_dicts = load_json_file(self.config.REPOS_FILE)
+        return [Repository.from_dict(r) for r in repo_dicts]
+
+    def get_repository_stats(self, repositories: list[Repository]) -> dict:
+        # Get statistics about fetched repositories.
+        if not repositories:
+            return {}
+
+        return {
+            "total": len(repositories),
+            "by_language": self._count_by_language(repositories),
+            "total_stars": sum(r.stars for r in repositories),
+            "avg_stars": sum(r.stars for r in repositories) / len(repositories),
+            "top_repo": max(repositories, key=lambda r: r.stars).full_name,
+        }
+
+    def _count_by_language(self, repositories: list[Repository]) -> dict:
+        # Count repositories by language.
+        counts = {}
+        for repo in repositories:
+            lang = repo.language or "Unknown"
+            counts[lang] = counts.get(lang, 0) + 1
+        return counts

{greenmining-1.0.7 → greenmining-1.0.8}/greenmining/presenters/console_presenter.py

@@ -113,7 +113,7 @@ class ConsolePresenter:
 
         table_data = []
         for phase, info in status.items():
-            status_icon = "" if info.get("completed") else "⏳"
+            status_icon = "done" if info.get("completed") else "pending"
             table_data.append(
                 [status_icon, phase, info.get("file", "N/A"), info.get("size", "N/A")]
             )

greenmining-1.0.8/greenmining/services/github_fetcher.py

@@ -0,0 +1,2 @@
+# Legacy GitHub REST API fetcher (deprecated).
+# Use github_graphql_fetcher.GitHubGraphQLFetcher instead.

{greenmining-1.0.7 → greenmining-1.0.8}/greenmining/services/github_graphql_fetcher.py

@@ -1,7 +1,4 @@
-# GitHub GraphQL API fetcher for faster and more efficient repository fetching.
-#
-# GraphQL allows fetching exactly the data you need in a single request,
-# reducing API calls and improving rate limit efficiency.
+# GitHub GraphQL API fetcher for repository search and data retrieval.
 
 import json
 import time
@@ -14,12 +11,6 @@ from greenmining.models.repository import Repository
 
 class GitHubGraphQLFetcher:
     # Fetch GitHub repositories using GraphQL API v4.
-    #
-    # Benefits over REST API:
-    # - Fetch repos + commits in 1 request instead of 100+ REST calls
-    # - Get exactly the fields you need (no over-fetching)
-    # - Better rate limit efficiency (5000 points/hour vs 5000 requests/hour)
-    # - More powerful search capabilities
 
     GRAPHQL_ENDPOINT = "https://api.github.com/graphql"
 
@@ -193,10 +184,10 @@ class GitHubGraphQLFetcher:
         # Star count
         query_parts.append(f"stars:>={min_stars}")
 
-        # Languages
-        if languages:
-            lang_query = " OR ".join([f"language:{lang}" for lang in languages])
-            query_parts.append(f"({lang_query})")
+        # Languages - skip filter if more than 5 to avoid exceeding GitHub query limits
+        if languages and len(languages) <= 5:
+            lang_query = " ".join([f"language:{lang}" for lang in languages])
+            query_parts.append(lang_query)
 
         # Date filters
         if created_after:
@@ -259,9 +250,6 @@ class GitHubGraphQLFetcher:
     ) -> List[Dict[str, Any]]:
         # Fetch commits for a specific repository using GraphQL.
         #
-        # This is much faster than REST API as it gets all commits in 1-2 requests
-        # instead of paginating through 100 individual REST calls.
-        #
         # Args:
         #     owner: Repository owner
         #     name: Repository name

{greenmining-1.0.7 → greenmining-1.0.8/greenmining.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: greenmining
-Version: 1.0.7
+Version: 1.0.8
 Summary: An empirical Python library for Mining Software Repositories (MSR) in Green IT research
 Author-email: Adam Bouafia <a.bouafia@student.vu.nl>
 License: MIT

{greenmining-1.0.7 → greenmining-1.0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "greenmining"
-version = "1.0.7"
+version = "1.0.8"
 description = "An empirical Python library for Mining Software Repositories (MSR) in Green IT research"
 readme = "README.md"
 requires-python = ">=3.9"

greenmining-1.0.7/greenmining/controllers/repository_controller.py

@@ -1,231 +0,0 @@
-# Repository Controller - Handles repository fetching operations.
-
-# ============================================================================
-# OLD REST API IMPLEMENTATION (DEADCODE - REPLACED WITH GRAPHQL)
-# ============================================================================
-# from github import Github, GithubException
-# from tqdm import tqdm
-#
-# from greenmining.config import Config
-# from greenmining.models.repository import Repository
-# from greenmining.utils import colored_print, load_json_file, save_json_file
-#
-#
-# class RepositoryController:
-#     # Controller for GitHub repository operations.
-#
-#     def __init__(self, config: Config):
-#         # Initialize controller with configuration.
-#         self.config = config
-#         self.github = Github(config.GITHUB_TOKEN)
-# ============================================================================
-
-# NEW GRAPHQL IMPLEMENTATION (5-10x faster)
-from tqdm import tqdm
-
-from greenmining.config import Config
-from greenmining.models.repository import Repository
-from greenmining.services.github_graphql_fetcher import GitHubGraphQLFetcher
-from greenmining.utils import colored_print, load_json_file, save_json_file
-
-
-class RepositoryController:
-    # Controller for GitHub repository operations using GraphQL API.
-
-    def __init__(self, config: Config):
-        # Initialize controller with configuration.
-        self.config = config
-        self.graphql_fetcher = GitHubGraphQLFetcher(config.GITHUB_TOKEN)
-
-    # ============================================================================
-    # OLD REST API METHOD (DEADCODE - 10x slower, high rate limit cost)
-    # ============================================================================
-    # def fetch_repositories(
-    #     self,
-    #     max_repos: int = None,
-    #     min_stars: int = None,
-    #     languages: list[str] = None,
-    #     keywords: str = None,
-    #     created_after: str = None,
-    #     created_before: str = None,
-    #     pushed_after: str = None,
-    #     pushed_before: str = None,
-    # ) -> list[Repository]:
-    #     # Fetch repositories from GitHub using REST API (slow).
-    #     max_repos = max_repos or self.config.MAX_REPOS
-    #     min_stars = min_stars or self.config.MIN_STARS
-    #     languages = languages or self.config.SUPPORTED_LANGUAGES
-    #     keywords = keywords or "microservices"
-    #
-    #     colored_print(f" Fetching up to {max_repos} repositories...", "cyan")
-    #     colored_print(f"   Keywords: {keywords}", "cyan")
-    #     colored_print(f"   Filters: min_stars={min_stars}", "cyan")
-    #
-    #     if created_after or created_before:
-    #         colored_print(
-    #             f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
-    #         )
-    #     if pushed_after or pushed_before:
-    #         colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
-    #
-    #     # Build search query with temporal filters
-    #     query = self._build_temporal_query(
-    #         keywords, min_stars, created_after, created_before, pushed_after, pushed_before
-    #     )
-    #
-    #     try:
-    #         # Execute search (REST API - many requests)
-    #         search_results = self.github.search_repositories(
-    #             query=query, sort="stars", order="desc"
-    #         )
-    #
-    #         total_found = search_results.totalCount
-    #         colored_print(f"   Found {total_found} repositories", "green")
-    #
-    #         # Fetch repositories (1 request per repo = slow)
-    #         repositories = []
-    #         with tqdm(total=min(max_repos, total_found), desc="Fetching", unit="repo") as pbar:
-    #             for idx, repo in enumerate(search_results):
-    #                 if idx >= max_repos:
-    #                     break
-    #
-    #                 try:
-    #                     repo_model = Repository.from_github_repo(repo, idx + 1)
-    #                     repositories.append(repo_model)
-    #                     pbar.update(1)
-    #                 except GithubException as e:
-    #                     colored_print(f"   Error: {repo.full_name}: {e}", "yellow")
-    #                     continue
-    #
-    #         # Save to file
-    #         repo_dicts = [r.to_dict() for r in repositories]
-    #         save_json_file(repo_dicts, self.config.REPOS_FILE)
-    #
-    #         colored_print(f" Fetched {len(repositories)} repositories", "green")
-    #         colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
-    #
-    #         return repositories
-    #
-    #     except Exception as e:
-    #         colored_print(f" Error fetching repositories: {e}", "red")
-    #         raise
-    # ============================================================================
-
-    def fetch_repositories(
-        self,
-        max_repos: int = None,
-        min_stars: int = None,
-        languages: list[str] = None,
-        keywords: str = None,
-        created_after: str = None,
-        created_before: str = None,
-        pushed_after: str = None,
-        pushed_before: str = None,
-    ) -> list[Repository]:
-        # Fetch repositories from GitHub using GraphQL API (5-10x faster).
-        max_repos = max_repos or self.config.MAX_REPOS
-        min_stars = min_stars or self.config.MIN_STARS
-        languages = languages or self.config.SUPPORTED_LANGUAGES
-        keywords = keywords or "microservices"
-
-        colored_print(f"🚀 Fetching up to {max_repos} repositories (GraphQL API)...", "cyan")
-        colored_print(f"   Keywords: {keywords}", "cyan")
-        colored_print(f"   Filters: min_stars={min_stars}", "cyan")
-
-        if created_after or created_before:
-            colored_print(
-                f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
-            )
-        if pushed_after or pushed_before:
-            colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
-
-        try:
-            # Use GraphQL API (much faster!)
-            repositories = self.graphql_fetcher.search_repositories(
-                keywords=keywords,
-                max_repos=max_repos,
-                min_stars=min_stars,
-                languages=languages,
-                created_after=created_after,
-                created_before=created_before,
-                pushed_after=pushed_after,
-                pushed_before=pushed_before,
-            )
-
-            # Save to file
-            repo_dicts = [r.to_dict() for r in repositories]
-            save_json_file(repo_dicts, self.config.REPOS_FILE)
-
-            colored_print(f"✓ Fetched {len(repositories)} repositories", "green")
-            colored_print(f"   Saved to: {self.config.REPOS_FILE}", "cyan")
-            colored_print(f"   API: GraphQL (5-10x faster than REST)", "green")
-
-            return repositories
-
-        except Exception as e:
-            colored_print(f"✗ Error fetching repositories: {e}", "red")
-            raise
-
-    # ============================================================================
-    # OLD REST API HELPER (DEADCODE - handled by GraphQL fetcher now)
-    # ============================================================================
-    # def _build_temporal_query(
-    #     self,
-    #     keywords: str,
-    #     min_stars: int,
-    #     created_after: str = None,
-    #     created_before: str = None,
-    #     pushed_after: str = None,
-    #     pushed_before: str = None,
-    # ) -> str:
-    #     # Build GitHub search query with temporal constraints.
-    #     query_parts = [keywords, f"stars:>={min_stars}"]
-    #
-    #     # Temporal filters
-    #     if created_after and created_before:
-    #         query_parts.append(f"created:{created_after}..{created_before}")
-    #     elif created_after:
-    #         query_parts.append(f"created:>={created_after}")
-    #     elif created_before:
-    #         query_parts.append(f"created:<={created_before}")
-    #
-    #     if pushed_after and pushed_before:
-    #         query_parts.append(f"pushed:{pushed_after}..{pushed_before}")
-    #     elif pushed_after:
-    #         query_parts.append(f"pushed:>={pushed_after}")
-    #     elif pushed_before:
-    #         query_parts.append(f"pushed:<={pushed_before}")
-    #
-    #     query = " ".join(query_parts)
-    #     colored_print(f"   Query: {query}", "cyan")
-    #     return query
-    # ============================================================================
-
-    def load_repositories(self) -> list[Repository]:
-        # Load repositories from file.
-        if not self.config.REPOS_FILE.exists():
-            raise FileNotFoundError(f"No repositories file found at {self.config.REPOS_FILE}")
-
-        repo_dicts = load_json_file(self.config.REPOS_FILE)
-        return [Repository.from_dict(r) for r in repo_dicts]
-
-    def get_repository_stats(self, repositories: list[Repository]) -> dict:
-        # Get statistics about fetched repositories.
-        if not repositories:
-            return {}
-
-        return {
-            "total": len(repositories),
-            "by_language": self._count_by_language(repositories),
-            "total_stars": sum(r.stars for r in repositories),
-            "avg_stars": sum(r.stars for r in repositories) / len(repositories),
-            "top_repo": max(repositories, key=lambda r: r.stars).full_name,
-        }
-
-    def _count_by_language(self, repositories: list[Repository]) -> dict:
-        # Count repositories by language.
-        counts = {}
-        for repo in repositories:
-            lang = repo.language or "Unknown"
-            counts[lang] = counts.get(lang, 0) + 1
-        return counts

greenmining-1.0.7/greenmining/services/github_fetcher.py

@@ -1,210 +0,0 @@
-# ================================================================================
-# DEADCODE - OLD REST API IMPLEMENTATION
-# ================================================================================
-#
-# This file contains the OLD GitHub REST API implementation.
-# It has been REPLACED by GitHubGraphQLFetcher for better performance.
-#
-# Performance comparison:
-#   REST API:    10+ requests for 100 repos, ~2 minutes
-#   GraphQL API: 1-2 requests for 100 repos, ~15 seconds (10x faster!)
-#
-# USE INSTEAD: greenmining.services.github_graphql_fetcher.GitHubGraphQLFetcher
-#
-# This file is kept for reference only. Do not use in production.
-#
-# ================================================================================
-
-# GitHub repository fetcher for green microservices mining.
-
-# from __future__ import annotations
-#
-# from datetime import datetime
-# from pathlib import Path
-# from typing import Any, Dict, List, Optional
-#
-# from github import Github, GithubException, RateLimitExceededException
-# from tqdm import tqdm
-#
-# from greenmining.config import get_config
-# from greenmining.utils import (
-#     colored_print,
-#     format_timestamp,
-#     print_banner,
-#     save_json_file,
-# )
-#
-#
-# class GitHubFetcher:
-#     # Fetches microservice repositories from GitHub using REST API (SLOW).
-#
-#     def __init__(
-#         self,
-#         token: str,
-#         max_repos: int = 100,
-#         min_stars: int = 100,
-#         languages: Optional[list[str]] = None,
-#         created_after: Optional[str] = None,
-#         created_before: Optional[str] = None,
-#         pushed_after: Optional[str] = None,
-#         pushed_before: Optional[str] = None,
-#     ):
-#         # Initialize GitHub fetcher.
-#         self.github = Github(token)
-#         self.max_repos = max_repos
-#         self.min_stars = min_stars
-#         self.languages = languages or [
-#             "Java",
-#             "Python",
-#             "Go",
-#             "JavaScript",
-#             "TypeScript",
-#             "C#",
-#             "Rust",
-#         ]
-#         self.created_after = created_after
-#         self.created_before = created_before
-#         self.pushed_after = pushed_after
-#         self.pushed_before = pushed_before
-#
-#     def search_repositories(self) -> list[dict[str, Any]]:
-#         # Search for microservice repositories (REST API - many requests).
-#         repositories = []
-#         keywords = ["microservices", "microservice-architecture", "cloud-native"]
-#
-#         colored_print(f"Searching for repositories with keywords: {', '.join(keywords)}", "cyan")
-#         colored_print(
-#             f"Filters: min_stars={self.min_stars}, languages={', '.join(self.languages)}", "cyan"
-#         )
-#
-#         # Build search query with temporal filters
-#         query = self._build_temporal_query(keywords)
-#
-#         try:
-#             # Execute search (1 request)
-#             search_results = self.github.search_repositories(
-#                 query=query, sort="stars", order="desc"
-#             )
-#
-#             total_found = search_results.totalCount
-#             colored_print(f"Found {total_found} repositories matching criteria", "green")
-#
-#             # Fetch repository details with progress bar (1 request per repo = SLOW)
-#             with tqdm(
-#                 total=min(self.max_repos, total_found), desc="Fetching repositories", unit="repo"
-#             ) as pbar:
-#                 for idx, repo in enumerate(search_results):
-#                     if idx >= self.max_repos:
-#                         break
-#
-#                     try:
-#                         repo_data = self._extract_repo_metadata(repo, idx + 1)
-#                         repositories.append(repo_data)
-#                         pbar.update(1)
-#                     except GithubException as e:
-#                         colored_print(f"Error fetching {repo.full_name}: {e}", "yellow")
-#                         continue
-#                     except RateLimitExceededException:
-#                         colored_print("Rate limit exceeded. Waiting...", "red")
-#                         self._handle_rate_limit()
-#                         continue
-#
-#             return repositories
-#
-#         except GithubException as e:
-#             colored_print(f"GitHub API error: {e}", "red")
-#             raise
-#         except Exception as e:
-#             colored_print(f"Unexpected error: {e}", "red")
-#             raise
-#
-#     def _extract_repo_metadata(self, repo, repo_id: int) -> dict[str, Any]:
-#         # Extract metadata from repository object.
-#         return {
-#             "repo_id": repo_id,
-#             "name": repo.name,
-#             "owner": repo.owner.login,
-#             "full_name": repo.full_name,
-#             "url": repo.html_url,
-#             "clone_url": repo.clone_url,
-#             "language": repo.language,
-#             "stars": repo.stargazers_count,
-#             "forks": repo.forks_count,
-#             "watchers": repo.watchers_count,
-#             "open_issues": repo.open_issues_count,
-#             "last_updated": repo.updated_at.isoformat() if repo.updated_at else None,
-#             "created_at": repo.created_at.isoformat() if repo.created_at else None,
-#             "description": repo.description or "",
-#             "main_branch": repo.default_branch,
-#             "topics": repo.get_topics() if hasattr(repo, "get_topics") else [],
-#             "size": repo.size,
-#             "has_issues": repo.has_issues,
-#             "has_wiki": repo.has_wiki,
-#             "archived": repo.archived,
-#             "license": repo.license.name if repo.license else None,
-#         }
-#
-#     def _build_temporal_query(self, keywords: list[str]) -> str:
-#         # Build GitHub search query with temporal constraints.
-#         query_parts = []
-#
-#         # Keywords
-#         keyword_query = " OR ".join(keywords)
-#         query_parts.append(f"({keyword_query})")
-#
-#         # Languages
-#         language_query = " OR ".join([f"language:{lang}" for lang in self.languages])
-#         query_parts.append(f"({language_query})")
-#
-#         # Stars
-#         query_parts.append(f"stars:>={self.min_stars}")
-#
-#         # Archived filter
-#         query_parts.append("archived:false")
-#
-#         # Temporal filters
-#         if self.created_after and self.created_before:
-#             query_parts.append(f"created:{self.created_after}..{self.created_before}")
-#         elif self.created_after:
-#             query_parts.append(f"created:>={self.created_after}")
-#         elif self.created_before:
-#             query_parts.append(f"created:<={self.created_before}")
-#
-#         if self.pushed_after and self.pushed_before:
-#             query_parts.append(f"pushed:{self.pushed_after}..{self.pushed_before}")
-#         elif self.pushed_after:
-#             query_parts.append(f"pushed:>={self.pushed_after}")
-#         elif self.pushed_before:
-#             query_parts.append(f"pushed:<={self.pushed_before}")
-#
-#         query = " ".join(query_parts)
-#         colored_print(f"Query: {query}", "cyan")
-#         return query
-#
-#     def _handle_rate_limit(self):
-#         # Handle GitHub API rate limiting.
-#         rate_limit = self.github.get_rate_limit()
-#         reset_time = rate_limit.core.reset
-#         wait_seconds = (reset_time - datetime.now()).total_seconds()
-#
-#         if wait_seconds > 0:
-#             colored_print(f"Rate limit will reset in {wait_seconds:.0f} seconds", "yellow")
-#             import time
-#
-#             time.sleep(min(wait_seconds + 10, 60))  # Wait with max 60 seconds
-#
-#     def save_results(self, repositories: list[dict[str, Any]], output_file: Path):
-#         # Save fetched repositories to JSON file.
-#         data = {
-#             "metadata": {
-#                 "fetched_at": format_timestamp(),
-#                 "total_repos": len(repositories),
-#                 "min_stars": self.min_stars,
-#                 "languages": self.languages,
-#                 "search_keywords": ["microservices", "microservice-architecture", "cloud-native"],
-#             },
-#             "repositories": repositories,
-#         }
-#
-#         save_json_file(data, output_file)
-#         colored_print(f"Saved {len(repositories)} repositories to {output_file}", "green")