greenmining 0.1.11__py3-none-any.whl → 1.0.1__py3-none-any.whl

greenmining/cli.py

@@ -29,8 +29,27 @@ def cli(config_file, verbose):
 @click.option(
     "--languages", default="Python,Java,Go,JavaScript,TypeScript", help="Comma-separated languages"
 )
-def fetch(max_repos, min_stars, languages):
-    """Fetch top microservice repositories from GitHub."""
+@click.option(
+    "--keywords",
+    default="microservices",
+    type=str,
+    help="Search keywords (e.g., 'kubernetes', 'docker', 'cloud-native')",
+)
+@click.option("--created-after", type=str, help="Repository created after (YYYY-MM-DD)")
+@click.option("--created-before", type=str, help="Repository created before (YYYY-MM-DD)")
+@click.option("--pushed-after", type=str, help="Repository pushed after (YYYY-MM-DD)")
+@click.option("--pushed-before", type=str, help="Repository pushed before (YYYY-MM-DD)")
+def fetch(
+    max_repos,
+    min_stars,
+    languages,
+    keywords,
+    created_after,
+    created_before,
+    pushed_after,
+    pushed_before,
+):
+    """Fetch repositories from GitHub based on custom search keywords."""
     presenter.show_banner()
     colored_print(f"\n🎯 Target: {max_repos} repositories\n", "cyan")
 
@@ -39,7 +58,14 @@ def fetch(max_repos, min_stars, languages):
 
     try:
         repositories = controller.fetch_repositories(
-            max_repos=max_repos, min_stars=min_stars, languages=lang_list
+            max_repos=max_repos,
+            min_stars=min_stars,
+            languages=lang_list,
+            keywords=keywords,
+            created_after=created_after,
+            created_before=created_before,
+            pushed_after=pushed_after,
+            pushed_before=pushed_before,
         )
 
         # Show results
@@ -61,11 +87,12 @@ def fetch(max_repos, min_stars, languages):
 @click.option("--max-commits", default=50, type=int, help="Max commits per repository")
 @click.option("--skip-merges", is_flag=True, default=True, help="Skip merge commits")
 @click.option("--days-back", default=730, type=int, help="Days to look back (default: 2 years)")
-def extract(max_commits, skip_merges, days_back):
+@click.option("--timeout", default=60, type=int, help="Timeout per repo in seconds (default: 60)")
+def extract(max_commits, skip_merges, days_back, timeout):
     """Extract commits from fetched repositories."""
     presenter.show_banner()
 
-    from services.commit_extractor import CommitExtractor
+    from greenmining.services.commit_extractor import CommitExtractor
 
     try:
         # Load repositories
@@ -80,14 +107,14 @@ def extract(max_commits, skip_merges, days_back):
 
         # Extract commits
         extractor = CommitExtractor(
-            max_commits=max_commits, skip_merges=skip_merges, days_back=days_back
+            max_commits=max_commits, skip_merges=skip_merges, days_back=days_back, timeout=timeout
         )
         commits = extractor.extract_from_repositories(
             repositories=[r.to_dict() for r in repositories]
         )
 
         # Save commits
-        from utils import save_json_file
+        from greenmining.utils import save_json_file
 
         save_json_file(commits, config.COMMITS_FILE)
         colored_print(f"   Saved to: {config.COMMITS_FILE}", "cyan")
@@ -113,12 +140,15 @@ def extract(max_commits, skip_merges, days_back):
 
 @cli.command()
 @click.option("--batch-size", default=10, type=int, help="Batch size for processing")
-def analyze(batch_size):
+@click.option("--enable-diff-analysis", is_flag=True, help="Enable code diff analysis (slower)")
+@click.option("--enable-nlp", is_flag=True, help="Enable NLP-enhanced pattern detection")
+@click.option("--enable-ml-features", is_flag=True, help="Enable ML feature extraction")
+def analyze(batch_size, enable_diff_analysis, enable_nlp, enable_ml_features):
     """Analyze commits for green software patterns."""
     presenter.show_banner()
 
-    from services.data_analyzer import DataAnalyzer
-    from utils import save_json_file
+    from greenmining.services.data_analyzer import DataAnalyzer
+    from greenmining.utils import save_json_file
 
     try:
         # Load commits
@@ -127,12 +157,27 @@ def analyze(batch_size):
 
         commits = load_json_file(config.COMMITS_FILE)
         colored_print(f"\n🔬 Analyzing {len(commits)} commits for green patterns...\n", "cyan")
-        colored_print("   Method: Keyword-based heuristic analysis\n", "cyan")
+
+        # Show enabled methods
+        methods = ["Keyword"]
+        if enable_diff_analysis:
+            methods.append("Code Diff")
+        if enable_nlp:
+            methods.append("NLP")
+        if enable_ml_features:
+            methods.append("ML Features")
+
+        colored_print(f"   Methods: {' + '.join(methods)}\n", "cyan")
         colored_print(f"   Batch size: {batch_size}\n", "cyan")
 
         # Analyze
-        analyzer = DataAnalyzer()
-        results = analyzer.analyze_commits_batch(commits, batch_size=batch_size)
+        analyzer = DataAnalyzer(
+            batch_size=batch_size,
+            enable_diff_analysis=enable_diff_analysis,
+            enable_nlp=enable_nlp,
+            enable_ml_features=enable_ml_features,
+        )
+        results = analyzer.analyze_commits(commits)
 
         # Save results
         save_json_file(results, config.ANALYSIS_FILE)
@@ -159,12 +204,20 @@ def analyze(batch_size):
 
 
 @cli.command()
-def aggregate():
+@click.option("--enable-enhanced-stats", is_flag=True, help="Enable enhanced statistical analysis")
+@click.option("--enable-temporal", is_flag=True, help="Enable temporal trend analysis")
+@click.option(
+    "--temporal-granularity",
+    default="quarter",
+    type=click.Choice(["day", "week", "month", "quarter", "year"]),
+    help="Temporal analysis granularity",
+)
+def aggregate(enable_enhanced_stats, enable_temporal, temporal_granularity):
     """Aggregate analysis results and generate statistics."""
     presenter.show_banner()
 
-    from services.data_aggregator import DataAggregator
-    from utils import save_json_file
+    from greenmining.services.data_aggregator import DataAggregator
+    from greenmining.utils import save_json_file
 
     try:
         # Load data
@@ -176,8 +229,20 @@ def aggregate():
 
         colored_print(f"\n📊 Aggregating results from {len(results)} commits...\n", "cyan")
 
+        # Show enabled features
+        if enable_enhanced_stats:
+            colored_print("   Enhanced statistics: Enabled\n", "cyan")
+        if enable_temporal:
+            colored_print(
+                f"   Temporal analysis: Enabled (granularity: {temporal_granularity})\n", "cyan"
+            )
+
         # Aggregate
-        aggregator = DataAggregator()
+        aggregator = DataAggregator(
+            enable_enhanced_stats=enable_enhanced_stats,
+            enable_temporal=enable_temporal,
+            temporal_granularity=temporal_granularity,
+        )
         aggregated = aggregator.aggregate(results, repos)
 
         # Save
@@ -187,7 +252,15 @@ def aggregate():
         presenter.show_analysis_results(aggregated)
 
         if aggregated.get("known_patterns"):
-            presenter.show_pattern_distribution(aggregated["known_patterns"], limit=10)
+            # Convert list format to dict format expected by presenter
+            patterns_dict = {}
+            for pattern in aggregated["known_patterns"]:
+                patterns_dict[pattern["pattern_name"]] = {
+                    "count": pattern["count"],
+                    "percentage": pattern["percentage"],
+                    "confidence_distribution": pattern.get("confidence_breakdown", {}),
+                }
+            presenter.show_pattern_distribution(patterns_dict, limit=10)
 
         presenter.show_success(f"Aggregation complete! Results saved to {config.AGGREGATED_FILE}")
 
@@ -202,20 +275,48 @@ def report(output):
     """Generate comprehensive markdown report."""
     presenter.show_banner()
 
-    from services.reports import ReportGenerator
+    from greenmining.services.reports import ReportGenerator
 
     try:
         # Load aggregated data
         if not config.AGGREGATED_FILE.exists():
             raise FileNotFoundError("No aggregated data found. Run 'aggregate' first.")
 
+        # Load analysis results
+        if not config.ANALYSIS_FILE.exists():
+            raise FileNotFoundError("No analysis results found. Run 'analyze' first.")
+
+        # Load repository data
+        if not config.REPOS_FILE.exists():
+            raise FileNotFoundError("No repository data found. Run 'fetch' first.")
+
         aggregated = load_json_file(config.AGGREGATED_FILE)
+        analysis_results = load_json_file(config.ANALYSIS_FILE)
+        repos_data = load_json_file(config.REPOS_FILE)
+
+        # Wrap analysis results if it's a list
+        if isinstance(analysis_results, list):
+            analysis = {"results": analysis_results, "total": len(analysis_results)}
+        else:
+            analysis = analysis_results
+
+        # Wrap repos data if it's a list
+        if isinstance(repos_data, list):
+            repos = {"repositories": repos_data, "total": len(repos_data)}
+        else:
+            repos = repos_data
 
         colored_print("\n📄 Generating comprehensive report...\n", "cyan")
 
         # Generate report
         generator = ReportGenerator()
-        report_path = generator.generate_report(aggregated, output)
+        report_content = generator.generate_report(aggregated, analysis, repos)
+
+        # Save report
+        from pathlib import Path
+
+        report_path = Path(output)
+        report_path.write_text(report_content)
 
         presenter.show_success(f"Report generated: {report_path}")
         colored_print("\n📖 The report includes:", "cyan")
@@ -317,8 +418,8 @@ def pipeline(max_repos, skip_fetch):
 
         # Phase 2: Extract
         colored_print("\n[2/5] 📝 Extracting commits...", "cyan")
-        from services.commit_extractor import CommitExtractor
-        from utils import save_json_file
+        from greenmining.services.commit_extractor import CommitExtractor
+        from greenmining.utils import save_json_file
 
         controller = RepositoryController(config)
         repos = controller.load_repositories()
@@ -329,7 +430,7 @@ def pipeline(max_repos, skip_fetch):
 
         # Phase 3: Analyze
         colored_print("\n[3/5] 🔬 Analyzing commits...", "cyan")
-        from services.data_analyzer import DataAnalyzer
+        from greenmining.services.data_analyzer import DataAnalyzer
 
         commits = load_json_file(config.COMMITS_FILE)
         analyzer = DataAnalyzer()
@@ -341,7 +442,7 @@ def pipeline(max_repos, skip_fetch):
 
         # Phase 4: Aggregate
         colored_print("\n[4/5] 📊 Aggregating results...", "cyan")
-        from services.data_aggregator import DataAggregator
+        from greenmining.services.data_aggregator import DataAggregator
 
         aggregator = DataAggregator()
         aggregated = aggregator.aggregate(results, [r.to_dict() for r in repos])
@@ -349,7 +450,7 @@ def pipeline(max_repos, skip_fetch):
 
         # Phase 5: Report
         colored_print("\n[5/5] 📄 Generating report...", "cyan")
-        from services.reports import ReportGenerator
+        from greenmining.services.reports import ReportGenerator
 
         generator = ReportGenerator()
         generator.generate_report(aggregated)

greenmining/config.py

@@ -49,6 +49,27 @@ class Config:
         self.COMMITS_PER_REPO = int(os.getenv("COMMITS_PER_REPO", "50"))
         self.DAYS_BACK = int(os.getenv("DAYS_BACK", "730"))  # 2 years
 
+        # Advanced Analyzer Configuration
+        self.ENABLE_NLP_ANALYSIS = os.getenv("ENABLE_NLP_ANALYSIS", "false").lower() == "true"
+        self.ENABLE_TEMPORAL_ANALYSIS = (
+            os.getenv("ENABLE_TEMPORAL_ANALYSIS", "false").lower() == "true"
+        )
+        self.TEMPORAL_GRANULARITY = os.getenv(
+            "TEMPORAL_GRANULARITY", "quarter"
+        )  # day, week, month, quarter, year
+        self.ENABLE_ML_FEATURES = os.getenv("ENABLE_ML_FEATURES", "false").lower() == "true"
+        self.VALIDATION_SAMPLE_SIZE = int(os.getenv("VALIDATION_SAMPLE_SIZE", "30"))
+
+        # Temporal Filtering (NEW)
+        self.CREATED_AFTER = os.getenv("CREATED_AFTER")  # YYYY-MM-DD
+        self.CREATED_BEFORE = os.getenv("CREATED_BEFORE")  # YYYY-MM-DD
+        self.PUSHED_AFTER = os.getenv("PUSHED_AFTER")  # YYYY-MM-DD
+        self.PUSHED_BEFORE = os.getenv("PUSHED_BEFORE")  # YYYY-MM-DD
+        self.COMMIT_DATE_FROM = os.getenv("COMMIT_DATE_FROM")  # YYYY-MM-DD
+        self.COMMIT_DATE_TO = os.getenv("COMMIT_DATE_TO")  # YYYY-MM-DD
+        self.MIN_COMMITS = int(os.getenv("MIN_COMMITS", "0"))
+        self.ACTIVITY_WINDOW_DAYS = int(os.getenv("ACTIVITY_WINDOW_DAYS", "730"))
+
         # Analysis Configuration
         self.BATCH_SIZE = int(os.getenv("BATCH_SIZE", "10"))
 

greenmining/controllers/repository_controller.py

@@ -17,7 +17,15 @@ class RepositoryController:
         self.github = Github(config.GITHUB_TOKEN)
 
     def fetch_repositories(
-        self, max_repos: int = None, min_stars: int = None, languages: list[str] = None
+        self,
+        max_repos: int = None,
+        min_stars: int = None,
+        languages: list[str] = None,
+        keywords: str = None,
+        created_after: str = None,
+        created_before: str = None,
+        pushed_after: str = None,
+        pushed_before: str = None,
     ) -> list[Repository]:
         """Fetch repositories from GitHub.
 
@@ -25,6 +33,11 @@ class RepositoryController:
             max_repos: Maximum number of repositories to fetch
             min_stars: Minimum stars filter
             languages: List of programming languages to filter
+            keywords: Custom search keywords (default: "microservices")
+            created_after: Repository created after date (YYYY-MM-DD)
+            created_before: Repository created before date (YYYY-MM-DD)
+            pushed_after: Repository pushed after date (YYYY-MM-DD)
+            pushed_before: Repository pushed before date (YYYY-MM-DD)
 
         Returns:
             List of Repository model instances
@@ -32,12 +45,23 @@ class RepositoryController:
         max_repos = max_repos or self.config.MAX_REPOS
         min_stars = min_stars or self.config.MIN_STARS
         languages = languages or self.config.SUPPORTED_LANGUAGES
+        keywords = keywords or "microservices"
 
         colored_print(f"🔍 Fetching up to {max_repos} repositories...", "cyan")
+        colored_print(f"   Keywords: {keywords}", "cyan")
         colored_print(f"   Filters: min_stars={min_stars}", "cyan")
 
-        # Build search query - simpler approach
-        query = f"microservices stars:>={min_stars}"
+        if created_after or created_before:
+            colored_print(
+                f"   Created: {created_after or 'any'} to {created_before or 'any'}", "cyan"
+            )
+        if pushed_after or pushed_before:
+            colored_print(f"   Pushed: {pushed_after or 'any'} to {pushed_before or 'any'}", "cyan")
+
+        # Build search query with temporal filters
+        query = self._build_temporal_query(
+            keywords, min_stars, created_after, created_before, pushed_after, pushed_before
+        )
 
         try:
             # Execute search
@@ -76,6 +100,37 @@ class RepositoryController:
             colored_print(f"❌ Error fetching repositories: {e}", "red")
             raise
 
+    def _build_temporal_query(
+        self,
+        keywords: str,
+        min_stars: int,
+        created_after: str = None,
+        created_before: str = None,
+        pushed_after: str = None,
+        pushed_before: str = None,
+    ) -> str:
+        """Build GitHub search query with temporal constraints."""
+        query_parts = [keywords, f"stars:>={min_stars}"]
+
+        # Temporal filters
+        if created_after and created_before:
+            query_parts.append(f"created:{created_after}..{created_before}")
+        elif created_after:
+            query_parts.append(f"created:>={created_after}")
+        elif created_before:
+            query_parts.append(f"created:<={created_before}")
+
+        if pushed_after and pushed_before:
+            query_parts.append(f"pushed:{pushed_after}..{pushed_before}")
+        elif pushed_after:
+            query_parts.append(f"pushed:>={pushed_after}")
+        elif pushed_before:
+            query_parts.append(f"pushed:<={pushed_before}")
+
+        query = " ".join(query_parts)
+        colored_print(f"   Query: {query}", "cyan")
+        return query
+
     def load_repositories(self) -> list[Repository]:
         """Load repositories from file.
 

greenmining/gsf_patterns.py

@@ -356,7 +356,12 @@ GSF_PATTERNS = {
     "compress_ml_models": {
         "name": "Compress ML Models for Inference",
         "category": "ai",
-        "keywords": ["compress", "model", "quantiz", "prune", "distill"],
+        "keywords": [
+            "model compression",
+            "quantization",
+            "model pruning",
+            "knowledge distillation",
+        ],
         "description": "Reduce model size through quantization, pruning, distillation",
         "sci_impact": "Dramatically reduces inference energy and memory",
     },
@@ -370,14 +375,14 @@ GSF_PATTERNS = {
     "energy_efficient_ai_edge": {
         "name": "Energy Efficient AI at Edge",
         "category": "ai",
-        "keywords": ["edge", "ai", "inference", "local", "device"],
+        "keywords": ["edge inference", "edge ai", "edge ml", "tflite", "onnx runtime"],
         "description": "Run inference on edge devices when possible",
         "sci_impact": "Eliminates network transfer, uses local compute",
     },
     "energy_efficient_framework": {
         "name": "Energy Efficient Framework",
         "category": "ai",
-        "keywords": ["framework", "tensorflow", "pytorch", "efficient"],
+        "keywords": ["tensorflow", "pytorch", "onnx", "jax", "huggingface"],
         "description": "Choose ML frameworks optimized for efficiency",
         "sci_impact": "Different frameworks have different energy profiles",
     },
@@ -405,14 +410,14 @@ GSF_PATTERNS = {
     "right_hardware_ai": {
         "name": "Right Hardware Type for AI",
         "category": "ai",
-        "keywords": ["hardware", "gpu", "tpu", "accelerator", "ai"],
+        "keywords": ["gpu training", "tpu", "cuda", "nvidia ai", "ml accelerator"],
         "description": "Use appropriate hardware (GPU/TPU) for AI workloads",
         "sci_impact": "Specialized hardware is more energy efficient",
     },
     "serverless_ml": {
         "name": "Serverless Model Development",
         "category": "ai",
-        "keywords": ["serverless", "ml", "sagemaker", "vertex", "lambda"],
+        "keywords": ["sagemaker", "vertex ai", "azure ml", "lambda inference", "serverless ml"],
         "description": "Use serverless platforms for ML development",
         "sci_impact": "Pay-per-use, no idle resources",
     },

greenmining/models/aggregated_stats.py

@@ -1,7 +1,9 @@
 """Aggregated Statistics Model - Represents aggregated analysis data."""
 
+from __future__ import annotations
+
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import Dict, List, Optional
 
 
 @dataclass

greenmining/models/commit.py

@@ -1,6 +1,9 @@
 """Commit Model - Represents a Git commit."""
 
+from __future__ import annotations
+
 from dataclasses import dataclass, field
+from typing import List
 
 
 @dataclass

greenmining/models/repository.py

@@ -1,7 +1,9 @@
 """Repository Model - Represents a GitHub repository."""
 
+from __future__ import annotations
+
 from dataclasses import dataclass, field
-from typing import Optional
+from typing import List, Optional
 
 
 @dataclass

greenmining/presenters/console_presenter.py

@@ -1,6 +1,8 @@
 """Console Presenter - Handles console output formatting."""
 
-from typing import Any
+from __future__ import annotations
+
+from typing import Any, Dict, List
 
 from tabulate import tabulate
 

greenmining/services/commit_extractor.py

@@ -1,9 +1,11 @@
 """Commit extractor for green microservices mining."""
 
+from __future__ import annotations
+
 import json
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, List, Optional
 
 import click
 from github import Github
@@ -29,6 +31,7 @@ class CommitExtractor:
         skip_merges: bool = True,
         days_back: int = 730,
         github_token: str | None = None,
+        timeout: int = 60,
     ):
         """Initialize commit extractor.
 
@@ -37,12 +40,14 @@ class CommitExtractor:
             skip_merges: Skip merge commits
             days_back: Only analyze commits from last N days
             github_token: GitHub API token (optional)
+            timeout: Timeout in seconds per repository (default: 60)
         """
         self.max_commits = max_commits
         self.skip_merges = skip_merges
         self.days_back = days_back
         self.cutoff_date = datetime.now() - timedelta(days=days_back)
         self.github = Github(github_token) if github_token else None
+        self.timeout = timeout
 
     def extract_from_repositories(self, repositories: list[dict[str, Any]]) -> list[dict[str, Any]]:
         """Extract commits from list of repositories.
@@ -62,14 +67,35 @@ class CommitExtractor:
             "cyan",
         )
 
+        import signal
+
+        def timeout_handler(signum, frame):
+            raise TimeoutError("Repository extraction timeout")
+
         with tqdm(total=len(repositories), desc="Processing repositories", unit="repo") as pbar:
             for repo in repositories:
                 try:
+                    # Set timeout alarm
+                    signal.signal(signal.SIGALRM, timeout_handler)
+                    signal.alarm(self.timeout)
+
                     commits = self._extract_repo_commits(repo)
                     all_commits.extend(commits)
+
+                    # Cancel alarm
+                    signal.alarm(0)
+
                     pbar.set_postfix({"commits": len(all_commits), "failed": len(failed_repos)})
                     pbar.update(1)
+                except TimeoutError:
+                    signal.alarm(0)  # Cancel alarm
+                    colored_print(
+                        f"\nTimeout processing {repo['full_name']} (>{self.timeout}s)", "yellow"
+                    )
+                    failed_repos.append(repo["full_name"])
+                    pbar.update(1)
                 except Exception as e:
+                    signal.alarm(0)  # Cancel alarm
                     colored_print(f"\nError processing {repo['full_name']}: {e}", "yellow")
                     failed_repos.append(repo["full_name"])
                     pbar.update(1)