greenmining 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

{greenmining-1.0.2.dist-info → greenmining-1.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: greenmining
-Version: 1.0.2
+Version: 1.0.4
 Summary: Analyze GitHub repositories to identify green software engineering patterns and energy-efficient practices
 Author-email: Adam Bouafia <a.bouafia@student.vu.nl>
 License: MIT
@@ -23,20 +23,19 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Operating System :: OS Independent
-Classifier: Environment :: Console
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: PyGithub>=2.1.1
 Requires-Dist: PyDriller>=2.5
 Requires-Dist: pandas>=2.2.0
-Requires-Dist: click>=8.1.7
 Requires-Dist: colorama>=0.4.6
 Requires-Dist: tabulate>=0.9.0
 Requires-Dist: tqdm>=4.66.0
 Requires-Dist: matplotlib>=3.8.0
 Requires-Dist: plotly>=5.18.0
 Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.31.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.4.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
@@ -44,7 +43,7 @@ Requires-Dist: pytest-mock>=3.12.0; extra == "dev"
 Requires-Dist: black>=23.12.0; extra == "dev"
 Requires-Dist: ruff>=0.1.9; extra == "dev"
 Requires-Dist: mypy>=1.8.0; extra == "dev"
-Requires-Dist: build>=1.0.3; extra == "dev"
+Requires-Dist: build>=1.0.4; extra == "dev"
 Requires-Dist: twine>=4.0.2; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: sphinx>=7.2.0; extra == "docs"
@@ -88,37 +87,6 @@ docker pull adambouafia/greenmining:latest
 
 ## Quick Start
 
-### CLI Usage
-
-```bash
-# Set your GitHub token
-export GITHUB_TOKEN="your_github_token"
-
-# Run full analysis pipeline
-greenmining pipeline --max-repos 100
-
-# Fetch repositories with custom keywords
-greenmining fetch --max-repos 100 --min-stars 100 --keywords "kubernetes docker cloud-native"
-
-# Fetch with default (microservices)
-greenmining fetch --max-repos 100 --min-stars 100
-
-# Extract commits
-greenmining extract --max-commits 50
-
-# Analyze for green patterns
-greenmining analyze
-
-# Analyze with advanced features
-greenmining analyze --enable-nlp --enable-ml-features --enable-diff-analysis
-
-# Aggregate results with temporal analysis
-greenmining aggregate --enable-temporal --temporal-granularity quarter --enable-enhanced-stats
-
-# Generate report
-greenmining report
-```
-
 ### Python API
 
 #### Basic Pattern Detection
@@ -197,23 +165,10 @@ extractor = CommitExtractor(
 # Initialize analyzer with advanced features
 analyzer = DataAnalyzer(
     enable_diff_analysis=False,      # Enable code diff analysis (slower but more accurate)
-    enable_nlp=True,                 # Enable NLP-enhanced pattern detection
-    enable_ml_features=True,         # Enable ML feature extraction
     patterns=None,                   # Custom pattern dict (default: GSF_PATTERNS)
     batch_size=10                    # Batch processing size (default: 10)
 )
 
-# Optional: Configure NLP analyzer separately
-nlp_analyzer = NLPAnalyzer(
-    enable_stemming=True,            # Enable morphological analysis (optimize→optimizing)
-    enable_synonyms=True             # Enable semantic synonym matching (cache→buffer)
-)
-
-# Optional: Configure ML feature extractor
-ml_extractor = MLFeatureExtractor(
-    green_keywords=None              # Custom keyword list (default: built-in 19 keywords)
-)
-
 # Extract commits from first repo
 commits = extractor.extract_commits(
     repository=repos[0],             # PyGithub Repository object
@@ -229,18 +184,9 @@ commits = extractor.extract_commits(
 
 **DataAnalyzer Parameters:**
 - `enable_diff_analysis` (bool, default=False): Enable code diff analysis (slower)
-- `enable_nlp` (bool, default=False): Enable NLP-enhanced pattern detection
-- `enable_ml_features` (bool, default=False): Enable ML feature extraction
 - `patterns` (dict, optional): Custom pattern dictionary (default: GSF_PATTERNS)
 - `batch_size` (int, default=10): Number of commits to process in each batch
 
-**NLPAnalyzer Parameters:**
-- `enable_stemming` (bool, default=True): Enable morphological variant matching
-- `enable_synonyms` (bool, default=True): Enable semantic synonym expansion
-
-**MLFeatureExtractor Parameters:**
-- `green_keywords` (list[str], optional): Custom green keywords list
-
 # Analyze commits for green patterns
 results = []
 for commit in commits:
@@ -306,7 +252,7 @@ from greenmining.analyzers.qualitative_analyzer import QualitativeAnalyzer
 # Initialize aggregator with all advanced features
 aggregator = DataAggregator(
     config=None,                        # Config object (optional)
-    enable_enhanced_stats=True,         # Enable statistical analysis (correlations, trends)
+    enable_stats=True,                  # Enable statistical analysis (correlations, trends)
     enable_temporal=True,               # Enable temporal trend analysis
     temporal_granularity="quarter"      # Time granularity: day/week/month/quarter/year
 )
@@ -330,7 +276,7 @@ aggregated = aggregator.aggregate(
 
 **DataAggregator Parameters:**
 - `config` (Config, optional): Configuration object
-- `enable_enhanced_stats` (bool, default=False): Enable pattern correlations and effect size analysis
+- `enable_stats` (bool, default=False): Enable pattern correlations and effect size analysis
 - `enable_temporal` (bool, default=False): Enable temporal trend analysis over time
 - `temporal_granularity` (str, default="quarter"): Time granularity (day/week/month/quarter/year)
 
@@ -397,20 +343,127 @@ controller.generate_report()
 print("Analysis complete! Check data/ directory for results.")
 ```
 
+#### Complete Working Example: Full Pipeline
+
+This is a complete, production-ready example that demonstrates the entire analysis pipeline. This example successfully analyzed 100 repositories with 30,543 commits in our testing.
+
+```python
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Import from greenmining package
+from greenmining import fetch_repositories
+from greenmining.services.commit_extractor import CommitExtractor
+from greenmining.services.data_analyzer import DataAnalyzer
+from greenmining.services.data_aggregator import DataAggregator
+
+# Configuration
+token = os.getenv("GITHUB_TOKEN")
+output_dir = Path("results")
+output_dir.mkdir(exist_ok=True)
+
+# STAGE 1: Fetch Repositories
+print("Fetching repositories...")
+repositories = fetch_repositories(
+    github_token=token,
+    max_repos=100,
+    min_stars=10,
+    keywords="software engineering",
+)
+print(f"✓ Fetched {len(repositories)} repositories")
+
+# STAGE 2: Extract Commits
+print("\nExtracting commits...")
+extractor = CommitExtractor(
+    github_token=token,
+    max_commits=1000,
+    skip_merges=True,
+    days_back=730,
+    timeout=120,
+)
+all_commits = extractor.extract_from_repositories(repositories)
+print(f"✓ Extracted {len(all_commits)} commits")
+
+# Save commits
+extractor.save_results(
+    all_commits,
+    output_dir / "commits.json",
+    len(repositories)
+)
+
+# STAGE 3: Analyze Commits
+print("\nAnalyzing commits...")
+analyzer = DataAnalyzer(
+    enable_diff_analysis=False,  # Set to True for detailed code analysis (slower)
+)
+analyzed_commits = analyzer.analyze_commits(all_commits)
+
+# Count green-aware commits
+green_count = sum(1 for c in analyzed_commits if c.get("green_aware", False))
+green_percentage = (green_count / len(analyzed_commits) * 100) if analyzed_commits else 0
+print(f"✓ Analyzed {len(analyzed_commits)} commits")
+print(f"✓ Green-aware: {green_count} ({green_percentage:.1f}%)")
+
+# Save analysis
+analyzer.save_results(analyzed_commits, output_dir / "analyzed.json")
+
+# STAGE 4: Aggregate Results
+print("\nAggregating results...")
+aggregator = DataAggregator(
+    enable_stats=True,
+    enable_temporal=True,
+    temporal_granularity="quarter",
+)
+results = aggregator.aggregate(analyzed_commits, repositories)
+
+# STAGE 5: Save Results
+print("\nSaving results...")
+aggregator.save_results(
+    results,
+    output_dir / "aggregated.json",
+    output_dir / "aggregated.csv",
+    analyzed_commits
+)
+
+# Print summary
+print("\n" + "="*80)
+print("ANALYSIS COMPLETE")
+print("="*80)
+aggregator.print_summary(results)
+print(f"\n📁 Results saved in: {output_dir.absolute()}")
+```
+
+**What this example does:**
+
+1. **Fetches repositories** from GitHub based on keywords and filters
+2. **Extracts commits** from each repository (up to 1000 per repo)
+3. **Analyzes commits** for green software patterns
+4. **Aggregates results** with temporal analysis and statistics
+5. **Saves results** to JSON and CSV files for further analysis
+
+**Expected output files:**
+- `commits.json` - All extracted commits with metadata
+- `analyzed.json` - Commits analyzed for green patterns
+- `aggregated.json` - Summary statistics and pattern distributions
+- `aggregated.csv` - Tabular format for spreadsheet analysis
+- `metadata.json` - Experiment configuration and timing
+
+**Performance:** This pipeline successfully processed 100 repositories (30,543 commits) in approximately 6.4 hours, identifying 7,600 green-aware commits (24.9%).
+
 ### Docker Usage
 
 ```bash
-# Run analysis pipeline
-docker run -v $(pwd)/data:/app/data \
-           adambouafia/greenmining:latest --help
+# Interactive shell with Python
+docker run -it -v $(pwd)/data:/app/data \
+           adambouafia/greenmining:latest python
 
-# With custom configuration
-docker run -v $(pwd)/.env:/app/.env:ro \
-           -v $(pwd)/data:/app/data \
-           adambouafia/greenmining:latest pipeline --max-repos 50
-
-# Interactive shell
-docker run -it adambouafia/greenmining:latest /bin/bash
+# Run Python script
+docker run -v $(pwd)/data:/app/data \
+           adambouafia/greenmining:latest python your_script.py
 ```
 
 ## Configuration
@@ -436,14 +489,12 @@ EXCLUDE_BOT_COMMITS=true
 
 # Optional - Analysis Features
 ENABLE_DIFF_ANALYSIS=false
-ENABLE_NLP=true
-ENABLE_ML_FEATURES=true
 BATCH_SIZE=10
 
 # Optional - Temporal Analysis
 ENABLE_TEMPORAL=true
 TEMPORAL_GRANULARITY=quarter
-ENABLE_ENHANCED_STATS=true
+ENABLE_STATS=true
 
 # Optional - Output
 OUTPUT_DIR=./data
@@ -473,14 +524,12 @@ config = Config(
     
     # Analysis Options
     enable_diff_analysis=False,             # Enable code diff analysis
-    enable_nlp=True,                        # Enable NLP features
-    enable_ml_features=True,                # Enable ML feature extraction
     batch_size=10,                          # Batch processing size
     
     # Temporal Analysis
     enable_temporal=True,                   # Enable temporal trend analysis
     temporal_granularity="quarter",         # day/week/month/quarter/year
-    enable_enhanced_stats=True,             # Enable statistical analysis
+    enable_stats=True,                      # Enable statistical analysis
     
     # Output Configuration
     output_dir="./data",                    # Output directory path
@@ -506,6 +555,50 @@ config = Config(
 - **Docker Support**: Pre-built images for containerized analysis
 - **Programmatic API**: Full Python API for custom workflows and integrations
 - **Clean Architecture**: Modular design with services layer (Fetcher, Extractor, Analyzer, Aggregator, Reports)
+- **Energy Measurement**: Real-time energy consumption tracking via RAPL (Linux) or CodeCarbon (cross-platform)
+
+### Energy Measurement
+
+greenmining includes built-in energy measurement capabilities for tracking the carbon footprint of your analysis:
+
+#### Backend Options
+
+| Backend | Platform | Metrics | Requirements |
+|---------|----------|---------|--------------|
+| **RAPL** | Linux (Intel/AMD) | CPU/RAM energy (Joules) | `/sys/class/powercap/` access |
+| **CodeCarbon** | Cross-platform | Energy + Carbon emissions (gCO2) | `pip install codecarbon` |
+
+#### Python API
+
+```python
+from greenmining.energy import RAPLEnergyMeter, CodeCarbonMeter
+
+# RAPL (Linux only)
+rapl = RAPLEnergyMeter()
+if rapl.is_available():
+    rapl.start()
+    # ... run analysis ...
+    result = rapl.stop()
+    print(f"Energy: {result.energy_joules:.2f} J")
+
+# CodeCarbon (cross-platform)
+cc = CodeCarbonMeter()
+if cc.is_available():
+    cc.start()
+    # ... run analysis ...
+    result = cc.stop()
+    print(f"Energy: {result.energy_joules:.2f} J")
+    print(f"Carbon: {result.carbon_grams:.4f} gCO2")
+```
+
+#### Experiment Results
+
+CodeCarbon was verified with a real experiment:
+- **Repository**: flask (pallets/flask)
+- **Commits analyzed**: 10
+- **Energy measured**: 160.6 J
+- **Carbon emissions**: 0.0119 gCO2
+- **Duration**: 11.28 seconds
 
 ### Pattern Database
 
@@ -570,77 +663,6 @@ Alpine containers, Infrastructure as Code, renewable energy regions, container o
 ### 15. General (8 patterns)
 Feature flags, incremental processing, precomputation, background jobs, workflow optimization
 
-## CLI Commands
-
-| Command | Description | Key Options |
-|---------|-------------|-------------|
-| `fetch` | Fetch repositories from GitHub with custom keywords | `--max-repos`, `--min-stars`, `--languages`, `--keywords` |
-| `extract` | Extract commit history from repositories | `--max-commits` per repository |
-| `analyze` | Analyze commits for green patterns | `--enable-nlp`, `--enable-ml-features`, `--enable-diff-analysis` |
-| `aggregate` | Aggregate analysis results | `--enable-temporal`, `--temporal-granularity`, `--enable-enhanced-stats` |
-| `report` | Generate comprehensive report | Creates Markdown and CSV outputs |
-| `pipeline` | Run complete analysis pipeline | `--max-repos`, `--max-commits` (all-in-one) |
-| `status` | Show current analysis status | Displays progress and file statistics |
-
-### Command Details
-
-#### Fetch Repositories
-```bash
-# Fetch with custom search keywords
-greenmining fetch --max-repos 100 --min-stars 50 --languages Python --keywords "kubernetes docker"
-
-# Fetch microservices (default)
-greenmining fetch --max-repos 100 --min-stars 50 --languages Python
-```
-Options:
-- `--max-repos`: Maximum repositories to fetch (default: 100)
-- `--min-stars`: Minimum GitHub stars (default: 100)
-- `--languages`: Filter by programming languages (default: "Python,Java,Go,JavaScript,TypeScript")
-- `--keywords`: Custom search keywords (default: "microservices")
-
-#### Extract Commits
-```bash
-greenmining extract --max-commits 50
-```
-Options:
-- `--max-commits`: Maximum commits per repository (default: 50)
-
-#### Analyze Commits (with Advanced Features)
-```bash
-# Basic analysis
-greenmining analyze
-
-# Advanced analysis with all features
-greenmining analyze --enable-nlp --enable-ml-features --enable-diff-analysis --batch-size 20
-```
-Options:
-- `--batch-size`: Batch size for processing (default: 10)
-- `--enable-diff-analysis`: Enable code diff analysis (slower but more accurate)
-- `--enable-nlp`: Enable NLP-enhanced pattern detection with morphological variants and synonyms
-- `--enable-ml-features`: Enable ML feature extraction for model training
-
-#### Aggregate Results (with Temporal Analysis)
-```bash
-# Basic aggregation
-greenmining aggregate
-
-# Advanced aggregation with temporal trends
-greenmining aggregate --enable-temporal --temporal-granularity quarter --enable-enhanced-stats
-```
-Options:
-- `--enable-enhanced-stats`: Enable enhanced statistical analysis (correlations, effect sizes)
-- `--enable-temporal`: Enable temporal trend analysis
-- `--temporal-granularity`: Time period granularity (choices: day, week, month, quarter, year)
-
-#### Run Pipeline
-```bash
-greenmining pipeline --max-repos 50 --max-commits 100
-```
-Options:
-- `--max-repos`: Repositories to analyze
-- `--max-commits`: Commits per repository
-- Executes: fetch → extract → analyze → aggregate → report
-
 ## Output Files
 
 All outputs are saved to the `data/` directory:
@@ -680,6 +702,7 @@ ruff check greenmining/ tests/
 - PyDriller >= 2.5
 - pandas >= 2.2.0
 - click >= 8.1.7
+- codecarbon >= 2.0.0 (optional, for cross-platform energy measurement)
 
 ## License
 

greenmining-1.0.4.dist-info/RECORD

@@ -0,0 +1,37 @@
+greenmining/__init__.py,sha256=m1foz4CUfEgBsgyWHEjehsT3qt1vogyer1fjhIVFqjg,992
+greenmining/__main__.py,sha256=NYOVS7D4w2XDLn6SyXHXPKE5GrNGOeoWSTb_KazgK5c,590
+greenmining/__version__.py,sha256=QpL6VzwpRtOZiXqutOetZBUCoiJKYFeTosucueZ4gpQ,62
+greenmining/config.py,sha256=00v1Ln8eZE0RxMrLxvonf8XOWqeRYaIme_iC1yDLR90,8228
+greenmining/gsf_patterns.py,sha256=hnd9GuWB8GEflrusEib5hjvl8CD5TSbGcBtb0gfxFp4,54193
+greenmining/utils.py,sha256=dSFwQzQwbS8rYZSgwLIxM_geLqbldwqVOIXMqEg08Qs,5609
+greenmining/analyzers/__init__.py,sha256=VI22zb3TnhZrwEuBy0J3pIbqNVVZl1wx8NYRlhR6Wok,362
+greenmining/analyzers/code_diff_analyzer.py,sha256=1dk68R3O0RZG8gx1cm9B_UlZ1Uwyb_Q3oScRbCVx4tM,10950
+greenmining/analyzers/qualitative_analyzer.py,sha256=RcjOMLj_DPH869ey9J0uI7JK_krCefMhNkPLOJUDFF8,15391
+greenmining/analyzers/statistical_analyzer.py,sha256=DzWAcCyw42Ig3FIxTwPPBikgt2uzMdktxklonOYfnOk,7166
+greenmining/analyzers/temporal_analyzer.py,sha256=JfTcAoI20oCFMehGrSRnDqhJTXI-RUbdCTMwDOTW9-g,14259
+greenmining/controllers/__init__.py,sha256=UiAT6zBvC1z_9cJWfzq1cLA0I4r9b2vURHipj8oDczI,180
+greenmining/controllers/repository_controller.py,sha256=fyL6Y8xpoixDplP4_rKWiwak42M2DaIihzyKVaBlivA,9680
+greenmining/energy/__init__.py,sha256=Y9RkNuZ3T6npEBxOZJhVc8wy6feXQePdXojLaZxkfGM,308
+greenmining/energy/base.py,sha256=s0yyRDhnEkrkCE5cgp2yHOrIhKbCpU9V7n4Rf1ejWLM,5559
+greenmining/energy/codecarbon_meter.py,sha256=z4T63qunEsU2R2qylZdGCtk3e-y_HYaBjBMD3nuFRU0,5102
+greenmining/energy/rapl.py,sha256=nZoVmdZshSsFLEYWNQxYyNg8fhhzgNME02bpQuIQL_U,5584
+greenmining/models/__init__.py,sha256=2hkB0quhMePvvA1AkYfj5uiF_HyGtXVxn0BU-5m_oSg,302
+greenmining/models/aggregated_stats.py,sha256=N-ZGcQO7IJ33Joa8luMVjtHhKYzNe48VW8hFqs9a5Jc,1016
+greenmining/models/analysis_result.py,sha256=YICTCEcrJxZ1R8Xaio3AZOjCGwMzC_62BMAL0J_XY1w,1509
+greenmining/models/commit.py,sha256=mnRDWSiIyGtJeGXI8sav9hukWUyVFpoNe6GixRlZjY4,2439
+greenmining/models/repository.py,sha256=SKjS01onOptpMioumtAPZxKpKheHAeVXnXyvatl7CfM,2856
+greenmining/presenters/__init__.py,sha256=d1CMtqtUAHYHYNzigPyjtGOUtnH1drtUwf7-bFQq2B8,138
+greenmining/presenters/console_presenter.py,sha256=XOahvlcr4qLbUdhk8cGq1ZWagvemEd3Wgriu8T5EI3s,4896
+greenmining/services/__init__.py,sha256=UhjS2X9x2v5iH991UDPazP3dTPuSgylMq4kQJaueQYs,481
+greenmining/services/commit_extractor.py,sha256=3EfUVBwd8hGSbl7pS-_jAL8gX8RxIASXTX5EZBbKQPI,8387
+greenmining/services/data_aggregator.py,sha256=TsFT0oGOnnHk0QGZ1tT6ZhKGc5X1H1D1u7-7OpiPo7Y,19566
+greenmining/services/data_analyzer.py,sha256=f0nlJkPAclHHCzzTyQW5bjhYrgE0XXiR1x7_o3fJaDs,9732
+greenmining/services/github_fetcher.py,sha256=WFyowC0tROKAhP3bA4QXH7QIYf42yaZf2ePpUEvOZ6s,8266
+greenmining/services/github_graphql_fetcher.py,sha256=mQOUZZEltYWusA7TR6Q7BZrHZ-N1Ijt0Wg2kV5T7lFM,11958
+greenmining/services/local_repo_analyzer.py,sha256=IrfqY1L6peGO78zufEj4uAU1N7nskc0edAYVzE0Ew_w,14785
+greenmining/services/reports.py,sha256=7Smc7a4KtpmkAJ8UoMlzH5BZerC_iO_jMyQw3_42n1s,23387
+greenmining-1.0.4.dist-info/licenses/LICENSE,sha256=M7ma3JHGeiIZIs3ea0HTcFl_wLFPX2NZElUliYs4bCA,1083
+greenmining-1.0.4.dist-info/METADATA,sha256=rP_4u0fkgp_Bmq2txIpBTLa7ZMj3By2ylTjoej7RSEw,25610
+greenmining-1.0.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+greenmining-1.0.4.dist-info/top_level.txt,sha256=nreXgXxZIWI-42yQknQ0HXtUrFnzZ8N1ra4Mdy2KcsI,12
+greenmining-1.0.4.dist-info/RECORD,,

{greenmining-1.0.2.dist-info → greenmining-1.0.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any