green-peft 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- green_peft-0.1.0/LICENSE +21 -0
- green_peft-0.1.0/PKG-INFO +109 -0
- green_peft-0.1.0/README.md +94 -0
- green_peft-0.1.0/green_peft/__init__.py +12 -0
- green_peft-0.1.0/green_peft/cli.py +147 -0
- green_peft-0.1.0/green_peft/recommender.py +343 -0
- green_peft-0.1.0/green_peft.egg-info/PKG-INFO +109 -0
- green_peft-0.1.0/green_peft.egg-info/SOURCES.txt +12 -0
- green_peft-0.1.0/green_peft.egg-info/dependency_links.txt +1 -0
- green_peft-0.1.0/green_peft.egg-info/entry_points.txt +2 -0
- green_peft-0.1.0/green_peft.egg-info/requires.txt +5 -0
- green_peft-0.1.0/green_peft.egg-info/top_level.txt +1 -0
- green_peft-0.1.0/pyproject.toml +25 -0
- green_peft-0.1.0/setup.cfg +4 -0
green_peft-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashraful Islam Tanzil
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: green-peft
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Constraint-aware, zero-shot PEFT strategy recommender using the GreenPEFT surrogate models.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: scikit-learn<1.9,>=1.8
|
|
10
|
+
Requires-Dist: pandas<2.3,>=1.5
|
|
11
|
+
Requires-Dist: numpy<2.1,>=1.23
|
|
12
|
+
Requires-Dist: joblib>=1.2
|
|
13
|
+
Requires-Dist: pyyaml>=6.0
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# green-peft
|
|
17
|
+
|
|
18
|
+
Constraint-aware PEFT strategy recommender. Wraps the surrogate models trained in
|
|
19
|
+
Cell 13 of the GreenPEFT notebook so you can ask "which method + model size should I
|
|
20
|
+
use?" from the command line, without running anything.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Export artifacts from your notebook run
|
|
29
|
+
|
|
30
|
+
The CLI needs two things out of your Kaggle/Colab `peft_bench/` folder:
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
peft_bench/results/surrogate_models.joblib # written by Cell 13
|
|
34
|
+
peft_bench/configs/backbones.yaml # written by Cell 3 (has model_zoo:)
|
|
35
|
+
peft_bench/configs/methods/*.yaml # written by Cell 3
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Zip `results/` and `configs/` together, download from Kaggle, unzip locally into
|
|
39
|
+
e.g. `./peft_bench_export/`, and point `--artifacts-dir` at it.
|
|
40
|
+
|
|
41
|
+
This repository includes a ready-to-use export at:
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
../../model/artifacts_export/
|
|
45
|
+
├── configs/backbones.yaml
|
|
46
|
+
├── configs/methods/*.yaml
|
|
47
|
+
├── configs/tasks.yaml
|
|
48
|
+
├── results/surrogate_models.joblib
|
|
49
|
+
├── results/surrogate_cv_metrics.json
|
|
50
|
+
└── results/surrogate_dataset.csv
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
From the repository root, install the package and use that export directly:
|
|
54
|
+
|
|
55
|
+
```powershell
|
|
56
|
+
cd green_peft_cli/green_peft_pkg
|
|
57
|
+
python -m pip install -e .
|
|
58
|
+
green-peft list-zoo --artifacts-dir ../../model/artifacts_export
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Usage
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# Basic recommendation under a VRAM + accuracy constraint
|
|
65
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
66
|
+
--vram 16 --accuracy 0.90 --profile balanced
|
|
67
|
+
|
|
68
|
+
# Strict carbon budget, custom weight profile, JSON output for scripting
|
|
69
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
70
|
+
--vram 24 --carbon 0.003 --weights 0.3,0.2,0.4,0.1 --top-k 5 --json
|
|
71
|
+
|
|
72
|
+
# See every backbone x method combination the engine can currently score
|
|
73
|
+
green-peft list-zoo --artifacts-dir ./peft_bench_export
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## What it actually does
|
|
77
|
+
|
|
78
|
+
1. Builds a candidate table crossing every backbone in your `model_zoo` (including
|
|
79
|
+
ones you never ran) with every configured method.
|
|
80
|
+
2. Runs the four surrogate models (feasibility classifier + accuracy / peak-VRAM /
|
|
81
|
+
energy / wall-clock regressors) over every candidate.
|
|
82
|
+
3. Drops candidates the feasibility classifier doesn't trust, or that violate your
|
|
83
|
+
`--vram` / `--carbon` / `--accuracy` / `--time` constraints.
|
|
84
|
+
4. Computes the Pareto front and a GEI score (same formula as the paper: weighted sum
|
|
85
|
+
of normalized accuracy/memory/carbon/time scores) over the surviving candidates.
|
|
86
|
+
5. Prints the top-k ranked options with a plain-English explanation, or an explicit
|
|
87
|
+
breakdown of why nothing survived if your constraints are infeasible together.
|
|
88
|
+
|
|
89
|
+
Predictions are only as reliable as the surrogate's leave-one-tier-out validation
|
|
90
|
+
(`results/surrogate_cv_metrics.json` from Cell 13) says they are for that target --
|
|
91
|
+
check `regression.<target>.best_model` R2 there before trusting a specific number.
|
|
92
|
+
|
|
93
|
+
## Current validation and limitations
|
|
94
|
+
|
|
95
|
+
The included export reports leave-one-tier-out feasibility accuracy of `0.7833`.
|
|
96
|
+
Selected regression models report the following validation results:
|
|
97
|
+
|
|
98
|
+
| Target | Model | MAE | R2 |
|
|
99
|
+
| --- | --- | ---: | ---: |
|
|
100
|
+
| Accuracy | Gradient boosting | 0.0182 | -0.4573 |
|
|
101
|
+
| Peak VRAM | Gradient boosting | 3.9703 GB | -0.5154 |
|
|
102
|
+
| Energy | Ridge | 0.000563 kWh | 0.3740 |
|
|
103
|
+
| Wall-clock time | Gradient boosting | 33.12 s | 0.4077 |
|
|
104
|
+
|
|
105
|
+
Use the CLI for experiment planning, shortlist generation, and explicit budget checks.
|
|
106
|
+
It does not run fine-tuning, measure the target hardware, or guarantee performance.
|
|
107
|
+
The benchmark evidence is primarily SST-2 classification on a Tesla T4; recommendations
|
|
108
|
+
for unbenchmarked catalog entries are extrapolations. Validate the selected option with
|
|
109
|
+
a real run and retain the JSON output alongside the measured results.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# green-peft
|
|
2
|
+
|
|
3
|
+
Constraint-aware PEFT strategy recommender. Wraps the surrogate models trained in
|
|
4
|
+
Cell 13 of the GreenPEFT notebook so you can ask "which method + model size should I
|
|
5
|
+
use?" from the command line, without running anything.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install -e .
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Export artifacts from your notebook run
|
|
14
|
+
|
|
15
|
+
The CLI needs two things out of your Kaggle/Colab `peft_bench/` folder:
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
peft_bench/results/surrogate_models.joblib # written by Cell 13
|
|
19
|
+
peft_bench/configs/backbones.yaml # written by Cell 3 (has model_zoo:)
|
|
20
|
+
peft_bench/configs/methods/*.yaml # written by Cell 3
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Zip `results/` and `configs/` together, download from Kaggle, unzip locally into
|
|
24
|
+
e.g. `./peft_bench_export/`, and point `--artifacts-dir` at it.
|
|
25
|
+
|
|
26
|
+
This repository includes a ready-to-use export at:
|
|
27
|
+
|
|
28
|
+
```text
|
|
29
|
+
../../model/artifacts_export/
|
|
30
|
+
├── configs/backbones.yaml
|
|
31
|
+
├── configs/methods/*.yaml
|
|
32
|
+
├── configs/tasks.yaml
|
|
33
|
+
├── results/surrogate_models.joblib
|
|
34
|
+
├── results/surrogate_cv_metrics.json
|
|
35
|
+
└── results/surrogate_dataset.csv
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
From the repository root, install the package and use that export directly:
|
|
39
|
+
|
|
40
|
+
```powershell
|
|
41
|
+
cd green_peft_cli/green_peft_pkg
|
|
42
|
+
python -m pip install -e .
|
|
43
|
+
green-peft list-zoo --artifacts-dir ../../model/artifacts_export
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
# Basic recommendation under a VRAM + accuracy constraint
|
|
50
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
51
|
+
--vram 16 --accuracy 0.90 --profile balanced
|
|
52
|
+
|
|
53
|
+
# Strict carbon budget, custom weight profile, JSON output for scripting
|
|
54
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
55
|
+
--vram 24 --carbon 0.003 --weights 0.3,0.2,0.4,0.1 --top-k 5 --json
|
|
56
|
+
|
|
57
|
+
# See every backbone x method combination the engine can currently score
|
|
58
|
+
green-peft list-zoo --artifacts-dir ./peft_bench_export
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## What it actually does
|
|
62
|
+
|
|
63
|
+
1. Builds a candidate table crossing every backbone in your `model_zoo` (including
|
|
64
|
+
ones you never ran) with every configured method.
|
|
65
|
+
2. Runs the four surrogate models (feasibility classifier + accuracy / peak-VRAM /
|
|
66
|
+
energy / wall-clock regressors) over every candidate.
|
|
67
|
+
3. Drops candidates the feasibility classifier doesn't trust, or that violate your
|
|
68
|
+
`--vram` / `--carbon` / `--accuracy` / `--time` constraints.
|
|
69
|
+
4. Computes the Pareto front and a GEI score (same formula as the paper: weighted sum
|
|
70
|
+
of normalized accuracy/memory/carbon/time scores) over the surviving candidates.
|
|
71
|
+
5. Prints the top-k ranked options with a plain-English explanation, or an explicit
|
|
72
|
+
breakdown of why nothing survived if your constraints are infeasible together.
|
|
73
|
+
|
|
74
|
+
Predictions are only as reliable as the surrogate's leave-one-tier-out validation
|
|
75
|
+
(`results/surrogate_cv_metrics.json` from Cell 13) says they are for that target --
|
|
76
|
+
check `regression.<target>.best_model` R2 there before trusting a specific number.
|
|
77
|
+
|
|
78
|
+
## Current validation and limitations
|
|
79
|
+
|
|
80
|
+
The included export reports leave-one-tier-out feasibility accuracy of `0.7833`.
|
|
81
|
+
Selected regression models report the following validation results:
|
|
82
|
+
|
|
83
|
+
| Target | Model | MAE | R2 |
|
|
84
|
+
| --- | --- | ---: | ---: |
|
|
85
|
+
| Accuracy | Gradient boosting | 0.0182 | -0.4573 |
|
|
86
|
+
| Peak VRAM | Gradient boosting | 3.9703 GB | -0.5154 |
|
|
87
|
+
| Energy | Ridge | 0.000563 kWh | 0.3740 |
|
|
88
|
+
| Wall-clock time | Gradient boosting | 33.12 s | 0.4077 |
|
|
89
|
+
|
|
90
|
+
Use the CLI for experiment planning, shortlist generation, and explicit budget checks.
|
|
91
|
+
It does not run fine-tuning, measure the target hardware, or guarantee performance.
|
|
92
|
+
The benchmark evidence is primarily SST-2 classification on a Tesla T4; recommendations
|
|
93
|
+
for unbenchmarked catalog entries are extrapolations. Validate the selected option with
|
|
94
|
+
a real run and retain the JSON output alongside the measured results.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from .recommender import (
|
|
2
|
+
GreenPEFTArtifacts, Constraints, GEI_PROFILES,
|
|
3
|
+
build_candidates, predict_all, apply_constraints, pareto_front, score_gei,
|
|
4
|
+
recommend, explain,
|
|
5
|
+
)
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
'GreenPEFTArtifacts', 'Constraints', 'GEI_PROFILES',
|
|
9
|
+
'build_candidates', 'predict_all', 'apply_constraints', 'pareto_front', 'score_gei',
|
|
10
|
+
'recommend', 'explain',
|
|
11
|
+
]
|
|
12
|
+
__version__ = '0.1.0'
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
green-peft: command-line recommender built on the GreenPEFT surrogate models.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
green-peft recommend --artifacts-dir ./peft_bench_export --vram 16 \\
|
|
6
|
+
--carbon 0.005 --accuracy 0.90 --profile balanced
|
|
7
|
+
|
|
8
|
+
green-peft recommend --artifacts-dir ./peft_bench_export --vram 16 \\
|
|
9
|
+
--weights 0.5,0.2,0.2,0.1 --top-k 5 --json
|
|
10
|
+
|
|
11
|
+
green-peft list-zoo --artifacts-dir ./peft_bench_export
|
|
12
|
+
|
|
13
|
+
The artifacts directory is whatever you exported from the notebook -- it must contain
|
|
14
|
+
results/surrogate_models.joblib (from Cell 13) and configs/backbones.yaml +
|
|
15
|
+
configs/methods/*.yaml (from Cell 3). Zip peft_bench/results and peft_bench/configs
|
|
16
|
+
together and unzip them here; nothing else from the run is needed.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
import argparse
|
|
21
|
+
import json
|
|
22
|
+
import sys
|
|
23
|
+
|
|
24
|
+
import pandas as pd
|
|
25
|
+
|
|
26
|
+
from .recommender import (
|
|
27
|
+
GreenPEFTArtifacts, Constraints, GEI_PROFILES, recommend, explain, build_candidates,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _parse_weights(s: str) -> tuple:
|
|
32
|
+
parts = [float(x) for x in s.split(',')]
|
|
33
|
+
if len(parts) != 4:
|
|
34
|
+
raise argparse.ArgumentTypeError(
|
|
35
|
+
'--weights needs exactly 4 comma-separated numbers: accuracy,memory,carbon,time')
|
|
36
|
+
return tuple(parts)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def cmd_recommend(args):
|
|
40
|
+
try:
|
|
41
|
+
artifacts = GreenPEFTArtifacts.load(args.artifacts_dir)
|
|
42
|
+
except FileNotFoundError as e:
|
|
43
|
+
print(f'error: {e}', file=sys.stderr)
|
|
44
|
+
return 2
|
|
45
|
+
|
|
46
|
+
constraints = Constraints(
|
|
47
|
+
max_vram_gb=args.vram,
|
|
48
|
+
max_carbon_kgco2eq=args.carbon,
|
|
49
|
+
min_accuracy=args.accuracy,
|
|
50
|
+
max_time_seconds=args.time,
|
|
51
|
+
)
|
|
52
|
+
weights = _parse_weights(args.weights) if args.weights else None
|
|
53
|
+
profile = args.profile if weights is None else None
|
|
54
|
+
|
|
55
|
+
result = recommend(
|
|
56
|
+
artifacts, constraints,
|
|
57
|
+
profile=profile or 'balanced', custom_weights=weights,
|
|
58
|
+
gpu_vram_gb=args.vram,
|
|
59
|
+
backbones=args.backbones.split(',') if args.backbones else None,
|
|
60
|
+
methods=args.methods.split(',') if args.methods else None,
|
|
61
|
+
top_k=args.top_k,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
if args.json:
|
|
65
|
+
payload = {
|
|
66
|
+
'n_candidates': result['n_candidates'], 'n_feasible': result['n_feasible'],
|
|
67
|
+
'profile': result['profile'], 'weights': result['weights'],
|
|
68
|
+
}
|
|
69
|
+
if result['ranked'] is not None:
|
|
70
|
+
payload['ranked'] = json.loads(result['ranked'].to_json(orient='records'))
|
|
71
|
+
payload['pareto_only'] = json.loads(result['pareto_only'].to_json(orient='records'))
|
|
72
|
+
else:
|
|
73
|
+
payload['dropped_summary'] = result['dropped_summary']
|
|
74
|
+
print(json.dumps(payload, indent=2))
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
print(explain(result))
|
|
78
|
+
if result['ranked'] is not None and args.top_k > 1:
|
|
79
|
+
print(f"\nTop {min(args.top_k, len(result['ranked']))} by GEI:")
|
|
80
|
+
cols = ['backbone', 'method', 'pred_accuracy', 'pred_peak_vram_gb',
|
|
81
|
+
'pred_carbon_kgco2eq', 'gei', 'on_pareto_front']
|
|
82
|
+
with pd.option_context('display.width', 120, 'display.float_format', '{:.4f}'.format):
|
|
83
|
+
print(result['ranked'][cols].to_string(index=False))
|
|
84
|
+
return 0
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_list_zoo(args):
|
|
88
|
+
try:
|
|
89
|
+
artifacts = GreenPEFTArtifacts.load(args.artifacts_dir)
|
|
90
|
+
except FileNotFoundError as e:
|
|
91
|
+
print(f'error: {e}', file=sys.stderr)
|
|
92
|
+
return 2
|
|
93
|
+
candidates = build_candidates(artifacts,
|
|
94
|
+
args.backbones.split(',') if args.backbones else None,
|
|
95
|
+
args.methods.split(',') if args.methods else None)
|
|
96
|
+
with pd.option_context('display.width', 120):
|
|
97
|
+
print(candidates[['backbone', 'model_id', 'family', 'params_b', 'method']]
|
|
98
|
+
.drop_duplicates(subset=['backbone', 'method']).to_string(index=False))
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def build_parser():
|
|
103
|
+
p = argparse.ArgumentParser(prog='green-peft',
|
|
104
|
+
description='Constraint-aware PEFT strategy recommender.')
|
|
105
|
+
sub = p.add_subparsers(dest='command', required=True)
|
|
106
|
+
|
|
107
|
+
r = sub.add_parser('recommend', help='Recommend a PEFT config under given constraints.')
|
|
108
|
+
r.add_argument('--artifacts-dir', required=True,
|
|
109
|
+
help='Directory with results/surrogate_models.joblib and configs/.')
|
|
110
|
+
r.add_argument('--vram', type=float, default=None, help='Max VRAM budget in GB.')
|
|
111
|
+
r.add_argument('--carbon', type=float, default=None, help='Max carbon budget in kgCO2eq.')
|
|
112
|
+
r.add_argument('--accuracy', type=float, default=None, help='Minimum required accuracy.')
|
|
113
|
+
r.add_argument('--time', type=float, default=None, help='Max wall-clock budget in seconds.')
|
|
114
|
+
r.add_argument('--profile', choices=list(GEI_PROFILES), default='balanced',
|
|
115
|
+
help='Named GEI weight profile (ignored if --weights is given).')
|
|
116
|
+
r.add_argument('--weights', type=str, default=None,
|
|
117
|
+
help='Custom GEI weights "w_acc,w_mem,w_carbon,w_time", must sum to 1.0.')
|
|
118
|
+
r.add_argument('--backbones', type=str, default=None,
|
|
119
|
+
help='Comma-separated backbone keys to consider (default: whole model_zoo).')
|
|
120
|
+
r.add_argument('--methods', type=str, default=None,
|
|
121
|
+
help='Comma-separated method keys to consider (default: all configured methods).')
|
|
122
|
+
r.add_argument('--top-k', type=int, default=3, help='How many ranked candidates to show.')
|
|
123
|
+
r.add_argument('--json', action='store_true', help='Emit machine-readable JSON instead of text.')
|
|
124
|
+
r.set_defaults(func=cmd_recommend)
|
|
125
|
+
|
|
126
|
+
z = sub.add_parser('list-zoo', help='List every backbone x method candidate the engine can score.')
|
|
127
|
+
z.add_argument('--artifacts-dir', required=True)
|
|
128
|
+
z.add_argument('--backbones', type=str, default=None)
|
|
129
|
+
z.add_argument('--methods', type=str, default=None)
|
|
130
|
+
z.set_defaults(func=cmd_list_zoo)
|
|
131
|
+
|
|
132
|
+
return p
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def main(argv=None):
|
|
136
|
+
parser = build_parser()
|
|
137
|
+
args = parser.parse_args(argv)
|
|
138
|
+
try:
|
|
139
|
+
return args.func(args)
|
|
140
|
+
except BrokenPipeError:
|
|
141
|
+
# e.g. `green-peft list-zoo ... | head` -- the reader closed early, not an error.
|
|
142
|
+
sys.stderr.close()
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == '__main__':
|
|
147
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"""
|
|
2
|
+
GreenPEFT decision engine.
|
|
3
|
+
|
|
4
|
+
Loads the surrogate models trained in Cell 13 (surrogate_models.joblib) and the
|
|
5
|
+
model/method catalog written by Cell 3 (configs/backbones.yaml, configs/methods/*.yaml),
|
|
6
|
+
then answers: "given VRAM / carbon / accuracy / time constraints, which PEFT
|
|
7
|
+
configuration should I run?" -- WITHOUT running it. This is RQ3 + RQ4 together:
|
|
8
|
+
zero-shot prediction (surrogate) feeding constraint-aware prescription (this module).
|
|
9
|
+
|
|
10
|
+
Design notes:
|
|
11
|
+
- Candidates are drawn from the FULL model_zoo in backbones.yaml, not just the tiers
|
|
12
|
+
that were actually benchmarked. Predicting for an untested backbone is the entire
|
|
13
|
+
point of having a surrogate instead of just looking up aggregated_gei.csv.
|
|
14
|
+
- GEI normalization (S_Acc, S_Mem, S_C, S_T) follows the formula in the README exactly:
|
|
15
|
+
higher-is-better for accuracy, lower-is-better for memory/carbon/time. Normalization
|
|
16
|
+
bounds are taken from the FEASIBLE, CONSTRAINT-FILTERED candidate set -- not the whole
|
|
17
|
+
catalog -- because GEI is meant to score trade-offs among options the user could
|
|
18
|
+
actually choose, not options that were already ruled out.
|
|
19
|
+
- A candidate whose predicted VRAM/accuracy/carbon/time we don't trust (feasibility
|
|
20
|
+
classifier says P(fits) < FEASIBILITY_THRESHOLD) is dropped before scoring, not
|
|
21
|
+
penalized within GEI -- an infeasible config isn't a worse option, it isn't an option.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
import json
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Optional
|
|
29
|
+
|
|
30
|
+
import joblib
|
|
31
|
+
import numpy as np
|
|
32
|
+
import pandas as pd
|
|
33
|
+
import yaml
|
|
34
|
+
|
|
35
|
+
CAT_FEATURES = ['method', 'family']
|
|
36
|
+
NUM_FEATURES = ['params_b', 'rank', 'quant_bits', 'is_adapter_method']
|
|
37
|
+
FEASIBILITY_THRESHOLD = 0.5
|
|
38
|
+
|
|
39
|
+
# Grid/cost assumptions -- mirror Cell 2 defaults. Override via GreenPEFTArtifacts if
|
|
40
|
+
# your notebook used different constants, so predicted carbon/cost stay consistent
|
|
41
|
+
# with what was actually measured.
|
|
42
|
+
DEFAULT_GRID_CARBON_KG_PER_KWH = 0.650
|
|
43
|
+
DEFAULT_ELECTRICITY_USD_PER_KWH = 0.12
|
|
44
|
+
DEFAULT_GPU_RENTAL_USD_PER_HOUR = 0.35
|
|
45
|
+
|
|
46
|
+
GEI_PROFILES = {
|
|
47
|
+
# (w_accuracy, w_memory, w_carbon, w_time) -- must sum to 1.0
|
|
48
|
+
'balanced': (0.35, 0.25, 0.25, 0.15),
|
|
49
|
+
'strict_carbon': (0.20, 0.20, 0.50, 0.10),
|
|
50
|
+
'high_accuracy': (0.60, 0.15, 0.15, 0.10),
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _method_rank(method_cfg: dict) -> int:
|
|
55
|
+
return int(method_cfg.get('rank', 0))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _method_quant_bits(method_name: str) -> int:
|
|
59
|
+
if method_name == 'qlora':
|
|
60
|
+
return 4
|
|
61
|
+
if method_name == 'full_ft':
|
|
62
|
+
return 32
|
|
63
|
+
return 16
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _method_is_adapter(method_name: str) -> int:
|
|
67
|
+
return int(method_name in ('lora', 'qlora', 'lora_fa'))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class GreenPEFTArtifacts:
|
|
72
|
+
"""Everything the engine needs, loaded from one export directory.
|
|
73
|
+
|
|
74
|
+
Expected layout (this is exactly what Cells 1-3 and 13 already write):
|
|
75
|
+
artifacts_dir/
|
|
76
|
+
results/surrogate_models.joblib
|
|
77
|
+
configs/backbones.yaml (has a top-level `model_zoo:` key)
|
|
78
|
+
configs/methods/*.yaml
|
|
79
|
+
"""
|
|
80
|
+
model_zoo: dict
|
|
81
|
+
method_configs: dict
|
|
82
|
+
models: dict # {'fits':..., 'accuracy':..., 'peak_gpu_memory_gb':..., 'energy_kwh':..., ['wall_clock_seconds':...]}
|
|
83
|
+
grid_carbon_kg_per_kwh: float = DEFAULT_GRID_CARBON_KG_PER_KWH
|
|
84
|
+
electricity_usd_per_kwh: float = DEFAULT_ELECTRICITY_USD_PER_KWH
|
|
85
|
+
gpu_rental_usd_per_hour: float = DEFAULT_GPU_RENTAL_USD_PER_HOUR
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def load(cls, artifacts_dir: str | Path) -> 'GreenPEFTArtifacts':
|
|
89
|
+
root = Path(artifacts_dir)
|
|
90
|
+
joblib_path = root / 'results' / 'surrogate_models.joblib'
|
|
91
|
+
backbones_path = root / 'configs' / 'backbones.yaml'
|
|
92
|
+
methods_dir = root / 'configs' / 'methods'
|
|
93
|
+
|
|
94
|
+
if not joblib_path.exists():
|
|
95
|
+
raise FileNotFoundError(
|
|
96
|
+
f'{joblib_path} not found. Run Cell 13 in the notebook and download/export '
|
|
97
|
+
f'the peft_bench/ folder (or just results/ + configs/) to this location.')
|
|
98
|
+
if not backbones_path.exists():
|
|
99
|
+
raise FileNotFoundError(f'{backbones_path} not found (from Cell 3).')
|
|
100
|
+
|
|
101
|
+
models = joblib.load(joblib_path)
|
|
102
|
+
backbones_yaml = yaml.safe_load(open(backbones_path))
|
|
103
|
+
model_zoo = backbones_yaml.get('model_zoo') or backbones_yaml.get('backbones')
|
|
104
|
+
|
|
105
|
+
method_configs = {}
|
|
106
|
+
if methods_dir.exists():
|
|
107
|
+
for p in methods_dir.glob('*.yaml'):
|
|
108
|
+
cfg = yaml.safe_load(open(p))
|
|
109
|
+
method_configs[cfg['method']] = cfg
|
|
110
|
+
else:
|
|
111
|
+
raise FileNotFoundError(f'{methods_dir} not found (from Cell 3).')
|
|
112
|
+
|
|
113
|
+
return cls(model_zoo=model_zoo, method_configs=method_configs, models=models)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class Constraints:
|
|
118
|
+
max_vram_gb: Optional[float] = None
|
|
119
|
+
max_carbon_kgco2eq: Optional[float] = None
|
|
120
|
+
min_accuracy: Optional[float] = None
|
|
121
|
+
max_time_seconds: Optional[float] = None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def build_candidates(artifacts: GreenPEFTArtifacts,
|
|
125
|
+
backbones: Optional[list[str]] = None,
|
|
126
|
+
methods: Optional[list[str]] = None) -> pd.DataFrame:
|
|
127
|
+
"""Cross-product of model_zoo x methods with surrogate feature columns filled in."""
|
|
128
|
+
backbones = backbones or list(artifacts.model_zoo.keys())
|
|
129
|
+
methods = methods or list(artifacts.method_configs.keys())
|
|
130
|
+
|
|
131
|
+
rows = []
|
|
132
|
+
for bname in backbones:
|
|
133
|
+
bb = artifacts.model_zoo[bname]
|
|
134
|
+
for mname in methods:
|
|
135
|
+
mcfg = artifacts.method_configs.get(mname, {'method': mname})
|
|
136
|
+
rows.append({
|
|
137
|
+
'backbone': bname, 'model_id': bb.get('model_id', bname),
|
|
138
|
+
'method': mname, 'family': bb.get('family', 'unknown'),
|
|
139
|
+
'params_b': float(bb['params_b']),
|
|
140
|
+
'rank': _method_rank(mcfg),
|
|
141
|
+
'quant_bits': _method_quant_bits(mname),
|
|
142
|
+
'is_adapter_method': _method_is_adapter(mname),
|
|
143
|
+
})
|
|
144
|
+
return pd.DataFrame(rows)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def predict_all(artifacts: GreenPEFTArtifacts, candidates: pd.DataFrame) -> pd.DataFrame:
|
|
148
|
+
"""Run every surrogate model over the candidate table; derive carbon and cost."""
|
|
149
|
+
X = candidates[CAT_FEATURES + NUM_FEATURES]
|
|
150
|
+
out = candidates.copy()
|
|
151
|
+
|
|
152
|
+
fits_model = artifacts.models.get('fits')
|
|
153
|
+
if fits_model is not None:
|
|
154
|
+
proba = fits_model.predict_proba(X)
|
|
155
|
+
classes = list(fits_model.named_steps['model'].classes_) if hasattr(fits_model, 'named_steps') \
|
|
156
|
+
else list(fits_model.classes_)
|
|
157
|
+
fit_idx = classes.index(1) if 1 in classes else -1
|
|
158
|
+
out['p_fits'] = proba[:, fit_idx]
|
|
159
|
+
else:
|
|
160
|
+
out['p_fits'] = np.nan
|
|
161
|
+
|
|
162
|
+
for target, col in [('accuracy', 'pred_accuracy'),
|
|
163
|
+
('peak_gpu_memory_gb', 'pred_peak_vram_gb'),
|
|
164
|
+
('energy_kwh', 'pred_energy_kwh'),
|
|
165
|
+
('wall_clock_seconds', 'pred_wall_clock_s')]:
|
|
166
|
+
model = artifacts.models.get(target)
|
|
167
|
+
out[col] = model.predict(X) if model is not None else np.nan
|
|
168
|
+
|
|
169
|
+
out['pred_carbon_kgco2eq'] = out['pred_energy_kwh'] * artifacts.grid_carbon_kg_per_kwh
|
|
170
|
+
hours = out['pred_wall_clock_s'].fillna(0) / 3600.0
|
|
171
|
+
out['pred_cost_usd'] = (out['pred_energy_kwh'] * artifacts.electricity_usd_per_kwh
|
|
172
|
+
+ hours * artifacts.gpu_rental_usd_per_hour)
|
|
173
|
+
return out
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def apply_constraints(df: pd.DataFrame, constraints: Constraints,
|
|
177
|
+
gpu_vram_gb: Optional[float] = None) -> pd.DataFrame:
|
|
178
|
+
"""Constraint filtering (RQ3): drop candidates the surrogate doesn't trust or that
|
|
179
|
+
violate the user's stated budget. Every drop is explainable -- callers can diff
|
|
180
|
+
df vs the return value to see exactly which rows were cut and why, by re-checking
|
|
181
|
+
each condition."""
|
|
182
|
+
out = df[df['p_fits'] >= FEASIBILITY_THRESHOLD].copy()
|
|
183
|
+
vram_cap = constraints.max_vram_gb or gpu_vram_gb
|
|
184
|
+
if vram_cap is not None:
|
|
185
|
+
out = out[out['pred_peak_vram_gb'] <= vram_cap]
|
|
186
|
+
if constraints.max_carbon_kgco2eq is not None:
|
|
187
|
+
out = out[out['pred_carbon_kgco2eq'] <= constraints.max_carbon_kgco2eq]
|
|
188
|
+
if constraints.min_accuracy is not None:
|
|
189
|
+
out = out[out['pred_accuracy'] >= constraints.min_accuracy]
|
|
190
|
+
if constraints.max_time_seconds is not None:
|
|
191
|
+
out = out[out['pred_wall_clock_s'] <= constraints.max_time_seconds]
|
|
192
|
+
return out
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _time_available(df: pd.DataFrame) -> bool:
|
|
196
|
+
return 'pred_wall_clock_s' in df.columns and df['pred_wall_clock_s'].notna().any()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def pareto_front(df: pd.DataFrame) -> pd.Series:
|
|
200
|
+
"""Boolean mask: True where no other row is at-least-as-good on every ACTIVE axis
|
|
201
|
+
and strictly better on at least one (accuracy up; VRAM, carbon, [time] down).
|
|
202
|
+
If no wall-clock model was trained (pred_wall_clock_s is all-NaN), time is dropped
|
|
203
|
+
from the comparison entirely rather than left in as NaN -- NaN comparisons are
|
|
204
|
+
always False in numpy, which would silently make every candidate "non-dominated."""
|
|
205
|
+
if df.empty:
|
|
206
|
+
return pd.Series([], dtype=bool)
|
|
207
|
+
acc = df['pred_accuracy'].values
|
|
208
|
+
mem = df['pred_peak_vram_gb'].values
|
|
209
|
+
car = df['pred_carbon_kgco2eq'].values
|
|
210
|
+
use_time = _time_available(df)
|
|
211
|
+
tim = df['pred_wall_clock_s'].values if use_time else None
|
|
212
|
+
n = len(df)
|
|
213
|
+
dominated = np.zeros(n, dtype=bool)
|
|
214
|
+
for i in range(n):
|
|
215
|
+
for j in range(n):
|
|
216
|
+
if i == j:
|
|
217
|
+
continue
|
|
218
|
+
better_or_equal = (acc[j] >= acc[i]) and (mem[j] <= mem[i]) and (car[j] <= car[i])
|
|
219
|
+
strictly_better = (acc[j] > acc[i]) or (mem[j] < mem[i]) or (car[j] < car[i])
|
|
220
|
+
if use_time:
|
|
221
|
+
better_or_equal = better_or_equal and (tim[j] <= tim[i])
|
|
222
|
+
strictly_better = strictly_better or (tim[j] < tim[i])
|
|
223
|
+
if better_or_equal and strictly_better:
|
|
224
|
+
dominated[i] = True
|
|
225
|
+
break
|
|
226
|
+
return pd.Series(~dominated, index=df.index)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _normalize(series: pd.Series, higher_is_better: bool) -> pd.Series:
|
|
230
|
+
lo, hi = series.min(), series.max()
|
|
231
|
+
if hi - lo < 1e-12:
|
|
232
|
+
return pd.Series(1.0, index=series.index) # only one distinct value -> no trade-off to score
|
|
233
|
+
s = (series - lo) / (hi - lo)
|
|
234
|
+
return s if higher_is_better else (1.0 - s)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def score_gei(df: pd.DataFrame, weights: tuple[float, float, float, float]) -> tuple[pd.DataFrame, bool]:
|
|
238
|
+
"""Adds S_Acc, S_Mem, S_C, [S_T] and gei columns, normalized over THIS df only --
|
|
239
|
+
call this on the constraint-filtered candidate set, not the full catalog.
|
|
240
|
+
Returns (scored_df, time_was_used). If no wall-clock surrogate was trained, the
|
|
241
|
+
time term is dropped from the sum and the remaining three weights are renormalized
|
|
242
|
+
to sum to 1 -- silently defaulting a missing objective to 0 would understate GEI
|
|
243
|
+
for every candidate uniformly, and dropping the row entirely would discard otherwise
|
|
244
|
+
valid accuracy/memory/carbon predictions over a target this run never produced."""
|
|
245
|
+
w_acc, w_mem, w_c, w_t = weights
|
|
246
|
+
out = df.copy()
|
|
247
|
+
out['S_Acc'] = _normalize(out['pred_accuracy'], higher_is_better=True)
|
|
248
|
+
out['S_Mem'] = _normalize(out['pred_peak_vram_gb'], higher_is_better=False)
|
|
249
|
+
out['S_C'] = _normalize(out['pred_carbon_kgco2eq'], higher_is_better=False)
|
|
250
|
+
|
|
251
|
+
use_time = _time_available(out)
|
|
252
|
+
if use_time:
|
|
253
|
+
out['S_T'] = _normalize(out['pred_wall_clock_s'], higher_is_better=False)
|
|
254
|
+
out['gei'] = w_acc * out['S_Acc'] + w_mem * out['S_Mem'] + w_c * out['S_C'] + w_t * out['S_T']
|
|
255
|
+
else:
|
|
256
|
+
out['S_T'] = np.nan
|
|
257
|
+
renorm = w_acc + w_mem + w_c
|
|
258
|
+
out['gei'] = (w_acc * out['S_Acc'] + w_mem * out['S_Mem'] + w_c * out['S_C']) / renorm
|
|
259
|
+
return out, use_time
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def recommend(artifacts: GreenPEFTArtifacts, constraints: Constraints,
|
|
263
|
+
profile: str = 'balanced', custom_weights: Optional[tuple] = None,
|
|
264
|
+
gpu_vram_gb: Optional[float] = None,
|
|
265
|
+
backbones: Optional[list[str]] = None, methods: Optional[list[str]] = None,
|
|
266
|
+
top_k: int = 3) -> dict:
|
|
267
|
+
weights = custom_weights or GEI_PROFILES.get(profile)
|
|
268
|
+
if weights is None:
|
|
269
|
+
raise ValueError(f'Unknown profile "{profile}", choose from {list(GEI_PROFILES)} '
|
|
270
|
+
f'or pass custom_weights=(w_acc, w_mem, w_carbon, w_time)')
|
|
271
|
+
if abs(sum(weights) - 1.0) > 1e-6:
|
|
272
|
+
raise ValueError(f'weights must sum to 1.0, got {weights} (sum={sum(weights):.3f})')
|
|
273
|
+
|
|
274
|
+
candidates = build_candidates(artifacts, backbones, methods)
|
|
275
|
+
predicted = predict_all(artifacts, candidates)
|
|
276
|
+
filtered = apply_constraints(predicted, constraints, gpu_vram_gb=gpu_vram_gb)
|
|
277
|
+
|
|
278
|
+
result = {
|
|
279
|
+
'n_candidates': len(predicted), 'n_feasible': len(filtered),
|
|
280
|
+
'constraints': constraints, 'profile': profile, 'weights': weights,
|
|
281
|
+
'ranked': None, 'pareto_only': None, 'dropped_summary': None,
|
|
282
|
+
}
|
|
283
|
+
if filtered.empty:
|
|
284
|
+
# Explain WHY nothing survived rather than just returning empty -- this is the
|
|
285
|
+
# difference between a decision engine and a silent failure.
|
|
286
|
+
reasons = []
|
|
287
|
+
no_fit = predicted[predicted['p_fits'] < FEASIBILITY_THRESHOLD]
|
|
288
|
+
reasons.append(f"{len(no_fit)}/{len(predicted)} candidates predicted infeasible "
|
|
289
|
+
f"(p_fits < {FEASIBILITY_THRESHOLD})")
|
|
290
|
+
vram_cap = constraints.max_vram_gb or gpu_vram_gb
|
|
291
|
+
if vram_cap is not None:
|
|
292
|
+
over = predicted[predicted['pred_peak_vram_gb'] > vram_cap]
|
|
293
|
+
reasons.append(f"{len(over)}/{len(predicted)} exceed {vram_cap} GB VRAM cap")
|
|
294
|
+
if constraints.min_accuracy is not None:
|
|
295
|
+
under = predicted[predicted['pred_accuracy'] < constraints.min_accuracy]
|
|
296
|
+
reasons.append(f"{len(under)}/{len(predicted)} predicted below "
|
|
297
|
+
f"{constraints.min_accuracy} accuracy floor")
|
|
298
|
+
result['dropped_summary'] = reasons
|
|
299
|
+
return result
|
|
300
|
+
|
|
301
|
+
scored, time_used = score_gei(filtered, weights)
|
|
302
|
+
scored['on_pareto_front'] = pareto_front(filtered)
|
|
303
|
+
scored = scored.sort_values('gei', ascending=False)
|
|
304
|
+
|
|
305
|
+
result['ranked'] = scored.head(top_k)
|
|
306
|
+
result['pareto_only'] = scored[scored['on_pareto_front']].sort_values('gei', ascending=False)
|
|
307
|
+
result['time_objective_used'] = time_used
|
|
308
|
+
return result
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def explain(result: dict) -> str:
|
|
312
|
+
"""Human-readable summary of a recommend() result."""
|
|
313
|
+
if result['ranked'] is None:
|
|
314
|
+
lines = [f"No candidate satisfies every constraint out of {result['n_candidates']} tried.",
|
|
315
|
+
'Reasons:']
|
|
316
|
+
lines += [f' - {r}' for r in result['dropped_summary']]
|
|
317
|
+
lines.append('Loosen one constraint (VRAM cap, carbon cap, or accuracy floor) and retry.')
|
|
318
|
+
return '\n'.join(lines)
|
|
319
|
+
|
|
320
|
+
top = result['ranked'].iloc[0]
|
|
321
|
+
lines = [
|
|
322
|
+
f"Recommendation ({result['profile']} profile, weights={result['weights']}):",
|
|
323
|
+
f" {top['method']} on {top['backbone']} ({top['model_id']}, {top['params_b']}B params)",
|
|
324
|
+
f" predicted accuracy : {top['pred_accuracy']:.4f}",
|
|
325
|
+
f" predicted peak VRAM : {top['pred_peak_vram_gb']:.2f} GB",
|
|
326
|
+
f" predicted energy : {top['pred_energy_kwh']:.6f} kWh",
|
|
327
|
+
f" predicted carbon : {top['pred_carbon_kgco2eq']:.6f} kgCO2eq",
|
|
328
|
+
f" predicted wall-clock : {top['pred_wall_clock_s']:.1f} s"
|
|
329
|
+
if not pd.isna(top['pred_wall_clock_s']) else " predicted wall-clock : (no time model trained)",
|
|
330
|
+
f" GEI score : {top['gei']:.4f}"
|
|
331
|
+
+ (' [on Pareto front]' if top['on_pareto_front'] else ' [dominated by another feasible option -- see note below]'),
|
|
332
|
+
f" {result['n_feasible']}/{result['n_candidates']} candidates satisfied all constraints, "
|
|
333
|
+
f"{result['pareto_only'].shape[0]} of those are Pareto-optimal.",
|
|
334
|
+
]
|
|
335
|
+
if not result.get('time_objective_used', True):
|
|
336
|
+
lines.append(' Note: no wall-clock surrogate is in this artifacts export, so GEI and the '
|
|
337
|
+
'Pareto front were computed over accuracy/VRAM/carbon only (weights renormalized '
|
|
338
|
+
'over those three). Add wall_clock_seconds to Cell 13\'s REGRESSION_TARGETS and '
|
|
339
|
+
're-run to include a time objective.')
|
|
340
|
+
if not top['on_pareto_front']:
|
|
341
|
+
lines.append(' Note: top GEI pick is not Pareto-optimal under these exact weights -- this can '
|
|
342
|
+
'happen when weights trade off two close options; check pareto_only for alternatives.')
|
|
343
|
+
return '\n'.join(lines)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: green-peft
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Constraint-aware, zero-shot PEFT strategy recommender using the GreenPEFT surrogate models.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: scikit-learn<1.9,>=1.8
|
|
10
|
+
Requires-Dist: pandas<2.3,>=1.5
|
|
11
|
+
Requires-Dist: numpy<2.1,>=1.23
|
|
12
|
+
Requires-Dist: joblib>=1.2
|
|
13
|
+
Requires-Dist: pyyaml>=6.0
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# green-peft
|
|
17
|
+
|
|
18
|
+
Constraint-aware PEFT strategy recommender. Wraps the surrogate models trained in
|
|
19
|
+
Cell 13 of the GreenPEFT notebook so you can ask "which method + model size should I
|
|
20
|
+
use?" from the command line, without running anything.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Export artifacts from your notebook run
|
|
29
|
+
|
|
30
|
+
The CLI needs two things out of your Kaggle/Colab `peft_bench/` folder:
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
peft_bench/results/surrogate_models.joblib # written by Cell 13
|
|
34
|
+
peft_bench/configs/backbones.yaml # written by Cell 3 (has model_zoo:)
|
|
35
|
+
peft_bench/configs/methods/*.yaml # written by Cell 3
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Zip `results/` and `configs/` together, download from Kaggle, unzip locally into
|
|
39
|
+
e.g. `./peft_bench_export/`, and point `--artifacts-dir` at it.
|
|
40
|
+
|
|
41
|
+
This repository includes a ready-to-use export at:
|
|
42
|
+
|
|
43
|
+
```text
|
|
44
|
+
../../model/artifacts_export/
|
|
45
|
+
├── configs/backbones.yaml
|
|
46
|
+
├── configs/methods/*.yaml
|
|
47
|
+
├── configs/tasks.yaml
|
|
48
|
+
├── results/surrogate_models.joblib
|
|
49
|
+
├── results/surrogate_cv_metrics.json
|
|
50
|
+
└── results/surrogate_dataset.csv
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
From the repository root, install the package and use that export directly:
|
|
54
|
+
|
|
55
|
+
```powershell
|
|
56
|
+
cd green_peft_cli/green_peft_pkg
|
|
57
|
+
python -m pip install -e .
|
|
58
|
+
green-peft list-zoo --artifacts-dir ../../model/artifacts_export
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Usage
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# Basic recommendation under a VRAM + accuracy constraint
|
|
65
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
66
|
+
--vram 16 --accuracy 0.90 --profile balanced
|
|
67
|
+
|
|
68
|
+
# Strict carbon budget, custom weight profile, JSON output for scripting
|
|
69
|
+
green-peft recommend --artifacts-dir ./peft_bench_export \
|
|
70
|
+
--vram 24 --carbon 0.003 --weights 0.3,0.2,0.4,0.1 --top-k 5 --json
|
|
71
|
+
|
|
72
|
+
# See every backbone x method combination the engine can currently score
|
|
73
|
+
green-peft list-zoo --artifacts-dir ./peft_bench_export
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## What it actually does
|
|
77
|
+
|
|
78
|
+
1. Builds a candidate table crossing every backbone in your `model_zoo` (including
|
|
79
|
+
ones you never ran) with every configured method.
|
|
80
|
+
2. Runs the four surrogate models (feasibility classifier + accuracy / peak-VRAM /
|
|
81
|
+
energy / wall-clock regressors) over every candidate.
|
|
82
|
+
3. Drops candidates the feasibility classifier doesn't trust, or that violate your
|
|
83
|
+
`--vram` / `--carbon` / `--accuracy` / `--time` constraints.
|
|
84
|
+
4. Computes the Pareto front and a GEI score (same formula as the paper: weighted sum
|
|
85
|
+
of normalized accuracy/memory/carbon/time scores) over the surviving candidates.
|
|
86
|
+
5. Prints the top-k ranked options with a plain-English explanation, or an explicit
|
|
87
|
+
breakdown of why nothing survived if your constraints are infeasible together.
|
|
88
|
+
|
|
89
|
+
Predictions are only as reliable as the surrogate's leave-one-tier-out validation
|
|
90
|
+
(`results/surrogate_cv_metrics.json` from Cell 13) says they are for that target --
|
|
91
|
+
check `regression.<target>.best_model` R2 there before trusting a specific number.
|
|
92
|
+
|
|
93
|
+
## Current validation and limitations
|
|
94
|
+
|
|
95
|
+
The included export reports leave-one-tier-out feasibility accuracy of `0.7833`.
|
|
96
|
+
Selected regression models report the following validation results:
|
|
97
|
+
|
|
98
|
+
| Target | Model | MAE | R2 |
|
|
99
|
+
| --- | --- | ---: | ---: |
|
|
100
|
+
| Accuracy | Gradient boosting | 0.0182 | -0.4573 |
|
|
101
|
+
| Peak VRAM | Gradient boosting | 3.9703 GB | -0.5154 |
|
|
102
|
+
| Energy | Ridge | 0.000563 kWh | 0.3740 |
|
|
103
|
+
| Wall-clock time | Gradient boosting | 33.12 s | 0.4077 |
|
|
104
|
+
|
|
105
|
+
Use the CLI for experiment planning, shortlist generation, and explicit budget checks.
|
|
106
|
+
It does not run fine-tuning, measure the target hardware, or guarantee performance.
|
|
107
|
+
The benchmark evidence is primarily SST-2 classification on a Tesla T4; recommendations
|
|
108
|
+
for unbenchmarked catalog entries are extrapolations. Validate the selected option with
|
|
109
|
+
a real run and retain the JSON output alongside the measured results.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
green_peft/__init__.py
|
|
5
|
+
green_peft/cli.py
|
|
6
|
+
green_peft/recommender.py
|
|
7
|
+
green_peft.egg-info/PKG-INFO
|
|
8
|
+
green_peft.egg-info/SOURCES.txt
|
|
9
|
+
green_peft.egg-info/dependency_links.txt
|
|
10
|
+
green_peft.egg-info/entry_points.txt
|
|
11
|
+
green_peft.egg-info/requires.txt
|
|
12
|
+
green_peft.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
green_peft
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "green-peft"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Constraint-aware, zero-shot PEFT strategy recommender using the GreenPEFT surrogate models."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"scikit-learn>=1.8,<1.9",
|
|
15
|
+
"pandas>=1.5,<2.3",
|
|
16
|
+
"numpy>=1.23,<2.1",
|
|
17
|
+
"joblib>=1.2",
|
|
18
|
+
"pyyaml>=6.0",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
green-peft = "green_peft.cli:main"
|
|
23
|
+
|
|
24
|
+
[tool.setuptools.packages.find]
|
|
25
|
+
include = ["green_peft*"]
|