greengate 0.1.0__py3-none-any.whl

greengate/__init__.py

@@ -0,0 +1,12 @@
+from greengate.core import GreenGate, RouteResult
+
+__version__ = "0.1.0"
+__all__ = ["GreenGate", "RouteResult"]
+
+
+def __getattr__(name):
+    # legacy demo class, loaded lazily to keep `import greengate` light
+    if name == "GreenGateRouter":
+        from greengate.router import GreenGateRouter
+        return GreenGateRouter
+    raise AttributeError(name)

greengate/api_tier.py

@@ -0,0 +1,153 @@
+"""Large-tier inference via OpenAI API with EcoLogits carbon estimation.
+
+Dual-Mode Carbon Profiler, API side: local models are measured with
+pynvml (profiler.py); API models are estimated with EcoLogits
+(Rince et al., 2025, JOSS). If EcoLogits is unavailable or fails,
+falls back to a documented per-token estimate and flags it.
+"""
+
+import os
+import time
+from dataclasses import dataclass
+
+# Fallback constants (used ONLY if EcoLogits fails) — documented in thesis:
+# Jegham et al. (2025) measure ~0.42 Wh for a mean GPT-4o query (~150 output
+# tokens). GPT-4o-mini is a distilled/smaller deployment; we conservatively
+# assume 0.5x the 4o figure, scaled linearly by output tokens.
+FALLBACK_WH_PER_OUTPUT_TOKEN = 0.42 * 0.5 / 150  # = 0.0014 Wh/token
+WORLD_CARBON_INTENSITY = 475.0  # gCO2/kWh
+DATACENTER_PUE = 1.2
+
+
+@dataclass
+class APIResult:
+    response: str
+    model: str
+    input_tokens: int
+    output_tokens: int
+    latency_s: float
+    energy_wh: float
+    carbon_grams: float
+    carbon_source: str  # "ecologits" or "fallback_estimate"
+    # confidence signals from top-20 logprobs (None unless logprobs=True):
+    avg_logprob: float | None = None      # mean log P(chosen token)
+    trunc_entropy: float | None = None    # mean entropy over renormalised top-20 (bits)
+
+
+class APILargeTier:
+    def __init__(self, model: str = "gpt-4o-mini", max_tokens: int = 300,
+                 dry_run: bool = False, logprobs: bool = False):
+        self.model = model
+        self.max_tokens = max_tokens
+        self.dry_run = dry_run
+        self.logprobs = logprobs
+        self._ecologits_ok = False
+
+        if dry_run:
+            return
+
+        # EcoLogits must be initialised BEFORE the OpenAI client is created.
+        try:
+            from ecologits import EcoLogits
+            EcoLogits.init(providers=["openai"])
+            self._ecologits_ok = True
+        except Exception as e:
+            print(f"  [api_tier] EcoLogits unavailable ({e}) — using documented fallback estimate")
+
+        from openai import OpenAI
+        self.client = OpenAI()  # reads OPENAI_API_KEY env var
+
+    def query_vision(self, prompt: str, image) -> APIResult:
+        """Vision escalation: PIL image sent as a base64 data URL."""
+        if self.dry_run:
+            return self.query(prompt)
+        import base64
+        import io
+        buf = io.BytesIO()
+        image.save(buf, format="JPEG", quality=85)
+        b64 = base64.b64encode(buf.getvalue()).decode()
+        content = [
+            {"type": "text", "text": prompt},
+            {"type": "image_url",
+             "image_url": {"url": f"data:image/jpeg;base64,{b64}"}},
+        ]
+        return self._chat(content)
+
+    def query(self, prompt: str) -> APIResult:
+        if self.dry_run:
+            return APIResult(
+                response="[DRY RUN — no API call made]",
+                model=self.model, input_tokens=len(prompt) // 4,
+                output_tokens=50, latency_s=0.5,
+                energy_wh=50 * FALLBACK_WH_PER_OUTPUT_TOKEN,
+                carbon_grams=50 * FALLBACK_WH_PER_OUTPUT_TOKEN / 1000
+                * DATACENTER_PUE * WORLD_CARBON_INTENSITY,
+                carbon_source="dry_run",
+            )
+
+        return self._chat(prompt)
+
+    def _chat(self, content, retries: int = 6) -> APIResult:
+        kwargs = {}
+        if self.logprobs:
+            kwargs = {"logprobs": True, "top_logprobs": 20}
+        from openai import RateLimitError, APIError
+        t0 = time.perf_counter()
+        for attempt in range(retries):
+            try:
+                resp = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": content}],
+                    max_tokens=self.max_tokens,
+                    **kwargs,
+                )
+                break
+            except (RateLimitError, APIError):
+                if attempt == retries - 1:
+                    raise
+                time.sleep(min(2 ** attempt, 30))  # 1,2,4,8,16,30s backoff
+        latency = time.perf_counter() - t0
+
+        avg_lp = trunc_H = None
+        lp_content = getattr(resp.choices[0].logprobs, "content", None) \
+            if self.logprobs and resp.choices[0].logprobs else None
+        if lp_content:
+            import math
+            lps, ents = [], []
+            for tok in lp_content:
+                lps.append(tok.logprob)
+                if tok.top_logprobs:
+                    ps = [math.exp(t.logprob) for t in tok.top_logprobs]
+                    z = sum(ps) or 1.0
+                    ents.append(-sum(p / z * math.log2(p / z) for p in ps if p > 0))
+            avg_lp = sum(lps) / len(lps) if lps else None
+            trunc_H = sum(ents) / len(ents) if ents else None
+
+        text = resp.choices[0].message.content or ""
+        in_tok = resp.usage.prompt_tokens
+        out_tok = resp.usage.completion_tokens
+
+        energy_wh, carbon_g, source = None, None, "fallback_estimate"
+        if self._ecologits_ok and hasattr(resp, "impacts"):
+            try:
+                # EcoLogits attaches .impacts; energy in kWh (value or range)
+                e = resp.impacts.energy.value
+                energy_kwh = (e.min + e.max) / 2 if hasattr(e, "min") else float(e)
+                g = resp.impacts.gwp.value
+                gwp_kg = (g.min + g.max) / 2 if hasattr(g, "min") else float(g)
+                energy_wh = energy_kwh * 1000
+                carbon_g = gwp_kg * 1000
+                source = "ecologits"
+            except Exception:
+                pass
+        if energy_wh is None:
+            energy_wh = out_tok * FALLBACK_WH_PER_OUTPUT_TOKEN
+            carbon_g = energy_wh / 1000 * DATACENTER_PUE * WORLD_CARBON_INTENSITY
+
+        return APIResult(
+            response=text.strip(), model=self.model,
+            input_tokens=in_tok, output_tokens=out_tok,
+            latency_s=latency, energy_wh=energy_wh,
+            carbon_grams=carbon_g, carbon_source=source,
+            avg_logprob=avg_lp, trunc_entropy=trunc_H,
+        )

greengate/budget.py

@@ -0,0 +1,31 @@
+"""Sliding-window carbon budget (Chapter 3, Design Analysis 2.5.3).
+
+Maintains a window of the last `window_s` seconds of routing decisions.
+If the carbon spent inside the window would exceed budget K, escalations
+are temporarily blocked (the query falls back to the small answer) until
+older queries age out of the window.
+"""
+
+from collections import deque
+
+
+class SlidingWindowBudget:
+    def __init__(self, budget_g: float, window_s: float = 3600.0):
+        self.budget_g = budget_g
+        self.window_s = window_s
+        self._events: deque[tuple[float, float]] = deque()  # (timestamp, carbon_g)
+
+    def _expire(self, now: float):
+        while self._events and self._events[0][0] < now - self.window_s:
+            self._events.popleft()
+
+    def window_carbon(self, now: float) -> float:
+        self._expire(now)
+        return sum(c for _, c in self._events)
+
+    def allows(self, now: float, escalation_cost_g: float) -> bool:
+        """Would escalating now keep the window under budget?"""
+        return self.window_carbon(now) + escalation_cost_g <= self.budget_g
+
+    def record(self, now: float, carbon_g: float):
+        self._events.append((now, carbon_g))

greengate/calibration.py

@@ -0,0 +1,53 @@
+"""Temperature scaling calibration (Guo et al., 2017).
+
+Fits a single scalar T on a held-out validation split by minimising
+negative log-likelihood. T > 1 softens overconfident distributions.
+Reports Expected Calibration Error (ECE) before and after.
+"""
+
+import torch
+import torch.nn.functional as F
+
+
+def fit_temperature(logits: torch.Tensor, labels: torch.Tensor,
+                    t_min: float = 0.25, t_max: float = 8.0,
+                    n_grid: int = 400) -> float:
+    """Fit T by fine grid search over NLL (robust, dependency-free).
+
+    Args:
+        logits: (n, k) raw choice logits from the validation split
+        labels: (n,) correct choice indices
+    """
+    best_t, best_nll = 1.0, float("inf")
+    for i in range(n_grid):
+        t = t_min + (t_max - t_min) * i / (n_grid - 1)
+        nll = F.cross_entropy(logits / t, labels).item()
+        if nll < best_nll:
+            best_nll, best_t = nll, t
+    return best_t
+
+
+def ece(logits: torch.Tensor, labels: torch.Tensor,
+        temperature: float = 1.0, n_bins: int = 15) -> float:
+    """Expected Calibration Error over equal-width confidence bins."""
+    probs = F.softmax(logits / temperature, dim=-1)
+    conf, pred = probs.max(dim=-1)
+    correct = (pred == labels).float()
+
+    total = 0.0
+    n = len(labels)
+    for b in range(n_bins):
+        lo, hi = b / n_bins, (b + 1) / n_bins
+        mask = (conf > lo) & (conf <= hi)
+        if mask.sum() == 0:
+            continue
+        avg_conf = conf[mask].mean().item()
+        avg_acc = correct[mask].mean().item()
+        total += (mask.sum().item() / n) * abs(avg_conf - avg_acc)
+    return total
+
+
+def calibrated_entropy(logits: torch.Tensor, temperature: float) -> float:
+    """Shannon entropy (bits) of the temperature-scaled distribution."""
+    probs = F.softmax(logits / temperature, dim=-1).clamp(min=1e-9)
+    return -(probs * probs.log2()).sum(dim=-1).item()

greengate/core.py

@@ -0,0 +1,280 @@
+"""GreenGate — the public library API.
+
+    import greengate
+    gw = greengate.GreenGate(small="Qwen/Qwen2.5-0.5B-Instruct",
+                             large="gpt-4o-mini", budget_g=0.5)
+    r = gw.route("Summarise this document ...")
+    r.response, r.decision, r.carbon_g
+    gw.profile()      # session totals: carbon, escalation rate, wasted cost
+    gw.calibrate()    # one-time temperature fit for unlisted small models
+
+Tier rules (by design, see thesis Ch.3):
+  small — any open-weight transformers model, runs locally (the entropy
+          signal needs token logits, which APIs do not expose)
+  large — a local transformers model ("org/name") OR an OpenAI API model
+          name ("gpt-4o-mini")
+"""
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+PRESETS_PATH = Path(__file__).parent / "presets.json"
+USER_CALIBRATIONS = Path.home() / ".greengate" / "calibrations.json"
+
+# threshold percentile used while auto-tuning on the user's own traffic
+AUTO_THRESHOLD_PERCENTILE = {"green": 80, "balanced": 60, "quality": 35}
+WARMUP_QUERIES = 20
+
+
+@dataclass
+class RouteResult:
+    response: str
+    decision: str            # "LOCAL" or "ESCALATE" (or "LOCAL(budget)")
+    signal: float            # entropy (bits) or semantic entropy
+    threshold: float | None
+    carbon_g: float          # full accounting: includes wasted small run
+    wasted_carbon_g: float
+    latency_s: float
+    small_model: str
+    large_model: str
+
+
+@dataclass
+class _Session:
+    queries: int = 0
+    escalated: int = 0
+    budget_blocked: int = 0
+    carbon_g: float = 0.0
+    wasted_carbon_g: float = 0.0
+    energy_j: float = 0.0
+    latency_s: float = 0.0
+
+
+def _load_temperature(model_name: str) -> tuple[float, str]:
+    """Preset registry first, then the user's own calibrations, else 1.0."""
+    for path, source in [(PRESETS_PATH, "preset"),
+                         (USER_CALIBRATIONS, "user-calibrated")]:
+        try:
+            data = json.loads(path.read_text())
+            if model_name in data:
+                return float(data[model_name]["temperature"]), source
+        except (OSError, json.JSONDecodeError):
+            pass
+    return 1.0, "uncalibrated"
+
+
+class GreenGate:
+    def __init__(self, small: str = "Qwen/Qwen2.5-0.5B-Instruct",
+                 large: str = "gpt-4o-mini",
+                 mode: str = "balanced",
+                 threshold: float | None = None,
+                 budget_g: float | None = None,
+                 budget_window_s: float = 3600.0,
+                 signal: str = "entropy",
+                 small_4bit: bool = False,
+                 large_4bit: bool = True,
+                 max_new_tokens: int = 200,
+                 dry_run_api: bool = False,
+                 large_is_api: bool | None = None):
+        if mode not in AUTO_THRESHOLD_PERCENTILE:
+            raise ValueError(f"mode must be one of {list(AUTO_THRESHOLD_PERCENTILE)}")
+        if signal not in ("entropy", "semantic"):
+            raise ValueError("signal must be 'entropy' or 'semantic'")
+
+        self.small_name, self.large_name = small, large
+        self.mode = mode
+        self.signal = signal
+        self._fixed_threshold = threshold
+        self._entropy_history: list[float] = []
+
+        T, source = _load_temperature(small)
+        self.temperature = T
+        if source == "uncalibrated":
+            print(f"[greengate] no calibration preset for {small} — routing on "
+                  f"raw entropy with auto-threshold; run gw.calibrate() to fit one")
+
+        from greengate.textgen import SmallTextModel
+        self._small = SmallTextModel(small, temperature_T=T,
+                                     max_new_tokens=max_new_tokens,
+                                     load_in_4bit=small_4bit)
+
+        if large_is_api is None:  # auto-detect: OpenAI naming vs HF hub id
+            large_is_api = large.startswith(("gpt-", "o1", "o3", "o4", "chatgpt"))
+        self._large_is_api = large_is_api
+        if self._large_is_api:
+            from greengate.api_tier import APILargeTier
+            self._large = APILargeTier(model=large, max_tokens=max_new_tokens,
+                                       dry_run=dry_run_api)
+        else:
+            self._large = SmallTextModel(large, max_new_tokens=max_new_tokens,
+                                         load_in_4bit=large_4bit)
+
+        self._budget = None
+        if budget_g is not None:
+            from greengate.budget import SlidingWindowBudget
+            self._budget = SlidingWindowBudget(budget_g, budget_window_s)
+
+        self._semantic = None  # lazy — only if signal="semantic"
+        self._session = _Session()
+
+    # ------------------------------------------------------------------ #
+
+    def _threshold(self) -> float | None:
+        if self._fixed_threshold is not None:
+            return self._fixed_threshold
+        if len(self._entropy_history) < WARMUP_QUERIES:
+            return None  # warmup: not enough traffic seen yet
+        h = sorted(self._entropy_history)
+        pct = AUTO_THRESHOLD_PERCENTILE[self.mode]
+        return h[int(pct / 100 * (len(h) - 1))]
+
+    def _semantic_entropy(self, query: str, k: int = 3) -> tuple[float, float]:
+        """(semantic entropy, extra energy J). EXPERIMENTAL — k extra samples."""
+        import torch
+        if self._semantic is None:
+            from greengate.semantic import NLIClusterer
+            self._semantic = NLIClusterer()
+        answers, extra_j = [], 0.0
+        for _ in range(k):
+            with torch.no_grad():
+                prompt = self._small._chat_wrap(query)
+                inputs = self._small.tokenizer(prompt, return_tensors="pt",
+                                               truncation=True, max_length=1024)
+                inputs = {kk: v.to(self._small.model.device)
+                          for kk, v in inputs.items()}
+                self._small.profiler.start()
+                out = self._small.model.generate(
+                    **inputs, max_new_tokens=80, do_sample=True,
+                    temperature=1.0, top_p=0.95,
+                    pad_token_id=self._small.tokenizer.pad_token_id)
+                e, _ = self._small.profiler.stop()
+                extra_j += e
+            gen = out[0][inputs["input_ids"].shape[1]:]
+            answers.append(self._small.tokenizer.decode(
+                gen, skip_special_tokens=True).strip())
+        from greengate.semantic import semantic_entropy
+        return semantic_entropy(self._semantic.cluster(answers)), extra_j
+
+    # ------------------------------------------------------------------ #
+
+    def route(self, query: str) -> RouteResult:
+        t0 = time.perf_counter()
+        small_r = self._small.generate(query)
+
+        if self.signal == "semantic":
+            sig, extra_j = self._semantic_entropy(query)
+            extra_c = extra_j / 3_600_000.0 * 1.2 * 475.0
+        else:
+            sig = small_r.entropy_calibrated
+            extra_c = 0.0
+        self._entropy_history.append(sig)
+
+        thr = self._threshold()
+        wants_escalation = thr is not None and sig > thr
+
+        decision = "LOCAL"
+        response = small_r.response
+        carbon = small_r.carbon_grams + extra_c
+        wasted = 0.0
+
+        if wants_escalation:
+            esc_cost_estimate = carbon * 3  # rough pre-check for the budget
+            now = time.monotonic()
+            if self._budget is not None and not self._budget.allows(now, esc_cost_estimate):
+                decision = "LOCAL(budget)"
+                self._session.budget_blocked += 1
+            else:
+                decision = "ESCALATE"
+                if self._large_is_api:
+                    large_r = self._large.query(query)
+                    large_carbon = large_r.carbon_grams
+                    response = large_r.response
+                else:
+                    large_r = self._large.generate(query)
+                    large_carbon = large_r.carbon_grams
+                    response = large_r.response
+                wasted = small_r.carbon_grams  # full accounting
+                carbon = large_carbon + wasted + extra_c
+
+        if self._budget is not None:
+            self._budget.record(time.monotonic(), carbon)
+
+        latency = time.perf_counter() - t0
+        s = self._session
+        s.queries += 1
+        s.escalated += decision == "ESCALATE"
+        s.carbon_g += carbon
+        s.wasted_carbon_g += wasted
+        s.latency_s += latency
+
+        return RouteResult(
+            response=response, decision=decision, signal=sig, threshold=thr,
+            carbon_g=carbon, wasted_carbon_g=wasted, latency_s=latency,
+            small_model=self.small_name, large_model=self.large_name)
+
+    def profile(self) -> dict:
+        s = self._session
+        return {
+            "queries": s.queries,
+            "escalation_rate": s.escalated / s.queries if s.queries else 0.0,
+            "budget_blocked": s.budget_blocked,
+            "total_carbon_g": round(s.carbon_g, 6),
+            "wasted_carbon_g": round(s.wasted_carbon_g, 6),
+            "avg_latency_s": round(s.latency_s / s.queries, 3) if s.queries else 0.0,
+            "small_model": self.small_name,
+            "large_model": self.large_name,
+            "signal": self.signal,
+            "threshold": self._threshold(),
+        }
+
+    def config(self, threshold: float | None = None,
+               budget_g: float | None = None,
+               mode: str | None = None):
+        if threshold is not None:
+            self._fixed_threshold = threshold
+        if mode is not None:
+            if mode not in AUTO_THRESHOLD_PERCENTILE:
+                raise ValueError(f"mode must be one of {list(AUTO_THRESHOLD_PERCENTILE)}")
+            self.mode = mode
+        if budget_g is not None:
+            from greengate.budget import SlidingWindowBudget
+            self._budget = SlidingWindowBudget(budget_g, 3600.0)
+        return self
+
+    def calibrate(self, n: int = 150, seed: int = 42) -> dict:
+        """Fit temperature scaling for this small model on held-out MMLU
+        validation (the thesis methodology), save it under ~/.greengate/."""
+        import torch
+        from greengate.mmlu import load_mmlu
+        from greengate.evaluator import ChoiceEvaluator
+        from greengate.calibration import fit_temperature, ece
+
+        print(f"[greengate] calibrating {self.small_name} on {n} MMLU "
+              f"validation questions...")
+        ev = ChoiceEvaluator(self.small_name)
+        logits, labels = [], []
+        for q in load_mmlu(n_questions=n, seed=seed, split="validation"):
+            r = ev.evaluate(q)
+            logits.append(r.choice_logits)
+            labels.append(q.answer_idx)
+        lt, lb = torch.tensor(logits), torch.tensor(labels)
+        T = fit_temperature(lt, lb)
+        result = {"temperature": round(T, 4),
+                  "ece_before": round(ece(lt, lb, 1.0), 5),
+                  "ece_after": round(ece(lt, lb, T), 5),
+                  "fitted_on": f"MMLU validation n={n}"}
+
+        USER_CALIBRATIONS.parent.mkdir(parents=True, exist_ok=True)
+        data = {}
+        if USER_CALIBRATIONS.exists():
+            data = json.loads(USER_CALIBRATIONS.read_text())
+        data[self.small_name] = result
+        USER_CALIBRATIONS.write_text(json.dumps(data, indent=2))
+
+        self.temperature = T
+        self._small.T = T
+        print(f"[greengate] T={T:.3f}, ECE {result['ece_before']:.4f} -> "
+              f"{result['ece_after']:.4f}, saved to {USER_CALIBRATIONS}")
+        return result

greengate/entropy.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn.functional as F
+
+
+def shannon_entropy(logits: torch.Tensor) -> float:
+    """Compute Shannon entropy H(x) = -sum(P(x) * log2(P(x))) over token logits.
+
+    Args:
+        logits: Raw logits tensor of shape (vocab_size,) from a single generation step.
+
+    Returns:
+        Entropy in bits. Low = confident, high = uncertain.
+    """
+    probs = F.softmax(logits, dim=-1)
+    probs = probs.clamp(min=1e-9)
+    entropy = -(probs * probs.log2()).sum(dim=-1)
+    return entropy.item()
+
+
+def mean_token_entropy(all_logits: list[torch.Tensor]) -> float:
+    """Average Shannon entropy across all generated tokens."""
+    if not all_logits:
+        return 0.0
+    entropies = [shannon_entropy(logits.squeeze()) for logits in all_logits]
+    return sum(entropies) / len(entropies)
+
+
+def first_token_entropy(all_logits: list[torch.Tensor]) -> float:
+    """Shannon entropy of just the first generated token.
+
+    The first token is often the most informative signal for routing:
+    a confident model commits early, an uncertain model hedges immediately.
+    """
+    if not all_logits:
+        return 0.0
+    return shannon_entropy(all_logits[0].squeeze())

greengate/evaluator.py

@@ -0,0 +1,102 @@
+"""Multiple-choice evaluator for MMLU-style questions.
+
+Instead of free-text generation, we do a SINGLE forward pass and read the
+next-token logits restricted to the four choice letters (A/B/C/D). This gives:
+
+- prediction: argmax over the four letter tokens
+- choice entropy: Shannon entropy over the renormalised 4-way distribution
+  (0 bits = fully confident, 2 bits = maximally uncertain)
+- energy/carbon: measured around the forward pass via CarbonProfiler
+
+One forward pass per question per model keeps the evaluation cheap enough
+for CPU smoke tests and free-tier GPUs.
+"""
+
+import torch
+import torch.nn.functional as F
+from dataclasses import dataclass
+
+from greengate.mmlu import MMLUQuestion, CHOICE_LETTERS
+from greengate.profiler import CarbonProfiler
+
+
+@dataclass
+class ChoiceResult:
+    predicted_idx: int
+    correct: bool
+    entropy: float  # bits, over the 4 choices (max = 2.0)
+    energy_joules: float
+    carbon_grams: float
+    choice_logits: list[float] | None = None  # raw A/B/C/D logits (for calibration)
+
+
+class ChoiceEvaluator:
+    """Wraps one causal LM for multiple-choice prediction with energy tracking."""
+
+    def __init__(self, model_name: str, device: str | None = None,
+                 load_in_4bit: bool = False,
+                 carbon_intensity: float = 475.0, pue: float = 1.2):
+        self.model_name = model_name
+        if device is None:
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.device = device
+
+        from transformers import AutoTokenizer, AutoModelForCausalLM
+
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        load_kwargs = {}
+        if device == "cuda":
+            load_kwargs["device_map"] = "auto"
+            if load_in_4bit:
+                from transformers import BitsAndBytesConfig
+                load_kwargs["quantization_config"] = BitsAndBytesConfig(
+                    load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16
+                )
+            else:
+                load_kwargs["dtype"] = torch.float16
+        else:
+            load_kwargs["dtype"] = torch.float32
+
+        self.model = AutoModelForCausalLM.from_pretrained(model_name, **load_kwargs)
+        if device == "cpu":
+            self.model = self.model.to("cpu")
+        self.model.eval()
+
+        self.profiler = CarbonProfiler(carbon_intensity=carbon_intensity, pue=pue)
+
+        # Token ids for " A", " B", " C", " D" (with leading space, as they
+        # follow "Answer:"). Fall back to bare letters if needed.
+        self.choice_token_ids = []
+        for letter in CHOICE_LETTERS:
+            ids = self.tokenizer.encode(" " + letter, add_special_tokens=False)
+            if len(ids) != 1:
+                ids = self.tokenizer.encode(letter, add_special_tokens=False)
+            self.choice_token_ids.append(ids[0])
+
+    @torch.no_grad()
+    def evaluate(self, q: MMLUQuestion) -> ChoiceResult:
+        prompt = q.to_prompt()
+        inputs = self.tokenizer(prompt, return_tensors="pt", truncation=True,
+                                max_length=1024)
+        inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
+
+        self.profiler.start()
+        logits = self.model(**inputs).logits[0, -1, :]
+        energy, carbon = self.profiler.stop()
+
+        choice_logits = logits[self.choice_token_ids]
+        probs = F.softmax(choice_logits, dim=-1).clamp(min=1e-9)
+        entropy = -(probs * probs.log2()).sum().item()
+        predicted_idx = int(probs.argmax().item())
+
+        return ChoiceResult(
+            predicted_idx=predicted_idx,
+            correct=(predicted_idx == q.answer_idx),
+            entropy=entropy,
+            energy_joules=energy,
+            carbon_grams=carbon,
+            choice_logits=[float(x) for x in choice_logits.tolist()],
+        )