gbra 0.1.0__py3-none-any.whl

gbra/__init__.py

@@ -0,0 +1,29 @@
+"""GBRA - Gember-Berke Rank Adaptation ("Gember RA").
+
+DoRA-style fine-tuning with a buffered (lazily refreshed) weight norm.
+GEMBER && Apex Logic Studio - Berke T. Adanc
+"""
+
+from .layer import GBRALinear
+from .model import (
+    DEFAULT_TARGETS,
+    apply_gbra, refresh_norms, save_gbra, load_gbra, merge_gbra, gbra_state_dict,
+    gbra_ekle, tum_normlari_yenile, gbra_kaydet, gbra_yukle, gbra_birlestir,
+)
+
+__version__ = "0.1.0"
+
+
+def __getattr__(name):
+    # GBRACallback needs transformers; import it lazily so the core works without it.
+    if name == "GBRACallback":
+        from .callback import GBRACallback
+        return GBRACallback
+    raise AttributeError(name)
+
+
+__all__ = [
+    "GBRALinear", "GBRACallback", "DEFAULT_TARGETS",
+    "apply_gbra", "refresh_norms", "save_gbra", "load_gbra", "merge_gbra", "gbra_state_dict",
+    "gbra_ekle", "tum_normlari_yenile", "gbra_kaydet", "gbra_yukle", "gbra_birlestir",
+]

gbra/callback.py

@@ -0,0 +1,23 @@
+"""Hugging Face Trainer integration."""
+
+from transformers import TrainerCallback
+
+from .model import refresh_norms
+
+
+class GBRACallback(TrainerCallback):
+    """Refreshes GBRA norms every `refresh_every` optimizer steps (K).
+
+    Use refresh_every=None to never refresh (the norm stays at its initial value).
+
+        trainer = Trainer(..., callbacks=[GBRACallback(refresh_every=10)])
+    """
+
+    def __init__(self, refresh_every: int | None = 10):
+        if refresh_every is not None and refresh_every < 1:
+            raise ValueError("refresh_every must be >= 1 or None")
+        self.k = refresh_every
+
+    def on_step_end(self, args, state, control, model=None, **kwargs):
+        if self.k is not None and model is not None and state.global_step % self.k == 0:
+            refresh_norms(model)

gbra/layer.py

@@ -0,0 +1,97 @@
+"""GBRALinear: the core GBRA layer.
+
+GBRA (Gember-Berke Rank Adaptation, read "Gember RA") keeps DoRA's
+magnitude/direction split but buffers the column-norm instead of
+recomputing it on every forward pass:
+
+    DoRA:  W' = m * (W + s*B@A) / ||W + s*B@A||     (norm recomputed every forward)
+    GBRA:  W' = m * (W + s*B@A) / n                 (n is a buffer, refreshed every K optimizer steps)
+
+Norms are taken per output row (W has shape [out, in]).
+"""
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+def _is_4bit(layer: nn.Module) -> bool:
+    return hasattr(getattr(layer, "weight", None), "quant_state")
+
+
+def base_weight(layer: nn.Linear) -> torch.Tensor:
+    """Return the base weight as a dense float32 tensor (dequantizes 4-bit layers)."""
+    if _is_4bit(layer):
+        import bitsandbytes.functional as bnbF  # optional dependency
+        return bnbF.dequantize_4bit(layer.weight.data, layer.weight.quant_state).float()
+    return layer.weight.detach().float()
+
+
+class GBRALinear(nn.Module):
+    def __init__(self, taban: nn.Linear, r: int = 16, alpha: int = 16):
+        super().__init__()
+        if r <= 0:
+            raise ValueError("r must be a positive integer")
+        self.taban = taban                      # original layer (frozen)
+        for p in self.taban.parameters():
+            p.requires_grad = False
+
+        cikis, giris = taban.out_features, taban.in_features
+        self.r, self.alpha = r, alpha
+        self.s = alpha / r                      # LoRA scale
+        device = taban.weight.device
+
+        # LoRA matrices. B starts at zero, so at init GBRA == the original layer.
+        self.A = nn.Parameter(torch.empty(r, giris, device=device, dtype=torch.float32))
+        self.B = nn.Parameter(torch.zeros(cikis, r, device=device, dtype=torch.float32))
+        nn.init.kaiming_uniform_(self.A, a=5 ** 0.5)
+
+        # Magnitude vector m (learned), initialised to the row norm of W.
+        with torch.no_grad():
+            norm0 = base_weight(taban).norm(dim=1)
+        self.m = nn.Parameter(norm0.clone())
+
+        # Buffered norm n (NOT learned). Updated only by normu_yenile().
+        self.register_buffer("n", norm0.clone())
+
+    @torch.no_grad()
+    def normu_yenile(self):
+        """Lazy norm refresh: compute ||W + s*B@A|| for real and store it in n."""
+        W = base_weight(self.taban)
+        delta = self.s * (self.B.float() @ self.A.float())
+        self.n.copy_((W + delta).norm(dim=1))
+
+    refresh_norm = normu_yenile  # English alias
+
+    def _ana(self, x):
+        """x @ W.T without bias (works for normal and 4-bit layers)."""
+        if _is_4bit(self.taban):
+            y = self.taban(x)
+            if self.taban.bias is not None:
+                y = y - self.taban.bias
+            return y
+        return F.linear(x, self.taban.weight, bias=None)
+
+    def forward(self, x):
+        # forward: Berke T. Adanc
+        # 1. Ana ağırlık: x @ W.T (bias eklemeden)
+        ana = self._ana(x)
+
+        # 2. LoRA: önce A, sonra B
+        lora_a = F.linear(x, self.A.to(x.dtype), bias=None)
+        lora_b = F.linear(lora_a, self.B.to(x.dtype), bias=None)
+
+        # 3. Ölçek: m / n, ardından x'in dtype'ına dönüştür
+        olcek = (self.m / self.n).to(x.dtype)
+
+        # 4. Ana ağırlık + LoRA toplamını ölçekle
+        y = olcek * (ana + self.s * lora_b)
+
+        # 5. Bias'ı en son ekle (None olabilir)
+        if self.taban.bias is not None:
+            y = y + self.taban.bias
+
+        return y
+
+    def extra_repr(self):
+        return f"in={self.taban.in_features}, out={self.taban.out_features}, r={self.r}, alpha={self.alpha}"

gbra/model.py

@@ -0,0 +1,123 @@
+"""Model-level helpers: attach GBRA, refresh norms, merge, save and load."""
+
+import json
+import os
+
+import torch
+import torch.nn as nn
+
+from .layer import GBRALinear, base_weight
+
+DEFAULT_TARGETS = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj")
+CONFIG_NAME = "gbra_config.json"
+WEIGHTS_NAME = "gbra_adapter.safetensors"
+
+
+def apply_gbra(model: nn.Module, targets=DEFAULT_TARGETS, r: int = 16, alpha: int = 16,
+               verbose: bool = True) -> nn.Module:
+    """Freeze the model and wrap every target nn.Linear (incl. 4-bit) with GBRALinear."""
+    for p in model.parameters():
+        p.requires_grad = False
+    count = 0
+    for module in list(model.modules()):
+        for name, child in list(module.named_children()):
+            if name in targets and isinstance(child, nn.Linear) and not isinstance(child, GBRALinear):
+                setattr(module, name, GBRALinear(child, r=r, alpha=alpha))
+                count += 1
+    if count == 0:
+        raise ValueError(f"No target layers found. Check the layer names of your model. targets={targets}")
+    model._gbra_config = {"r": r, "alpha": alpha, "targets": list(targets)}
+    if verbose:
+        trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
+        total = sum(p.numel() for p in model.parameters())
+        print(f"GBRA: {count} layers | trainable {trainable:,} / {total:,} ({100 * trainable / total:.2f}%)")
+    return model
+
+
+@torch.no_grad()
+def refresh_norms(model: nn.Module) -> int:
+    """Refresh the buffered norm of every GBRA layer. Returns the number of layers."""
+    n = 0
+    for m in model.modules():
+        if isinstance(m, GBRALinear):
+            m.normu_yenile()
+            n += 1
+    return n
+
+
+def gbra_state_dict(model: nn.Module) -> dict:
+    """Only the GBRA tensors (A, B, m, n) - small enough to share as an adapter."""
+    out = {}
+    for name, m in model.named_modules():
+        if isinstance(m, GBRALinear):
+            for key in ("A", "B", "m", "n"):
+                out[f"{name}.{key}"] = getattr(m, key).detach().cpu().contiguous()
+    return out
+
+
+def save_gbra(model: nn.Module, path: str):
+    os.makedirs(path, exist_ok=True)
+    cfg = getattr(model, "_gbra_config", None)
+    if cfg is None:
+        raise ValueError("This model has no GBRA layers (call apply_gbra first).")
+    with open(os.path.join(path, CONFIG_NAME), "w", encoding="utf-8") as f:
+        json.dump({"format": "gbra", "version": 1, **cfg}, f, indent=2)
+    from safetensors.torch import save_file
+    save_file(gbra_state_dict(model), os.path.join(path, WEIGHTS_NAME))
+
+
+def load_gbra(model: nn.Module, path: str) -> nn.Module:
+    """Attach GBRA to a fresh base model and load adapter weights saved with save_gbra."""
+    with open(os.path.join(path, CONFIG_NAME), encoding="utf-8") as f:
+        cfg = json.load(f)
+    if not any(isinstance(m, GBRALinear) for m in model.modules()):
+        apply_gbra(model, targets=tuple(cfg["targets"]), r=cfg["r"], alpha=cfg["alpha"], verbose=False)
+    from safetensors.torch import load_file
+    tensors = load_file(os.path.join(path, WEIGHTS_NAME))
+    modules = dict(model.named_modules())
+    used = set()
+    for full, t in tensors.items():
+        name, key = full.rsplit(".", 1)
+        layer = modules.get(name)
+        if not isinstance(layer, GBRALinear):
+            raise KeyError(f"Adapter tensor {full} has no matching GBRA layer in this model.")
+        getattr(layer, key).data.copy_(t.to(getattr(layer, key).device))
+        used.add(name)
+    missing = [n for n, m in modules.items() if isinstance(m, GBRALinear) and n not in used]
+    if missing:
+        raise KeyError(f"No adapter weights for {len(missing)} GBRA layers, e.g. {missing[0]}")
+    return model
+
+
+@torch.no_grad()
+def merge_gbra(model: nn.Module, dtype=None) -> nn.Module:
+    """Fold every GBRA layer into a plain nn.Linear: W' = (m/n) * (W + s*B@A).
+
+    The merged model gives the same outputs as the GBRA model (using the current
+    buffered norm). 4-bit layers are dequantized, so the result is a dense model
+    you can export (e.g. to GGUF).
+    """
+    for module in list(model.modules()):
+        for name, child in list(module.named_children()):
+            if isinstance(child, GBRALinear):
+                W = base_weight(child.taban)
+                Wm = (child.m / child.n)[:, None] * (W + child.s * child.B.float() @ child.A.float())
+                bias = child.taban.bias
+                out_dtype = dtype or (bias.dtype if bias is not None else torch.float32)
+                lin = nn.Linear(Wm.shape[1], Wm.shape[0], bias=bias is not None,
+                                device=Wm.device, dtype=out_dtype)
+                lin.weight.copy_(Wm.to(out_dtype))
+                if bias is not None:
+                    lin.bias.copy_(bias.detach().to(out_dtype))
+                setattr(module, name, lin)
+    if hasattr(model, "_gbra_config"):
+        del model._gbra_config
+    return model
+
+
+# Turkish aliases
+gbra_ekle = apply_gbra
+tum_normlari_yenile = refresh_norms
+gbra_kaydet = save_gbra
+gbra_yukle = load_gbra
+gbra_birlestir = merge_gbra

gbra-0.1.0.dist-info/METADATA

@@ -0,0 +1,102 @@
+Metadata-Version: 2.4
+Name: gbra
+Version: 0.1.0
+Summary: GBRA (Gember-Berke Rank Adaptation): DoRA-style fine-tuning with a buffered, lazily refreshed weight norm.
+Author: Berke T. Adanc
+License: MIT
+Keywords: lora,dora,peft,fine-tuning,llm
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.0
+Requires-Dist: safetensors>=0.4
+Provides-Extra: hf
+Requires-Dist: transformers>=4.40; extra == "hf"
+Provides-Extra: 4bit
+Requires-Dist: bitsandbytes>=0.43; extra == "4bit"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: transformers>=4.40; extra == "test"
+Dynamic: license-file
+
+# GBRA: Gember-Berke Rank Adaptation
+
+*Pronounced "Gember RA". A GEMBER && Apex Logiç Studio project by Berke T. Adanç.*
+
+GBRA is a DoRA-style parameter-efficient fine-tuning method. It keeps DoRA's split of each weight into a learned **magnitude** and a low-rank-updated **direction**, but **buffers the weight norm** instead of recomputing it in every forward pass.
+
+```
+DoRA:  W' = m · (W + s·BA) / ‖W + s·BA‖     ← norm recomputed every forward (and again during gradient-checkpoint recompute)
+GBRA:  W' = m · (W + s·BA) / n               ← n is a stored buffer, refreshed every K optimizer steps
+```
+
+**Why it helps:** With gradient accumulation, the weights only change once per optimizer step. Recomputing the norm for every micro-batch repeats the same work many times. GBRA computes it once and reuses it.
+
+## Install
+
+```bash
+pip install -e .            # from this folder
+pip install -e ".[hf]"      # + Hugging Face Trainer callback
+pip install -e ".[4bit]"    # + 4-bit (bitsandbytes) base models
+```
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, Trainer
+from gbra import apply_gbra, GBRACallback, save_gbra, merge_gbra
+
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B-Instruct")
+model = apply_gbra(model, r=16, alpha=16)          # wraps q/k/v/o/gate/up/down projections
+
+trainer = Trainer(model=model, ..., callbacks=[GBRACallback(refresh_every=10)])
+trainer.train()
+
+save_gbra(model, "my_adapter")                     # small adapter (A, B, m, n)
+merge_gbra(model)                                  # fold into plain nn.Linear for export
+```
+
+Turkish aliases are also available: `gbra_ekle`, `tum_normlari_yenile`, `gbra_kaydet`, `gbra_yukle`, `gbra_birlestir`.
+
+## Preliminary results
+
+Qwen2.5-0.5B-Instruct, 3,000 Turkish chat/tool-use examples, 375 optimizer steps (batch 1 × 8 accumulation), r=16, single Kaggle T4, fp16 autocast. Same data, seed and hyper-parameters for every method. DoRA and LoRA via 🤗 PEFT.
+
+| Method | s / step | vs LoRA | Peak memory | Val. loss |
+|---|---|---|---|---|
+| LoRA | **5.77** | 1.00× | 8.11 GB | 1.6376 |
+| DoRA (PEFT) | 9.10 | 1.58× | 8.11 GB | 1.6285 |
+| **GBRA K=10** | 6.47 | 1.12× | 8.11 GB | **1.6249** |
+| GBRA K=100 | 6.47 | 1.12× | 8.11 GB | 1.6253 |
+| GBRA K=never | 6.46 | 1.12× | 8.11 GB | 1.6263 |
+
+**In this run, GBRA was ~29% faster than DoRA per step, with equal or slightly lower validation loss.** A shorter 125-step run gave the same speed ratio.
+
+### Limitations (please read)
+
+- **Single seed, single small model, single dataset.** The loss differences between GBRA variants and DoRA are small (≈0.001–0.004) and may be noise; repeated runs with more seeds are in progress.
+- Much of the speed-up comes from **not recomputing the norm per micro-batch**. With no gradient accumulation and no gradient checkpointing, the gap to DoRA should be smaller.
+- The comparison is against PEFT's DoRA implementation at the time of testing, not against every possible DoRA implementation.
+- 4-bit (bitsandbytes) support is implemented but **has not yet been benchmarked**.
+- Unlike PEFT LoRA, GBRA does not currently use fused kernels such as Unsloth's.
+
+## Credits
+
+GBRA builds directly on **DoRA** (Liu et al., 2024, *"DoRA: Weight-Decomposed Low-Rank Adaptation"*) and **LoRA** (Hu et al., 2021). Related ideas on learned scaling vectors appear in (IA)³ and VeRA. The contribution here is the buffered / lazily refreshed norm and its evaluation.
+
+The core `forward` of `GBRALinear` was written by Berke T. Adanç. Parts of the surrounding library and tests were developed with the help of an AI assistant.
+
+## License
+
+MIT
+
+---
+
+## Türkçe özet
+
+GBRA, DoRA'nın "büyüklük + yön" ayrımını korur, ama ağırlık normunu her adımda yeniden hesaplamak yerine **saklar** ve sadece **K optimizer adımında bir** günceller. Gradient birikimi kullanılan eğitimde ağırlıklar mini-batch'ler arasında değişmediği için DoRA aynı normu tekrar tekrar hesaplar. GBRA bu tekrarı ortadan kaldırır.
+
+İlk deneyde (Qwen2.5-0.5B, 375 adım, tek seed) GBRA, DoRA'dan **~%29 daha hızlı** ve doğrulama loss'u aynı ya da biraz daha iyiydi. Sonuçlar tek seed'e dayandığı için **ön sonuç** olarak değerlendirilmelidir.

gbra-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+gbra/__init__.py,sha256=dj5Ib35X7o9McgZ8bySFZd1AXUmeyWlaUZzh30GbacY,926
+gbra/callback.py,sha256=EwxnSEHOnss0zmppHheOBtK2J0cmtQqJUnXh2yIvlpo,803
+gbra/layer.py,sha256=L3rG-DR6d43PC9oE4cSh6wqktjOFVuSO0RrjpJNv7gM,3640
+gbra/model.py,sha256=k3UR3QyrLEZsBqKmWsZdY1hyOrPpEO1XBMPF2npfclI,5126
+gbra-0.1.0.dist-info/licenses/LICENSE,sha256=AJaZTTeRPlE_b0EgsutlchTPFn-JXY8Sdeg06uVmYXM,1101
+gbra-0.1.0.dist-info/METADATA,sha256=ADTPdkZ5eGp7wsrWYgdw4tTkX5GS9hEU82Fk34UN6tE,5194
+gbra-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+gbra-0.1.0.dist-info/top_level.txt,sha256=YS-1qm9iN_uHTzfqaocQiQ21mXfMWPLD-ZYDuCbSFQo,5
+gbra-0.1.0.dist-info/RECORD,,

gbra-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

gbra-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Berke T. Adanc (GEMBER && Apex Logic Studio)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gbra-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+gbra