hakuli 1.0.0__py3-none-any.whl

hakuli/__init__.py

@@ -0,0 +1,9 @@
+"""Hakuli: memory-efficient optimizer for full-parameter LLM training.
+
+Developed by XAEDRO.
+"""
+
+from hakuli.optim import Hakuli
+
+__all__ = ["Hakuli"]
+__version__ = "1.0.0"

hakuli/optim.py

@@ -0,0 +1,246 @@
+"""Core Hakuli optimizer implementation.
+
+Hakuli is an independent, clean-room implementation developed by XAEDRO,
+written from the published algorithmic description of low-rank-moment
+optimizers. No source code from any reference implementation is included.
+See the NOTICE file at the repository root for academic attributions.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Callable, Iterable, Optional
+
+import torch
+from torch.optim import Optimizer
+
+__all__ = ["Hakuli"]
+
+
+class Hakuli(Optimizer):
+    r"""Memory-efficient optimizer for full-parameter LLM training.
+
+    Hakuli keeps Adam-style first- and second-moment estimates in a low-rank
+    auxiliary subspace built from a Gaussian random projection of the
+    gradient, then applies a channel-wise (or tensor-wise) rescaling of the
+    *full* gradient. Optimizer memory is therefore close to SGD while the
+    weights receive full-rank updates. No singular value decomposition is
+    performed at any point.
+
+    For a 2-D parameter :math:`G_t \in \mathbb{R}^{m \times n}` the update is
+
+    .. math::
+
+        R_t &= P_t^{\top} G_t \\
+        M_t &= \beta_1 M_{t-1} + (1 - \beta_1) R_t \\
+        V_t &= \beta_2 V_{t-1} + (1 - \beta_2) R_t^2 \\
+        S_t &= \frac{M_t}{\sqrt{V_t} + \epsilon} \\
+        s_t &= \frac{\lVert S_t \rVert}{\lVert R_t \rVert}
+               \quad \text{(per output channel)} \\
+        \hat{G}_t &= \sqrt{\lambda}\, s_t \odot G_t \\
+        W_t &= W_{t-1} - \eta_t \hat{U}_t,
+        \qquad
+        \eta_t = \eta \frac{\sqrt{1 - \beta_2^t}}{1 - \beta_1^t}
+
+    where :math:`P_t \in \mathbb{R}^{m \times r}` is resampled from a standard
+    Gaussian every ``update_proj_gap`` steps, :math:`r` is ``rank`` and
+    :math:`\lambda` is ``scale``. When the parameter is wider than it is tall
+    the projection is applied on the right and channel norms are taken along
+    rows. Parameters whose smallest dimension does not exceed ``rank``
+    (including all 1-D parameters) are updated with the bias-corrected
+    Adam-style direction :math:`S_t` directly.
+
+    A relative norm-growth limiter (``nl_threshold``) caps the update norm at
+    ``nl_threshold`` times the previous step's update norm, converting abrupt
+    gradient spikes into smooth growth.
+
+    Args:
+        params: Iterable of parameters or parameter-group dictionaries. The
+            hyperparameters below can be overridden per group.
+        lr: Learning rate (default: ``1e-5``).
+        rank: Rank of the auxiliary random subspace (default: ``256``).
+        update_proj_gap: Steps between projection resamples (default: ``200``).
+        scale: Global multiplier on the structured update, applied as
+            ``sqrt(scale)`` (default: ``1.0``).
+        scale_type: ``"channel"`` for per-channel rescaling or ``"tensor"``
+            for a single scalar per tensor (default: ``"channel"``).
+        betas: Moment decay coefficients (default: ``(0.9, 0.999)``).
+        eps: Numerical stability term (default: ``1e-8``).
+        weight_decay: Decoupled weight decay (default: ``0.0``).
+        nl_threshold: Relative cap on update-norm growth between consecutive
+            steps (default: ``1.01``).
+
+    Example:
+        >>> import torch
+        >>> import torch.nn as nn
+        >>> from hakuli import Hakuli
+        >>> model = nn.Linear(128, 128)
+        >>> optimizer = Hakuli(model.parameters(), lr=1e-4, rank=64)
+        >>> optimizer.zero_grad()
+        >>> loss = model(torch.randn(8, 128)).pow(2).mean()
+        >>> loss.backward()
+        >>> optimizer.step()
+
+    Note:
+        Hakuli was designed for fine-tuning large transformers (attention and
+        MLP projection matrices, embeddings). Route 1-D parameters and
+        normalization layers to the same optimizer or any standard optimizer;
+        Hakuli handles both cases transparently.
+    """
+
+    def __init__(
+        self,
+        params: Iterable[torch.Tensor],
+        lr: float = 1e-5,
+        rank: int = 256,
+        update_proj_gap: int = 200,
+        scale: float = 1.0,
+        scale_type: str = "channel",
+        betas: tuple = (0.9, 0.999),
+        eps: float = 1e-8,
+        weight_decay: float = 0.0,
+        nl_threshold: float = 1.01,
+    ) -> None:
+        if lr < 0.0:
+            raise ValueError(f"Invalid learning rate: {lr}")
+        if rank < 0:
+            raise ValueError(f"Invalid rank: {rank}")
+        if update_proj_gap < 1:
+            raise ValueError(f"Invalid update_proj_gap: {update_proj_gap}")
+        if scale < 0.0:
+            raise ValueError(f"Invalid scale: {scale}")
+        if scale_type not in ("channel", "tensor"):
+            raise ValueError(
+                f"Invalid scale_type: {scale_type!r} (expected 'channel' or 'tensor')"
+            )
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError(f"Invalid beta1: {betas[0]}")
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError(f"Invalid beta2: {betas[1]}")
+        if eps < 0.0:
+            raise ValueError(f"Invalid epsilon: {eps}")
+        if weight_decay < 0.0:
+            raise ValueError(f"Invalid weight_decay: {weight_decay}")
+        if nl_threshold <= 0.0:
+            raise ValueError(f"Invalid nl_threshold: {nl_threshold}")
+
+        defaults = dict(
+            lr=lr,
+            rank=rank,
+            update_proj_gap=update_proj_gap,
+            scale=scale,
+            scale_type=scale_type,
+            betas=betas,
+            eps=eps,
+            weight_decay=weight_decay,
+            nl_threshold=nl_threshold,
+        )
+        super().__init__(params, defaults)
+
+    @torch.no_grad()
+    def step(self, closure: Optional[Callable[[], torch.Tensor]] = None) -> Optional[torch.Tensor]:
+        """Perform a single optimization step.
+
+        Args:
+            closure: Optional callable that re-evaluates the model and
+                returns the loss. Executed under ``torch.enable_grad()``.
+
+        Returns:
+            The loss returned by ``closure``, if provided.
+        """
+        loss = None
+        if closure is not None:
+            with torch.enable_grad():
+                loss = closure()
+
+        for group in self.param_groups:
+            beta1, beta2 = group["betas"]
+            rank = group["rank"]
+            eps = group["eps"]
+            lr = group["lr"]
+            scale = group["scale"]
+            weight_decay = group["weight_decay"]
+            scale_type = group["scale_type"]
+            update_proj_gap = group["update_proj_gap"]
+            nl_threshold = group["nl_threshold"]
+
+            for p in group["params"]:
+                grad = p.grad
+                if grad is None:
+                    continue
+                if grad.is_sparse:
+                    raise RuntimeError("Hakuli does not support sparse gradients")
+
+                state = self.state[p]
+                state["step"] = state.get("step", 0) + 1
+                t = state["step"]
+
+                projected = grad.dim() == 2 and rank > 0 and min(grad.shape) > rank
+
+                if projected:
+                    rows, cols = grad.shape
+                    if rows >= cols:
+                        if "proj" not in state or t % update_proj_gap == 0:
+                            state["proj"] = torch.randn(
+                                rows, rank, device=grad.device, dtype=torch.bfloat16
+                            )
+                        low = torch.mm(state["proj"].t().to(grad.dtype), grad).float()
+                        channel_dim = 0
+                    else:
+                        if "proj" not in state or t % update_proj_gap == 0:
+                            state["proj"] = torch.randn(
+                                cols, rank, device=grad.device, dtype=torch.bfloat16
+                            )
+                        low = torch.mm(grad, state["proj"].to(grad.dtype)).float()
+                        channel_dim = 1
+
+                    exp_avg = state.get("exp_avg")
+                    if exp_avg is None:
+                        exp_avg = torch.zeros_like(low)
+                        exp_avg_sq = torch.zeros_like(low)
+                    else:
+                        exp_avg_sq = state["exp_avg_sq"]
+                    exp_avg.mul_(beta1).add_(low, alpha=1.0 - beta1)
+                    exp_avg_sq.mul_(beta2).addcmul_(low, low, value=1.0 - beta2)
+                    state["exp_avg"], state["exp_avg_sq"] = exp_avg, exp_avg_sq
+
+                    norm_grad = exp_avg / (exp_avg_sq.sqrt() + eps)
+                    if scale_type == "channel":
+                        numer = norm_grad.norm(dim=channel_dim)
+                        denom = low.norm(dim=channel_dim).clamp(min=1e-8)
+                        scaling = numer / denom
+                        if channel_dim == 1:
+                            scaling = scaling.unsqueeze(1)
+                    else:
+                        scaling = norm_grad.norm() / low.norm().clamp(min=1e-8)
+                    update = grad * scaling.to(grad.dtype)
+                else:
+                    exp_avg = state.get("exp_avg")
+                    if exp_avg is None:
+                        exp_avg = torch.zeros_like(grad)
+                        exp_avg_sq = torch.zeros_like(grad)
+                    else:
+                        exp_avg_sq = state["exp_avg_sq"]
+                    exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1)
+                    exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2)
+                    state["exp_avg"], state["exp_avg_sq"] = exp_avg, exp_avg_sq
+
+                    update = exp_avg / (exp_avg_sq.sqrt() + eps)
+
+                update = update * math.sqrt(max(scale, 0.0))
+
+                update_norm = update.norm().item()
+                prev_norm = state.get("prev_norm")
+                if prev_norm is not None and prev_norm > 0.0:
+                    limiter = max(update_norm / (prev_norm + 1e-8), nl_threshold) / nl_threshold
+                    update = update / limiter
+                    state["prev_norm"] = update_norm / limiter
+                else:
+                    state["prev_norm"] = update_norm
+
+                step_size = lr * math.sqrt(1.0 - beta2**t) / max(1.0 - beta1**t, 1e-16)
+                if weight_decay > 0:
+                    p.mul_(1.0 - lr * weight_decay)
+                p.add_(update, alpha=-step_size)
+
+        return loss

hakuli-1.0.0.dist-info/METADATA

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: hakuli
+Version: 1.0.0
+Summary: Hakuli: memory-efficient optimizer for full-parameter LLM training without SVD.
+Author: XAEDRO
+Author-email: Evaristo Cordoba Ancajima <research@xaedro.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/xaedro-ai/hakuli
+Project-URL: Repository, https://github.com/xaedro-ai/hakuli
+Keywords: llm,optimizer,memory-efficient,fine-tuning,training,low-rank
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: torch>=2.1
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Dynamic: license-file
+
+# Hakuli
+
+**Memory-efficient optimizer for full-parameter LLM training: no SVD, low-rank optimizer state, PyTorch-only.**
+
+Hakuli is a memory-efficient optimizer for training and fine-tuning large
+language models on a single consumer GPU. It performs full-parameter (not
+adapter-based) updates while keeping optimizer memory close to SGD, by
+approximating channel-wise gradient scaling with an auxiliary low-rank state
+built from pure random projections. Hakuli performs **no SVD**, not even
+periodically, which removes the periodic SVD stalls of GaLore-style
+projection methods.
+
+Hakuli is developed by XAEDRO and is used in the company's internal LLM
+training pipeline.
+
+Hakuli is an independent, clean-room implementation of the APOLLO optimization
+method (Zhu et al., MLSys 2025). The algorithm follows the published
+specification; no source code from the reference implementation was used.
+Hakuli is not affiliated with, or endorsed by, the APOLLO authors. See
+[`NOTICE`](NOTICE) for academic attributions.
+
+[![CI](https://github.com/xaedro-ai/hakuli/actions/workflows/ci.yml/badge.svg)](https://github.com/xaedro-ai/hakuli/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/hakuli.svg)](https://pypi.org/project/hakuli/)
+[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.9%2B-blue.svg)](pyproject.toml)
+[![PyTorch](https://img.shields.io/badge/pytorch-2.1%2B-ee4c2c.svg)](https://pytorch.org)
+
+---
+
+## Why Hakuli
+
+| | AdamW | GaLore | **Hakuli** |
+|---|---|---|---|
+| Full-parameter training | Yes | Yes | Yes |
+| Optimizer memory (7B, rank 256) | ≈54 GB (LLaMA-7B) | ~9 GB | **~2.4 GB** |
+| Periodic SVD | None | Every `gap` steps | **None** |
+| Dependency footprint | torch | galore-torch | **torch only** |
+
+- **No SVD anywhere.** Projection matrices are Gaussian random and resampled on
+  a fixed cadence. There is no `O(mn²)` decomposition, no rebuild stalls.
+- **Low-rank auxiliary state.** Moments are tracked in a `rank × n` subspace;
+  the update is scaled channel-wise from that state and applied to the full
+  gradient. The weights remain full-rank.
+- **Norm-growth limiter.** A relative update-norm limiter (`nl_threshold`,
+  default `1.01`) caps update-norm growth between consecutive steps,
+  converting abrupt gradient spikes into smooth increases.
+- **Minimal API.** A single `torch.optim.Optimizer` subclass. Works with
+  plain PyTorch training loops, gradient accumulation, and per-layer setups.
+
+## Installation
+
+Hakuli is dependency-light (PyTorch only). Install from PyPI:
+
+```bash
+pip install hakuli
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/xaedro-ai/hakuli.git
+cd hakuli
+pip install -e .
+```
+
+Requires Python 3.9+ and PyTorch 2.1+. CUDA is recommended but not required.
+
+## Quick start
+
+```python
+import torch
+import torch.nn as nn
+from hakuli import Hakuli
+
+model = nn.Sequential(nn.Linear(512, 2048), nn.GELU(), nn.Linear(2048, 512)).cuda()
+
+# Hakuli routes 2-D weights through the low-rank path; parameters whose
+# smallest dimension is not larger than `rank` (including 1-D parameters)
+# automatically use its Adam-style fallback.
+lowrank, regular = [], []
+for name, p in model.named_parameters():
+    (lowrank if p.ndim == 2 else regular).append(p)
+
+optimizer = Hakuli(
+    [
+        {"params": regular},
+        {"params": lowrank, "rank": 256, "scale": 1.0,
+         "scale_type": "channel", "update_proj_gap": 200},
+    ],
+    lr=1e-5,
+)
+
+for step, (x, y) in enumerate(loader):
+    optimizer.zero_grad()
+    loss = criterion(model(x), y)
+    loss.backward()
+    optimizer.step()
+```
+
+A complete runnable example (small model, CPU) is provided in
+[`examples/train_tiny.py`](examples/train_tiny.py).
+
+### Per-parameter (layer-wise) setups
+
+Hakuli is a standard optimizer and can be instantiated per parameter. This is
+the recommended pattern when gradients must be released eagerly:
+
+```python
+optimizers = {id(p): Hakuli([p], lr=1e-5, rank=256) for p in lowrank_params}
+
+def hook(p):
+    if p.grad is None:
+        return
+    opt = optimizers[id(p)]
+    opt.step()
+    opt.zero_grad(set_to_none=True)
+
+for p in lowrank_params:
+    p.register_post_accumulate_grad_hook(hook)
+```
+
+## Configuration
+
+| Argument | Default | Description |
+|---|---|---|
+| `lr` | `1e-5` | Learning rate. The APOLLO-style scale-corrected step size is applied internally; no external scheduler is required. |
+| `rank` | `256` | Rank of the auxiliary random subspace. 256 is a good default for 1B–7B models. |
+| `scale` | `1.0` | Global multiplier on the structured update (applied under a square root). Tune together with `rank`. |
+| `scale_type` | `"channel"` | `"channel"` (APOLLO-style) or `"tensor"` (extreme low-memory). |
+| `update_proj_gap` | `200` | Steps between random-projection resamples. |
+| `betas` | `(0.9, 0.999)` | Moment decay coefficients. |
+| `eps` | `1e-8` | Numerical stability term. |
+| `weight_decay` | `0.0` | Decoupled weight decay. |
+| `nl_threshold` | `1.01` | Relative cap on update-norm growth between steps. |
+
+## Recommended usage notes
+
+- Apply Hakuli to attention and MLP projection matrices plus embeddings; keep
+  normalization layers and biases in a conventional optimizer.
+- `rank` 256 with `scale` ~14 (`hidden_size / rank` rule of thumb) is a good
+  starting point for 7B-scale fine-tuning.
+- With eager gradient release (per-parameter hooks), peak memory is dominated
+  by activations, not optimizer state; combine with activation checkpointing
+  for long-context training.
+- Hakuli is orthogonal to precision schemes: it operates on whatever gradient
+  dtype your model produces (`bf16` recommended).
+
+## Documentation
+
+- [`paper/hakuli-technical-report.pdf`](paper/hakuli-technical-report.pdf):
+  compiled technical report.
+- [`paper/`](paper/): report sources — LaTeX (`main.tex`, `references.bib`)
+  and Markdown (`paper.md`).
+- [`benchmarks/RESULTS.md`](benchmarks/RESULTS.md): measured memory and
+  throughput results.
+- [`docs/design.md`](docs/design.md): algorithm, update rule, memory model.
+- [`docs/faq.md`](docs/faq.md): batch sizes, rank selection, integrations.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest -q
+```
+
+## Citation
+
+If you use Hakuli in academic work, please cite the underlying research that
+inspired its design:
+
+```bibtex
+@inproceedings{zhu2025apollo,
+  title     = {{APOLLO}: {SGD}-like Memory, {AdamW}-level Performance},
+  author    = {Zhu, Hanqing and Zhang, Zhenyu and Cong, Wenyan and Liu, Xi and
+               Park, Sem and Chandra, Vikas and Long, Bo and Pan, David Z. and
+               Wang, Zhangyang and Lee, Jinwon},
+  booktitle = {Conference on Machine Learning and Systems (MLSys)},
+  year      = {2025}
+}
+
+@inproceedings{zhao2024galore,
+  title     = {{GaLore}: Memory-Efficient {LLM} Training by Gradient Low-Rank Projection},
+  author    = {Zhao, Jiawei and Zhang, Zhenyu and Chen, Beidi and Wang, Zhangyang and
+               Anandkumar, Anima and Tian, Yuandong},
+  booktitle = {International Conference on Machine Learning (ICML)},
+  year      = {2024}
+}
+```
+
+See [`CITATION.cff`](CITATION.cff) for machine-readable metadata.
+
+## License
+
+Apache License 2.0. See [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE).
+
+"Hakuli" and "XAEDRO" are trademarks of XAEDRO. Hakuli is an independent
+clean-room implementation; see the NOTICE file for academic attributions.

hakuli-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+hakuli/__init__.py,sha256=1i94VtwBlcuyxPOSkxXezg3dUf4nTxVqmRheHTq-HJk,174
+hakuli/optim.py,sha256=bdXwm14zlLEfD2gstrZE1ZzpAOhfrHi1VrkALx05VQE,10341
+hakuli-1.0.0.dist-info/licenses/LICENSE,sha256=Z072a2eA8Ta6Rc8xGDaQtQOJ5gTFdJwVdy1FwFk7St8,11336
+hakuli-1.0.0.dist-info/licenses/NOTICE,sha256=5UuSt4JDDxtHPrklyyfSCn7xNxvw7Nl-kqbaXsDBjSU,1403
+hakuli-1.0.0.dist-info/METADATA,sha256=4qXVAvWUpkjMnv3WCTbxg6T-2GVO0zjfjX8UJiU1j00,8651
+hakuli-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+hakuli-1.0.0.dist-info/top_level.txt,sha256=RPoyN5id62IFTq_4-ZNlHulldbo-YDdeb4XrCUl9_js,7
+hakuli-1.0.0.dist-info/RECORD,,

hakuli-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

hakuli-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2026 XAEDRO
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

hakuli-1.0.0.dist-info/licenses/NOTICE

@@ -0,0 +1,30 @@
+Hakuli
+Copyright 2026 XAEDRO
+
+This product is an independent, clean-room implementation of optimization
+concepts described in the following academic publications. No source code
+from these projects is included in this distribution:
+
+  - GaLore: Memory-Efficient LLM Training by Gradient Low-Rank Projection.
+    J. Zhao, Z. Zhang, B. Chen, Z. Wang, A. Anandkumar, Y. Tian.
+    ICML 2024. arXiv:2403.03507
+
+  - APOLLO: SGD-like Memory, AdamW-level Performance.
+    H. Zhu, Z. Zhang, W. Cong, X. Liu, S. Park, V. Chandra, B. Long,
+    D. Z. Pan, Z. Wang, J. Lee. MLSys 2025. arXiv:2412.05270
+
+  - Fira: Can We Achieve Full-Rank Training of LLMs Under Low-rank
+    Constraint? X. Chen, K. Feng, C. Li, X. Lai, X. Yue, Y. Yuan, G. Wang.
+    NeurIPS 2025. arXiv:2410.01623 (norm-growth limiter concept)
+
+  - Q-GaLore: Quantized GaLore with INT4 Projection and Layer-Adaptive
+    Low-Rank Gradients. Z. Zhang, A. Jaiswal, L. Yin, S. Liu, J. Zhao,
+    Y. Tian, Z. Wang. arXiv:2407.08296, 2024
+
+The citations above are provided for academic attribution of the underlying
+research ideas only. Hakuli is not affiliated with, endorsed by, or derived
+from the source code of any of these projects. Hakuli was implemented from
+published algorithmic descriptions and independently written source code.
+
+"XAEDRO" and "Hakuli" are trademarks of XAEDRO. All other trademarks are
+the property of their respective owners.

hakuli-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+hakuli