vlora-dev 0.2.0__py3-none-any.whl

vlora/__init__.py

@@ -0,0 +1,73 @@
+"""vLoRA — Shared low-rank subspaces for LoRA adapter management.
+
+Based on the Share paper (arXiv:2602.06043): LoRA adapters across tasks
+share a common low-rank subspace. Instead of storing N separate adapters,
+maintain one shared basis and per-task coefficient vectors.
+"""
+
+__version__ = "0.1.0"
+
+from vlora.io import LoRAWeights, load_adapter, load_adapter_from_hub, save_adapter
+from vlora.ops import (
+    compute_svd,
+    explained_variance_ratio,
+    gram_schmidt,
+    project_onto_subspace,
+    reconstruct_from_subspace,
+    select_num_components,
+)
+from vlora.model import VLoRAModel
+from vlora.ops import incremental_svd_update
+from vlora.analysis import (
+    adapter_diff,
+    compute_similarity_matrix,
+    find_clusters,
+    find_outliers,
+    subspace_coverage,
+)
+from vlora.pipeline import absorb_task, extract_adapter, init_subspace
+from vlora.router import TaskRouter
+from vlora.subspace import SharedSubspace, TaskProjection
+from vlora.training import SubspaceTrainer, orthogonal_init
+from vlora.merge import task_arithmetic, ties_merge, dare_merge
+
+__all__ = [
+    # Core
+    "SharedSubspace",
+    "TaskProjection",
+    "LoRAWeights",
+    # I/O
+    "load_adapter",
+    "load_adapter_from_hub",
+    "save_adapter",
+    # Pipeline
+    "init_subspace",
+    "absorb_task",
+    "extract_adapter",
+    # Ops
+    "compute_svd",
+    "project_onto_subspace",
+    "reconstruct_from_subspace",
+    "gram_schmidt",
+    "explained_variance_ratio",
+    "select_num_components",
+    # Analysis
+    "compute_similarity_matrix",
+    "find_clusters",
+    "adapter_diff",
+    "subspace_coverage",
+    "find_outliers",
+    # Model
+    "VLoRAModel",
+    # Router
+    "TaskRouter",
+    # Training
+    "SubspaceTrainer",
+    "orthogonal_init",
+    # Incremental
+    "incremental_svd_update",
+    # Merging
+    "task_arithmetic",
+    "ties_merge",
+    "dare_merge",
+]

vlora/_validate.py

@@ -0,0 +1,82 @@
+"""Input validation helpers for vlora public APIs."""
+
+from __future__ import annotations
+
+import logging
+import warnings
+from typing import TYPE_CHECKING
+
+import torch
+from torch import Tensor
+
+if TYPE_CHECKING:
+    from vlora.io import LoRAWeights
+    from vlora.subspace import SharedSubspace
+
+logger = logging.getLogger("vlora")
+
+
+def check_adapters_compatible(adapters: list[LoRAWeights]) -> None:
+    """Validate that adapters can be used together (same rank, overlapping layers)."""
+    if not adapters:
+        raise ValueError("Need at least one adapter.")
+
+    ranks = {a.rank for a in adapters}
+    if len(ranks) > 1:
+        raise ValueError(
+            f"Adapters have inconsistent ranks: {sorted(ranks)}. "
+            "All adapters must have the same LoRA rank to share a subspace."
+        )
+
+
+def check_adapter_matches_subspace(
+    adapter: LoRAWeights, subspace: SharedSubspace, operation: str = "operation"
+) -> None:
+    """Validate that an adapter is compatible with a subspace."""
+    if adapter.rank != subspace.rank:
+        raise ValueError(
+            f"Cannot {operation}: adapter rank ({adapter.rank}) does not match "
+            f"subspace rank ({subspace.rank})."
+        )
+
+    common = set(adapter.layer_names) & set(subspace.layer_names)
+    if not common:
+        raise ValueError(
+            f"Cannot {operation}: adapter and subspace share no common layers. "
+            f"Adapter layers: {adapter.layer_names[:3]}... "
+            f"Subspace layers: {subspace.layer_names[:3]}..."
+        )
+
+    missing = set(subspace.layer_names) - set(adapter.layer_names)
+    if missing:
+        warnings.warn(
+            f"Adapter is missing {len(missing)} subspace layers "
+            f"(e.g. {sorted(missing)[:2]}). These will use mean values.",
+            stacklevel=3,
+        )
+
+
+def check_task_exists(subspace: SharedSubspace, task_id: str) -> None:
+    """Raise KeyError with helpful message if task not found."""
+    if task_id not in subspace.tasks:
+        available = ", ".join(sorted(subspace.tasks.keys()))
+        raise KeyError(
+            f"Unknown task '{task_id}'. "
+            f"Available tasks: [{available}]. "
+            f"Use subspace.tasks.keys() to list all tasks."
+        )
+
+
+def check_tensor_health(tensor: Tensor, name: str = "tensor") -> None:
+    """Check for NaN/Inf in a tensor."""
+    if torch.isnan(tensor).any():
+        raise ValueError(
+            f"NaN detected in {name}. This usually indicates numerical "
+            "instability during SVD. Try using fewer components or "
+            "checking your input adapters for degenerate weights."
+        )
+    if torch.isinf(tensor).any():
+        raise ValueError(
+            f"Inf detected in {name}. This usually indicates overflow "
+            "during computation. Try using float32 precision."
+        )

vlora/analysis.py

@@ -0,0 +1,191 @@
+"""Adapter analysis — similarity, clustering, diffing, and coverage."""
+
+from __future__ import annotations
+
+import torch
+from torch import Tensor
+
+from vlora.io import LoRAWeights
+from vlora.ops import project_onto_subspace
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from vlora.subspace import SharedSubspace
+
+
+def _flatten_adapter(adapter: LoRAWeights) -> Tensor:
+    """Flatten all A and B weights into a single 1D vector."""
+    parts = []
+    for layer in adapter.layer_names:
+        parts.append(adapter.lora_a[layer].flatten())
+        parts.append(adapter.lora_b[layer].flatten())
+    return torch.cat(parts)
+
+
+def compute_similarity_matrix(adapters: list[LoRAWeights]) -> Tensor:
+    """Compute pairwise cosine similarity between adapters.
+
+    Returns:
+        (N, N) similarity matrix where entry [i,j] is the cosine
+        similarity between adapter i and adapter j.
+    """
+    if len(adapters) < 2:
+        raise ValueError("Need at least 2 adapters for similarity")
+
+    vectors = torch.stack([_flatten_adapter(a) for a in adapters])
+    # Normalize rows
+    norms = vectors.norm(dim=1, keepdim=True).clamp(min=1e-8)
+    normalized = vectors / norms
+    return normalized @ normalized.T
+
+
+def find_clusters(
+    similarity_matrix: Tensor,
+    threshold: float = 0.9,
+) -> list[list[int]]:
+    """Group adapters into clusters based on similarity threshold.
+
+    Uses simple greedy clustering: iterate through adapters, assign each
+    to the first cluster where similarity to all members >= threshold,
+    or create a new cluster.
+
+    Returns:
+        List of clusters, where each cluster is a list of adapter indices.
+    """
+    n = similarity_matrix.shape[0]
+    clusters: list[list[int]] = []
+
+    for i in range(n):
+        placed = False
+        for cluster in clusters:
+            # Check similarity to all members
+            if all(similarity_matrix[i, j].item() >= threshold for j in cluster):
+                cluster.append(i)
+                placed = True
+                break
+        if not placed:
+            clusters.append([i])
+
+    return clusters
+
+
+def adapter_diff(
+    adapter_a: LoRAWeights,
+    adapter_b: LoRAWeights,
+) -> dict[str, dict[str, float]]:
+    """Per-layer comparison of two adapters.
+
+    Returns:
+        Dict mapping layer_name -> {"l2_distance": float, "cosine_sim": float}
+        for each layer present in both adapters.
+    """
+    common_layers = sorted(set(adapter_a.layer_names) & set(adapter_b.layer_names))
+    result: dict[str, dict[str, float]] = {}
+
+    for layer in common_layers:
+        vec_a = torch.cat([
+            adapter_a.lora_a[layer].flatten(),
+            adapter_a.lora_b[layer].flatten(),
+        ])
+        vec_b = torch.cat([
+            adapter_b.lora_a[layer].flatten(),
+            adapter_b.lora_b[layer].flatten(),
+        ])
+
+        l2 = (vec_a - vec_b).norm().item()
+        cos = torch.nn.functional.cosine_similarity(
+            vec_a.unsqueeze(0), vec_b.unsqueeze(0)
+        ).item()
+
+        result[layer] = {"l2_distance": l2, "cosine_sim": cos}
+
+    return result
+
+
+def subspace_coverage(
+    subspace: SharedSubspace,
+    adapter: LoRAWeights,
+) -> dict[str, dict[str, float]]:
+    """Measure how well a subspace represents a given adapter.
+
+    For each layer and side (A/B), projects the adapter onto the subspace
+    and measures the fraction of the adapter's norm that is captured.
+
+    Returns:
+        Dict mapping layer_name -> {"coverage_a": float, "coverage_b": float,
+        "coverage_mean": float}
+    """
+    result: dict[str, dict[str, float]] = {}
+
+    for layer in subspace.layer_names:
+        if layer not in adapter.lora_a:
+            continue
+
+        coverages = {}
+        for side, weights_dict, components, means in [
+            ("a", adapter.lora_a, subspace.components_a, subspace.means_a),
+            ("b", adapter.lora_b, subspace.components_b, subspace.means_b),
+        ]:
+            flat = weights_dict[layer].flatten() - means[layer]
+            original_norm = flat.norm().item()
+            if original_norm < 1e-8:
+                coverages[f"coverage_{side}"] = 1.0
+                continue
+
+            loadings = project_onto_subspace(flat, components[layer])
+            reconstructed = loadings @ components[layer]
+            residual_norm = (flat - reconstructed).norm().item()
+            coverages[f"coverage_{side}"] = 1.0 - (residual_norm / original_norm)
+
+        coverages["coverage_mean"] = (
+            coverages["coverage_a"] + coverages["coverage_b"]
+        ) / 2.0
+        result[layer] = coverages
+
+    return result
+
+
+def find_outliers(
+    adapters: list[LoRAWeights],
+    threshold: float = 2.0,
+) -> list[dict]:
+    """Detect adapter outliers based on distance from the group mean.
+
+    Computes each adapter's flattened weight vector, measures the L2
+    distance from the group centroid, and flags adapters whose distance
+    exceeds `threshold` standard deviations above the mean distance.
+
+    Args:
+        adapters: List of adapters to analyze.
+        threshold: Number of standard deviations above mean distance
+            to consider an outlier. Default 2.0.
+
+    Returns:
+        List of dicts with keys: {"index", "distance", "z_score"} for
+        each outlier adapter.
+    """
+    if len(adapters) < 3:
+        return []
+
+    vectors = torch.stack([_flatten_adapter(a) for a in adapters])
+    centroid = vectors.mean(dim=0)
+    distances = (vectors - centroid).norm(dim=1)
+
+    mean_dist = distances.mean().item()
+    std_dist = distances.std().item()
+
+    if std_dist < 1e-8:
+        return []
+
+    outliers = []
+    for i in range(len(adapters)):
+        z = (distances[i].item() - mean_dist) / std_dist
+        if z > threshold:
+            outliers.append({
+                "index": i,
+                "distance": distances[i].item(),
+                "z_score": z,
+            })
+
+    return outliers