compressed-tensors 0.10.2a20250613__tar.gz → 0.10.2a20250617__tar.gz

{compressed_tensors-0.10.2a20250613/src/compressed_tensors.egg-info → compressed_tensors-0.10.2a20250617}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compressed-tensors
-Version: 0.10.2a20250613
+Version: 0.10.2a20250617
 Summary: Library for utilization of compressed safetensors of neural network models
 Home-page: https://github.com/neuralmagic/compressed-tensors
 Author: Neuralmagic, Inc.

{compressed_tensors-0.10.2a20250613 → compressed_tensors-0.10.2a20250617}/src/compressed_tensors/utils/offload.py

@@ -85,6 +85,7 @@ __all__ = [
     "delete_offload_module",
     "offloaded_dispatch",
     "disable_offloading",
+    "remove_dispatch",
 ]
 
 
@@ -206,9 +207,24 @@ def register_offload_parameter(
     has_onload = any(p.device != torch.device("meta") for p in module.parameters())
     module.register_parameter(name, parameter)
 
+    # do everything AlignDevicesHook.init_hook does
+    # https://github.com/huggingface/accelerate/blob/main/src/accelerate/hooks.py#L281
     if has_offloaded_params(module):
-        weights_map = module._hf_hook.weights_map
-        offload_to_weights_map(weights_map, name, parameter.data, offload_device)
+        hook: AlignDevicesHook = module._hf_hook
+        assert hook.weights_map is not None
+
+        # append to original_devices
+        hook.original_devices[name] = parameter.device
+
+        # append to weights map
+        offload_to_weights_map(hook.weights_map, name, parameter.data, offload_device)
+
+        # append to tied_params_map
+        offloaded = hook.weights_map[name]
+        if hook.tied_params_map is not None:
+            hook.tied_params_map[offloaded.data_ptr()] = {}  # (1)
+
+        # perform offloading
         if not has_onload:
             set_module_tensor_to_device(module, name, "meta")
 
@@ -422,7 +438,6 @@ def register_offload_module(base: torch.nn.Module, name: str, module: torch.nn.M
         hook: AlignDevicesHook = base._hf_hook
         assert hook.offload
         assert hook.weights_map is not None
-        assert hook.tied_params_map is not None
 
         # offloading kwargs for submodule
         place_submodules = False
@@ -437,7 +452,8 @@ def register_offload_module(base: torch.nn.Module, name: str, module: torch.nn.M
             module, include_buffers=offload_buffers, recurse=place_submodules
         ):
             offloaded = param.to(offload_device)
-            hook.tied_params_map[offloaded.data_ptr()] = {}  # (1)
+            if hook.tied_params_map is not None:
+                hook.tied_params_map[offloaded.data_ptr()] = {}  # (1)
             offload_to_weights_map(hook.weights_map, f"{name}.{param_name}", offloaded)
 
             # if the parent places submodules, offload here
@@ -465,9 +481,6 @@ def register_offload_module(base: torch.nn.Module, name: str, module: torch.nn.M
 
     base.register_module(name, module)
 
-    # (1): Since we cannot know which pointers are shared when we add parameters in an
-    # online way, assume that all pointers are shared. This comes at no runtime cost
-
 
 def delete_offload_module(base: torch.nn.Module, name: str):
     """
@@ -502,6 +515,9 @@ def offloaded_dispatch(
     if offload_device == "disk":
         raise NotImplementedError("Disk offloading is not currently supported")
 
+    # remove any existing hooks
+    remove_dispatch(module)
+
     # create weights map
     state_dict = module.state_dict()
     state_dict = {key: val.to(offload_device) for key, val in state_dict.items()}
@@ -523,6 +539,33 @@ def offloaded_dispatch(
         weights_map=weights_map,
         tied_params_map=tied_params_map,
     )
+
+    # when saving a model, `PretrainedModel.save_pretrained` will only
+    # onload weights if the following requirements are met
+    # if (
+    #     hasattr(self, "hf_device_map")
+    #     and len(set(self.hf_device_map.values())) > 1
+    #     and ("cpu" in self.hf_device_map.values()
+    #          or "disk" in self.hf_device_map.values())
+    # ):
+    # because this function always offloads, disregard actual devices and
+    # always use `cpu` and `cuda:0` to guarantee this condition passes
+    setattr(module, "hf_device_map", {"fake_offload": "cpu", "fake_exec": "cuda:0"})
+
+    return module
+
+
+def remove_dispatch(module: torch.nn.Module) -> torch.nn.Module:
+    """
+    Remove any existing dispatches from module
+
+    :param module: module which may be dispatched with hf hooks
+    :return: module without dispatch
+    """
+    remove_hook_from_module(module, recurse=True)
+    if hasattr(module, "hf_device_map"):
+        delattr(module, "hf_device_map")
+
     return module
 
 
@@ -551,7 +594,7 @@ def disable_offloading():
     # update any parameters which may have changed
     for module, (hook, offload) in onloaded_modules.items():
         hook.offload = offload
-        for name, param in module.named_parameters():
+        for name, param in module.named_parameters(recurse=False):
             update_offload_parameter(module, name, param.data)
         hook.post_forward(module, None)
 
@@ -623,3 +666,7 @@ def align_module_device(
 
     else:
         yield
+
+
+# (1): Since we cannot know which pointers are shared when we add parameters in an
+# online way, assume that all pointers are shared. This has virtually no runtime cost

{compressed_tensors-0.10.2a20250613 → compressed_tensors-0.10.2a20250617}/src/compressed_tensors/version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.2.a20250613'
+__version__ = version = '0.10.2.a20250617'
 __version_tuple__ = version_tuple = (0, 10, 2)

{compressed_tensors-0.10.2a20250613 → compressed_tensors-0.10.2a20250617/src/compressed_tensors.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compressed-tensors
-Version: 0.10.2a20250613
+Version: 0.10.2a20250617
 Summary: Library for utilization of compressed safetensors of neural network models
 Home-page: https://github.com/neuralmagic/compressed-tensors
 Author: Neuralmagic, Inc.

{compressed_tensors-0.10.2a20250613 → compressed_tensors-0.10.2a20250617}/tests/test_utils/test_offload.py

@@ -149,6 +149,47 @@ def test_register_offload_parameter():
     assert module.a.device == module.b.device == module.c.device == torch.device("meta")
 
 
+@requires_accelerate()
+@requires_gpu
+def test_register_offload_parameter_hook_replacement():
+    module = ExampleModule()
+    parameter_c = torch.nn.Parameter(torch.tensor(1.0, device="cuda"))
+    parameter_d = torch.nn.Parameter(torch.tensor(1.0, device="cpu"))
+
+    offloaded_dispatch(module, "cuda")
+    register_offload_parameter(module, "c", parameter_c)
+    register_offload_parameter(module, "d", parameter_d)
+
+    with disable_hf_hook(module):
+        assert module.a.device == torch.device("cpu")
+        assert module.b.device == torch.device("cpu")
+        assert module.c.device == torch.device("cuda:0")
+        assert module.d.device == torch.device("cpu")
+
+    assert module.a.device == torch.device("meta")
+    assert module.b.device == torch.device("meta")
+    assert module.c.device == torch.device("meta")
+    assert module.d.device == torch.device("meta")
+    assert module._hf_hook.weights_map["a"].device == torch.device("cpu")
+    assert module._hf_hook.weights_map["b"].device == torch.device("cpu")
+    assert module._hf_hook.weights_map["c"].device == torch.device("cpu")
+    assert module._hf_hook.weights_map["d"].device == torch.device("cpu")
+
+
+@requires_accelerate()
+@requires_gpu
+def test_register_offload_parameter_shared():
+    module = ExampleModule()
+    parameter = torch.nn.Parameter(torch.tensor(1.0))
+
+    offloaded_dispatch(module, "cuda")
+    register_offload_parameter(module, "c", parameter)
+    register_offload_parameter(module, "d", parameter)
+
+    with align_module_device(module):
+        assert module.c is module.d
+
+
 @requires_accelerate()
 def test_update_offload_parameter():
     from accelerate.hooks import attach_align_device_hook