compressed-tensors-nightly 0.3.3.20240529__py3-none-any.whl → 0.3.3.20240531__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compressed_tensors/compressors/int_quantized.py +3 -2
- compressed_tensors/compressors/pack_quantized.py +3 -2
- compressed_tensors/quantization/lifecycle/forward.py +13 -6
- {compressed_tensors_nightly-0.3.3.20240529.dist-info → compressed_tensors_nightly-0.3.3.20240531.dist-info}/METADATA +3 -3
- {compressed_tensors_nightly-0.3.3.20240529.dist-info → compressed_tensors_nightly-0.3.3.20240531.dist-info}/RECORD +8 -8
- {compressed_tensors_nightly-0.3.3.20240529.dist-info → compressed_tensors_nightly-0.3.3.20240531.dist-info}/LICENSE +0 -0
- {compressed_tensors_nightly-0.3.3.20240529.dist-info → compressed_tensors_nightly-0.3.3.20240531.dist-info}/WHEEL +0 -0
- {compressed_tensors_nightly-0.3.3.20240529.dist-info → compressed_tensors_nightly-0.3.3.20240531.dist-info}/top_level.txt +0 -0
@@ -57,13 +57,14 @@ class IntQuantizationCompressor(Compressor):
|
|
57
57
|
:return: compressed state dict
|
58
58
|
"""
|
59
59
|
compressed_dict = {}
|
60
|
+
weight_suffix = ".weight"
|
60
61
|
_LOGGER.debug(
|
61
62
|
f"Compressing model with {len(model_state)} parameterized layers..."
|
62
63
|
)
|
63
64
|
|
64
65
|
for name, value in tqdm(model_state.items(), desc="Compressing model"):
|
65
|
-
if name.endswith(
|
66
|
-
prefix = name
|
66
|
+
if name.endswith(weight_suffix):
|
67
|
+
prefix = name[: -(len(weight_suffix))]
|
67
68
|
scale = model_state.get(merge_names(prefix, "weight_scale"), None)
|
68
69
|
zp = model_state.get(merge_names(prefix, "weight_zero_point"), None)
|
69
70
|
if scale is not None and zp is not None:
|
@@ -62,13 +62,14 @@ class PackedQuantizationCompressor(Compressor):
|
|
62
62
|
:return: compressed state dict
|
63
63
|
"""
|
64
64
|
compressed_dict = {}
|
65
|
+
weight_suffix = ".weight"
|
65
66
|
_LOGGER.debug(
|
66
67
|
f"Compressing model with {len(model_state)} parameterized layers..."
|
67
68
|
)
|
68
69
|
|
69
70
|
for name, value in tqdm(model_state.items(), desc="Compressing model"):
|
70
|
-
if name.endswith(
|
71
|
-
prefix = name
|
71
|
+
if name.endswith(weight_suffix):
|
72
|
+
prefix = name[: -(len(weight_suffix))]
|
72
73
|
scale = model_state.get(merge_names(prefix, "weight_scale"), None)
|
73
74
|
zp = model_state.get(merge_names(prefix, "weight_zero_point"), None)
|
74
75
|
shape = torch.tensor(value.shape)
|
@@ -89,11 +89,17 @@ def dequantize(
|
|
89
89
|
if scale.ndim == 0:
|
90
90
|
args = QuantizationArgs(strategy=QuantizationStrategy.TENSOR)
|
91
91
|
elif scale.ndim == 2:
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
92
|
+
if scale.shape[1] == 1:
|
93
|
+
args = QuantizationArgs(strategy=QuantizationStrategy.CHANNEL)
|
94
|
+
else:
|
95
|
+
group_size = int(x_q.shape[1] / scale.shape[1])
|
96
|
+
args = QuantizationArgs(
|
97
|
+
strategy=QuantizationStrategy.GROUP, group_size=group_size
|
98
|
+
)
|
99
|
+
else:
|
100
|
+
raise ValueError(
|
101
|
+
f"Could not infer a quantization strategy from scale with {scale.ndim} "
|
102
|
+
"dimmensions. Expected 0-2 dimmensions."
|
97
103
|
)
|
98
104
|
return _process_quantization(
|
99
105
|
x=x_q,
|
@@ -152,7 +158,8 @@ def _process_quantization(
|
|
152
158
|
|
153
159
|
if args.strategy == QuantizationStrategy.GROUP:
|
154
160
|
|
155
|
-
if do_dequantize
|
161
|
+
if do_dequantize and not do_quantize:
|
162
|
+
# if dequantizing a quantized type infer the output type from the scale
|
156
163
|
output = torch.zeros_like(x, dtype=scale.dtype)
|
157
164
|
else:
|
158
165
|
output_dtype = dtype if dtype is not None else x.dtype
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: compressed-tensors-nightly
|
3
|
-
Version: 0.3.3.
|
3
|
+
Version: 0.3.3.20240531
|
4
4
|
Summary: Library for utilization of compressed safetensors of neural network models
|
5
5
|
Home-page: https://github.com/neuralmagic/compressed-tensors
|
6
6
|
Author: Neuralmagic, Inc.
|
@@ -89,7 +89,7 @@ from compressed_tensors import save_compressed_model, load_compressed, BitmaskCo
|
|
89
89
|
from transformers import AutoModelForCausalLM
|
90
90
|
|
91
91
|
model_name = "neuralmagic/llama2.c-stories110M-pruned50"
|
92
|
-
model = AutoModelForCausalLM.from_pretrained(model_name)
|
92
|
+
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")
|
93
93
|
|
94
94
|
original_state_dict = model.state_dict()
|
95
95
|
|
@@ -111,7 +111,7 @@ We can use compressed-tensors to run basic post training quantization (PTQ) and
|
|
111
111
|
|
112
112
|
```python
|
113
113
|
model_name = "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T"
|
114
|
-
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cuda:0")
|
114
|
+
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="cuda:0", torch_dtype="auto")
|
115
115
|
|
116
116
|
config = QuantizationConfig.parse_file("./examples/bit_packing/int4_config.json")
|
117
117
|
config.quantization_status = QuantizationStatus.CALIBRATION
|
@@ -5,9 +5,9 @@ compressed_tensors/compressors/__init__.py,sha256=3yyoNICHll3F4HS6Yu-cgNZpDhfuob
|
|
5
5
|
compressed_tensors/compressors/base.py,sha256=LWEgbpgTxzmoqQ7Xhq2OQszUgWoDtFuGCiV1Y8nlBGw,2134
|
6
6
|
compressed_tensors/compressors/dense.py,sha256=G_XHbvuENyupIKlXSITOQgvPkNkcMEOLcLWQr70V9EE,1257
|
7
7
|
compressed_tensors/compressors/helpers.py,sha256=k9avlkmeYj6vkOAvl-MgcixtP7ib24SCfhzZ-RusXfw,5403
|
8
|
-
compressed_tensors/compressors/int_quantized.py,sha256=
|
8
|
+
compressed_tensors/compressors/int_quantized.py,sha256=Ct2vCK0yoPm6vkIFlzDMGQ7m14xT1GyURsSwH9DP770,5242
|
9
9
|
compressed_tensors/compressors/model_compressor.py,sha256=gHD2VMbXkXaZiJu3ibOaWiYb4oJDz2hxX03wDuu1yhI,10481
|
10
|
-
compressed_tensors/compressors/pack_quantized.py,sha256=
|
10
|
+
compressed_tensors/compressors/pack_quantized.py,sha256=VPiLlgJlDgARrn7YmiQoLqUfxErKBfj54epMYWRsF8k,8451
|
11
11
|
compressed_tensors/compressors/sparse_bitmask.py,sha256=H9oZSTYI1oRCzAMbd4zThUnZd1h2rfs8DmA3tPcvuNE,8637
|
12
12
|
compressed_tensors/config/__init__.py,sha256=ZBqWn3r6ku1qfmlHHYp0mQueY0i7Pwhr9rbQk9dDlMc,704
|
13
13
|
compressed_tensors/config/base.py,sha256=grf5tDaLep8i2-W_p7H-fW9DOGXDi4Zz7su7zjs1Qqc,1454
|
@@ -21,7 +21,7 @@ compressed_tensors/quantization/lifecycle/__init__.py,sha256=ggRGWRqhCxCaTTDWRcg
|
|
21
21
|
compressed_tensors/quantization/lifecycle/apply.py,sha256=yLTDT1zkJp1Nti-aKZGOMW8-TELanF8dXiqDvAkVUQo,7984
|
22
22
|
compressed_tensors/quantization/lifecycle/calibration.py,sha256=mLns4jlaWmBwOW8Jtlm5bMX-JET1AiZYUBO7qa-XuxI,1776
|
23
23
|
compressed_tensors/quantization/lifecycle/compressed.py,sha256=VreB10xPwgSLQQlTu20UCrFpRS--cA7-lx5s7nrPPrg,2247
|
24
|
-
compressed_tensors/quantization/lifecycle/forward.py,sha256=
|
24
|
+
compressed_tensors/quantization/lifecycle/forward.py,sha256=xeHaUbFxcUyqHffhCBZiRk-ObxjAF99rTnPR1Cweym0,10822
|
25
25
|
compressed_tensors/quantization/lifecycle/frozen.py,sha256=h1XYt89MouBTf3jTYLG_6OdFxIu5q2N8tPjsy6J4E6Y,1726
|
26
26
|
compressed_tensors/quantization/lifecycle/initialize.py,sha256=pFfcu-pxdQKzlnn-18-RlkEktt2yDi6woNXJsiv1A2c,3732
|
27
27
|
compressed_tensors/quantization/observers/__init__.py,sha256=DNH31NQYrIBBcmHsMyFA6whh4pbRsLwuNa6L8AeXaGc,745
|
@@ -36,8 +36,8 @@ compressed_tensors/registry/registry.py,sha256=fxjOjh2wklCvJhQxwofdy-zV8q7MkQ85S
|
|
36
36
|
compressed_tensors/utils/__init__.py,sha256=5DrYjoZbaEvSkJcC-GRSbM_RBHVF4tG9gMd3zsJnjLw,665
|
37
37
|
compressed_tensors/utils/helpers.py,sha256=h0jfl9drs5FAx40tCHRcVtJqXixB5hT5yq_IG2aY_-w,1735
|
38
38
|
compressed_tensors/utils/safetensors_load.py,sha256=wo9UirGrGlenBqZeqotvpCT7D5MEdjCo2J3HeRaIFoU,8502
|
39
|
-
compressed_tensors_nightly-0.3.3.
|
40
|
-
compressed_tensors_nightly-0.3.3.
|
41
|
-
compressed_tensors_nightly-0.3.3.
|
42
|
-
compressed_tensors_nightly-0.3.3.
|
43
|
-
compressed_tensors_nightly-0.3.3.
|
39
|
+
compressed_tensors_nightly-0.3.3.20240531.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
40
|
+
compressed_tensors_nightly-0.3.3.20240531.dist-info/METADATA,sha256=C02ZlsO9SJ14Zly_UFEtYDmiOdXVb-0upcrn_sgM1QM,5673
|
41
|
+
compressed_tensors_nightly-0.3.3.20240531.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
42
|
+
compressed_tensors_nightly-0.3.3.20240531.dist-info/top_level.txt,sha256=w2i-GyPs2s1UwVxvutSvN_lM22SXC2hQFBmoMcPnV7Y,19
|
43
|
+
compressed_tensors_nightly-0.3.3.20240531.dist-info/RECORD,,
|
File without changes
|
File without changes
|