gptqmodel 1.2.1.dev0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gptqmodel-1.2.1.dev0/gptqmodel.egg-info → gptqmodel-1.3.0}/PKG-INFO +52 -56
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/README.md +43 -54
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/quantizer.py +19 -5
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/utils.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/_const.py +2 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/auto.py +18 -6
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/base.py +102 -248
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/__init__.py +2 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/chatglm.py +4 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gemma2.py +2 -9
- gptqmodel-1.3.0/gptqmodel/models/definitions/glm.py +15 -0
- gptqmodel-1.3.0/gptqmodel/models/definitions/hymba.py +31 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mllama.py +1 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/xverse.py +2 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/loader.py +71 -60
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/writer.py +85 -77
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py +3 -2
- gptqmodel-1.3.0/gptqmodel/nn_modules/qlinear/qlinear_cuda.py +304 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_ipex.py +4 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py +2 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/config.py +8 -13
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/gptq.py +2 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/quantizer.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/backend.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/bitblas.py +5 -6
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/device.py +13 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/importer.py +8 -6
- gptqmodel-1.3.0/gptqmodel/utils/lm_eval.py +125 -0
- gptqmodel-1.3.0/gptqmodel/utils/logger.py +17 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/marlin.py +4 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/model.py +7 -13
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/perplexity.py +3 -2
- gptqmodel-1.3.0/gptqmodel/utils/plotly.py +18 -0
- gptqmodel-1.3.0/gptqmodel/utils/progress.py +123 -0
- gptqmodel-1.3.0/gptqmodel/version.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0/gptqmodel.egg-info}/PKG-INFO +52 -56
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/SOURCES.txt +13 -10
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/requires.txt +10 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/top_level.txt +2 -0
- gptqmodel-1.3.0/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
- gptqmodel-1.3.0/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
- gptqmodel-1.3.0/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
- gptqmodel-1.3.0/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
- gptqmodel-1.3.0/requirements.txt +11 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/setup.py +20 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_dynamic.py +5 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_estimate_vram.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_lm_eval.py +7 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_pt.py +9 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_bitblas.py +1 -7
- gptqmodel-1.3.0/tests/test_q4_cuda.py +108 -0
- gptqmodel-1.3.0/tests/test_q4_ipex.py +23 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_marlin.py +4 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_save_loaded_quantized_model.py +9 -7
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_transformers_integration.py +4 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_verify_hash.py +8 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_vllm.py +4 -3
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/__init__.py +0 -3
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_base.py +0 -65
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -42
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/data_utils.py +0 -263
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -34
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/language_modeling_task.py +0 -45
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/sequence_classification_task.py +0 -101
- gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/text_summarization_task.py +0 -70
- gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
- gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
- gptqmodel-1.2.1.dev0/requirements.txt +0 -18
- gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/LICENSE +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/MANIFEST.in +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/constants.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/data.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/hf_quantizer_gptq.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/baichuan.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/bloom.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/codegen.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/cohere.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/dbrx.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/decilm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/exaone.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gemma.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt_neox.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gptj.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/granite.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/grinmoe.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/internlm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/internlm2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/llama.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/longllama.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/minicpm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/minicpm3.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mistral.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mixtral.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mobilellm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/moss.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mpt.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/opt.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/phi.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/phi3.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/rw.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/starcoder2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/yi.py +0 -0
- {gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils → gptqmodel-1.3.0/gptqmodel/nn_modules}/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py +0 -0
- {gptqmodel-1.2.1.dev0/gptqmodel/nn_modules → gptqmodel-1.3.0/gptqmodel/nn_modules/triton_utils}/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/data.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/sglang.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/vllm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/vram.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/dependency_links.txt +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/config.h +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_repack.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_repack.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_dtypes.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_repack.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_repack.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/setup.cfg +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_lm_head.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_packing.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_perplexity.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_exllama_v2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_triton.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_batch.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_formats.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_trust_remote.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_serialization.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_sglang.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_sharded.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_tgi.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_triton.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: gptqmodel
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
|
|
5
5
|
Home-page: https://github.com/ModelCloud/GPTQModel
|
|
6
6
|
Author: ModelCloud
|
|
@@ -34,64 +34,63 @@ Provides-Extra: sglang
|
|
|
34
34
|
Requires-Dist: sglang>=0.3.2; extra == "sglang"
|
|
35
35
|
Requires-Dist: flashinfer==0.1.6; extra == "sglang"
|
|
36
36
|
Provides-Extra: bitblas
|
|
37
|
-
Requires-Dist: bitblas
|
|
37
|
+
Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
|
|
38
38
|
Provides-Extra: hf
|
|
39
39
|
Requires-Dist: optimum>=1.21.2; extra == "hf"
|
|
40
40
|
Provides-Extra: ipex
|
|
41
41
|
Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
|
|
42
42
|
Provides-Extra: auto-round
|
|
43
43
|
Requires-Dist: auto_round>=0.3; extra == "auto-round"
|
|
44
|
+
Provides-Extra: logger
|
|
45
|
+
Requires-Dist: clearml; extra == "logger"
|
|
46
|
+
Requires-Dist: random_word; extra == "logger"
|
|
47
|
+
Requires-Dist: device-smi; extra == "logger"
|
|
48
|
+
Requires-Dist: plotly; extra == "logger"
|
|
49
|
+
Provides-Extra: eval
|
|
50
|
+
Requires-Dist: lm_eval>=0.4.4; extra == "eval"
|
|
44
51
|
|
|
45
52
|
<h1 align="center">GPTQModel</h1>
|
|
46
53
|
<p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
|
|
47
54
|
<p align="center">
|
|
48
55
|
<a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
|
|
49
56
|
<a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
|
|
50
|
-
<a href="https://
|
|
57
|
+
<a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
|
|
58
|
+
<a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
|
|
51
59
|
</p>
|
|
52
60
|
|
|
53
61
|
## News
|
|
54
|
-
* 11/
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
* 10/
|
|
58
|
-
* 10/
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* 09/26/2024 ✨ [
|
|
62
|
-
* 09/
|
|
63
|
-
* 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
64
|
-
* 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
62
|
+
* 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
|
|
63
|
+
* 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
|
|
64
|
+
* 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
|
|
65
|
+
* 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
|
|
66
|
+
* 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
|
|
67
|
+
* 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
|
|
68
|
+
* 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
|
|
69
|
+
* 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
|
|
70
|
+
* 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
|
|
65
71
|
|
|
66
72
|
<details>
|
|
67
73
|
|
|
68
74
|
<summary>Archived News:</summary>
|
|
69
|
-
*
|
|
70
|
-
|
|
71
|
-
*
|
|
72
|
-
|
|
73
|
-
*
|
|
74
|
-
|
|
75
|
-
* 07/
|
|
75
|
+
* 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
|
|
76
|
+
* 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
77
|
+
* 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
78
|
+
* 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
79
|
+
* 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
|
|
80
|
+
* 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
|
|
81
|
+
* 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
|
|
82
|
+
* 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
|
|
76
83
|
Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
|
|
77
84
|
got full end-to-end in/out features padding to enhance current/future model compatibility.
|
|
78
|
-
|
|
79
|
-
* 07/08/2024 🚀 [
|
|
80
|
-
|
|
81
|
-
* 07/
|
|
82
|
-
|
|
83
|
-
*
|
|
84
|
-
|
|
85
|
-
* 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
86
|
-
|
|
87
|
-
* 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
88
|
-
|
|
89
|
-
* 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
85
|
+
* 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
|
|
86
|
+
* 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
|
|
87
|
+
* 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
|
|
88
|
+
* 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
89
|
+
* 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
90
|
+
* 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
90
91
|
Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
|
|
91
|
-
|
|
92
|
-
* 06/
|
|
93
|
-
|
|
94
|
-
* 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
92
|
+
* 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
|
|
93
|
+
* 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
95
94
|
</details>
|
|
96
95
|
|
|
97
96
|
## Why should you use GPTQModel?
|
|
@@ -103,7 +102,7 @@ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphe
|
|
|
103
102
|
Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
|
|
104
103
|
|
|
105
104
|
## Features
|
|
106
|
-
* 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted)
|
|
105
|
+
* 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
|
|
107
106
|
* ✨ 100% CI coverage for all supported models including quality/ppl regression.
|
|
108
107
|
* 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
|
|
109
108
|
* 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
|
|
@@ -119,27 +118,24 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
119
118
|
* 🚀 Over 50% faster PPL calculations (OPT)
|
|
120
119
|
* 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
|
|
121
120
|
|
|
121
|
+
## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
|
|
122
|
+
🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
|
|
122
123
|
|
|
123
|
-
|
|
124
|
-
[🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
|
|
125
|
-
|
|
124
|
+

|
|
126
125
|
|
|
126
|
+
## Model Support: 🚀 (Added by GPTQModel)
|
|
127
127
|
| Model | | | | | | | | | | |
|
|
128
|
-
| ---------------- | --- | -------------- | --- | ---------------- | ---
|
|
129
|
-
| Baichuan | ✅ | Falon | ✅ | Llama 3
|
|
130
|
-
| Bloom | ✅ | Gemma 2 | 🚀 |
|
|
131
|
-
| ChatGLM | 🚀 | GPTBigCod | ✅ |
|
|
132
|
-
| CodeGen | ✅ | GPTNeoX | ✅ |
|
|
133
|
-
| Cohere | ✅ | GPT-2 | ✅ |
|
|
134
|
-
| DBRX Converted | 🚀 | GPT-J | ✅ |
|
|
135
|
-
| Deci | ✅ | Granite | 🚀 |
|
|
136
|
-
| DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 |
|
|
137
|
-
| DeepSeek-V2-Lite | 🚀 |
|
|
138
|
-
| EXAONE 3.0 | 🚀 |
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
|
|
142
|
-

|
|
128
|
+
| ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- | --- | --- | --- |
|
|
129
|
+
| Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | |
|
|
130
|
+
| Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
|
|
131
|
+
| ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
|
|
132
|
+
| CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | RefinedWeb | ✅ | | | |
|
|
133
|
+
| Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
|
|
134
|
+
| DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
|
|
135
|
+
| Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | XVERSE | ✅ | | | |
|
|
136
|
+
| DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
|
|
137
|
+
| DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | | | | | |
|
|
138
|
+
| EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |## Compatiblity
|
|
143
139
|
|
|
144
140
|
|
|
145
141
|
## Platform Requirements
|
|
@@ -3,51 +3,43 @@
|
|
|
3
3
|
<p align="center">
|
|
4
4
|
<a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
|
|
5
5
|
<a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
|
|
6
|
-
<a href="https://
|
|
6
|
+
<a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
|
|
7
|
+
<a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
|
|
7
8
|
</p>
|
|
8
9
|
|
|
9
10
|
## News
|
|
10
|
-
* 11/
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* 10/
|
|
14
|
-
* 10/
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* 09/26/2024 ✨ [
|
|
18
|
-
* 09/
|
|
19
|
-
* 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
20
|
-
* 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
11
|
+
* 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
|
|
12
|
+
* 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
|
|
13
|
+
* 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
|
|
14
|
+
* 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
|
|
15
|
+
* 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
|
|
16
|
+
* 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
|
|
17
|
+
* 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
|
|
18
|
+
* 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
|
|
19
|
+
* 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
|
|
21
20
|
|
|
22
21
|
<details>
|
|
23
22
|
|
|
24
23
|
<summary>Archived News:</summary>
|
|
25
|
-
*
|
|
26
|
-
|
|
27
|
-
*
|
|
28
|
-
|
|
29
|
-
*
|
|
30
|
-
|
|
31
|
-
* 07/
|
|
24
|
+
* 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
|
|
25
|
+
* 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
26
|
+
* 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
27
|
+
* 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
28
|
+
* 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
|
|
29
|
+
* 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
|
|
30
|
+
* 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
|
|
31
|
+
* 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
|
|
32
32
|
Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
|
|
33
33
|
got full end-to-end in/out features padding to enhance current/future model compatibility.
|
|
34
|
-
|
|
35
|
-
* 07/08/2024 🚀 [
|
|
36
|
-
|
|
37
|
-
* 07/
|
|
38
|
-
|
|
39
|
-
*
|
|
40
|
-
|
|
41
|
-
* 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
42
|
-
|
|
43
|
-
* 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
44
|
-
|
|
45
|
-
* 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
34
|
+
* 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
|
|
35
|
+
* 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
|
|
36
|
+
* 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
|
|
37
|
+
* 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
38
|
+
* 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
39
|
+
* 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
46
40
|
Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
|
|
47
|
-
|
|
48
|
-
* 06/
|
|
49
|
-
|
|
50
|
-
* 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
41
|
+
* 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
|
|
42
|
+
* 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
51
43
|
</details>
|
|
52
44
|
|
|
53
45
|
## Why should you use GPTQModel?
|
|
@@ -59,7 +51,7 @@ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphe
|
|
|
59
51
|
Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
|
|
60
52
|
|
|
61
53
|
## Features
|
|
62
|
-
* 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted)
|
|
54
|
+
* 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
|
|
63
55
|
* ✨ 100% CI coverage for all supported models including quality/ppl regression.
|
|
64
56
|
* 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
|
|
65
57
|
* 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
|
|
@@ -75,27 +67,24 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
75
67
|
* 🚀 Over 50% faster PPL calculations (OPT)
|
|
76
68
|
* 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
|
|
77
69
|
|
|
70
|
+
## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
|
|
71
|
+
🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
|
|
78
72
|
|
|
79
|
-
|
|
80
|
-
[🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
|
|
81
|
-
|
|
73
|
+

|
|
82
74
|
|
|
75
|
+
## Model Support: 🚀 (Added by GPTQModel)
|
|
83
76
|
| Model | | | | | | | | | | |
|
|
84
|
-
| ---------------- | --- | -------------- | --- | ---------------- | ---
|
|
85
|
-
| Baichuan | ✅ | Falon | ✅ | Llama 3
|
|
86
|
-
| Bloom | ✅ | Gemma 2 | 🚀 |
|
|
87
|
-
| ChatGLM | 🚀 | GPTBigCod | ✅ |
|
|
88
|
-
| CodeGen | ✅ | GPTNeoX | ✅ |
|
|
89
|
-
| Cohere | ✅ | GPT-2 | ✅ |
|
|
90
|
-
| DBRX Converted | 🚀 | GPT-J | ✅ |
|
|
91
|
-
| Deci | ✅ | Granite | 🚀 |
|
|
92
|
-
| DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 |
|
|
93
|
-
| DeepSeek-V2-Lite | 🚀 |
|
|
94
|
-
| EXAONE 3.0 | 🚀 |
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
|
|
98
|
-

|
|
77
|
+
| ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- | --- | --- | --- |
|
|
78
|
+
| Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | |
|
|
79
|
+
| Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
|
|
80
|
+
| ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
|
|
81
|
+
| CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | RefinedWeb | ✅ | | | |
|
|
82
|
+
| Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
|
|
83
|
+
| DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
|
|
84
|
+
| Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | XVERSE | ✅ | | | |
|
|
85
|
+
| DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
|
|
86
|
+
| DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | | | | | |
|
|
87
|
+
| EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |## Compatiblity
|
|
99
88
|
|
|
100
89
|
|
|
101
90
|
## Platform Requirements
|
|
@@ -15,12 +15,10 @@
|
|
|
15
15
|
import json
|
|
16
16
|
import os
|
|
17
17
|
from dataclasses import dataclass
|
|
18
|
-
from logging import getLogger
|
|
19
18
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
20
19
|
|
|
21
20
|
import torch
|
|
22
21
|
from torch import nn
|
|
23
|
-
from tqdm.auto import tqdm
|
|
24
22
|
from transformers import AutoTokenizer
|
|
25
23
|
from transformers.pytorch_utils import Conv1D
|
|
26
24
|
from transformers.utils.quantization_config import GPTQConfig, QuantizationMethod
|
|
@@ -28,6 +26,7 @@ from transformers.utils.quantization_config import GPTQConfig, QuantizationMetho
|
|
|
28
26
|
from .constants import GPTQ_CONFIG
|
|
29
27
|
from .data import get_dataset, prepare_dataset
|
|
30
28
|
from .utils import get_block_name_with_pattern, get_device, get_layers, get_preceding_modules, get_seqlen
|
|
29
|
+
from ...utils.progress import ProgressBar
|
|
31
30
|
|
|
32
31
|
try:
|
|
33
32
|
from optimum.utils import is_accelerate_available
|
|
@@ -46,9 +45,10 @@ if OPTIMUM_AVAILABLE and is_accelerate_available():
|
|
|
46
45
|
from ...quantization import FORMAT, FORMAT_FIELD_JSON, GPTQ, QuantizeConfig
|
|
47
46
|
from ...utils.backend import BACKEND
|
|
48
47
|
from ...utils.importer import select_quant_linear
|
|
48
|
+
from ...utils.logger import setup_logger
|
|
49
49
|
from ...utils.model import convert_gptq_v1_to_v2_format, convert_gptq_v2_to_v1_format, gptqmodel_post_init
|
|
50
50
|
|
|
51
|
-
logger =
|
|
51
|
+
logger = setup_logger()
|
|
52
52
|
|
|
53
53
|
OPTIMUM_INSTALL_HINT = "optimum not installed. Please install via `pip install optimum`."
|
|
54
54
|
|
|
@@ -289,6 +289,20 @@ class GPTQModelQuantizer(object):
|
|
|
289
289
|
return model
|
|
290
290
|
|
|
291
291
|
def convert_gptq_v2_to_v1(self, model: nn.Module):
|
|
292
|
+
if self.bits != 4:
|
|
293
|
+
cuda_name_modules = {}
|
|
294
|
+
from gptqmodel.nn_modules.qlinear.qlinear_cuda import CudaQuantLinear
|
|
295
|
+
for name, module in model.named_modules():
|
|
296
|
+
if isinstance(module, CudaQuantLinear):
|
|
297
|
+
cuda_name_modules[name] = module.gptqmodel_cuda
|
|
298
|
+
module.gptqmodel_cuda = None
|
|
299
|
+
|
|
300
|
+
for name, module in model.named_modules():
|
|
301
|
+
if isinstance(module, CudaQuantLinear) and name in cuda_name_modules:
|
|
302
|
+
module.gptqmodel_cuda = cuda_name_modules[name]
|
|
303
|
+
|
|
304
|
+
del cuda_name_modules
|
|
305
|
+
|
|
292
306
|
model = convert_gptq_v2_to_v1_format(
|
|
293
307
|
model, quantize_config=self.quantize_config, qlinear_kernel=self.qlinear_kernel
|
|
294
308
|
)
|
|
@@ -501,7 +515,7 @@ class GPTQModelQuantizer(object):
|
|
|
501
515
|
|
|
502
516
|
# Step 3: Quantize the blocks
|
|
503
517
|
quantizers = {}
|
|
504
|
-
for i, block in enumerate(
|
|
518
|
+
for i, block in enumerate(ProgressBar(blocks, desc=f"Quantizing {self.block_name_to_quantize} blocks ")):
|
|
505
519
|
logger.info(f"Start quantizing block {self.block_name_to_quantize} {i + 1}/{len(blocks)}")
|
|
506
520
|
|
|
507
521
|
if not self.cache_block_outputs:
|
|
@@ -533,7 +547,7 @@ class GPTQModelQuantizer(object):
|
|
|
533
547
|
else:
|
|
534
548
|
layers_name_list = [list(layers.keys())]
|
|
535
549
|
logger.info(f"Module to quantize {layers_name_list}")
|
|
536
|
-
for subset_name_list in
|
|
550
|
+
for subset_name_list in ProgressBar(layers_name_list, desc="Quantizing layers inside the block"):
|
|
537
551
|
subset_layers = {name: layers[name] for name in subset_name_list}
|
|
538
552
|
gptq = {}
|
|
539
553
|
handles = []
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import os
|
|
15
|
-
from logging import getLogger
|
|
16
15
|
from typing import Callable, Optional, Union
|
|
17
16
|
|
|
18
17
|
import torch
|
|
@@ -20,11 +19,13 @@ from torch import nn
|
|
|
20
19
|
from transformers.modeling_utils import PreTrainedModel
|
|
21
20
|
from transformers.pytorch_utils import Conv1D
|
|
22
21
|
|
|
22
|
+
from ...utils.logger import setup_logger
|
|
23
23
|
from .constants import BLOCK_PATTERNS, SEQLEN_KEYS_TRANFORMERS
|
|
24
24
|
|
|
25
25
|
ori_save_pretrained = PreTrainedModel.save_pretrained
|
|
26
26
|
|
|
27
|
-
logger =
|
|
27
|
+
logger = setup_logger()
|
|
28
|
+
|
|
28
29
|
|
|
29
30
|
"""
|
|
30
31
|
Set of utilities to get specific attributes of a model
|
|
@@ -27,6 +27,7 @@ SUPPORTED_MODELS = [
|
|
|
27
27
|
"gpt_bigcode",
|
|
28
28
|
"codegen",
|
|
29
29
|
"chatglm",
|
|
30
|
+
"glm",
|
|
30
31
|
"RefinedWebModel",
|
|
31
32
|
"RefinedWeb",
|
|
32
33
|
"baichuan",
|
|
@@ -60,6 +61,7 @@ SUPPORTED_MODELS = [
|
|
|
60
61
|
"mllama",
|
|
61
62
|
"granite",
|
|
62
63
|
"mobilellm",
|
|
64
|
+
"hymba"
|
|
63
65
|
]
|
|
64
66
|
|
|
65
67
|
EXLLAMA_DEFAULT_MAX_INPUT_LENGTH = 2048
|
|
@@ -5,10 +5,11 @@ from os.path import isdir, join
|
|
|
5
5
|
from typing import Dict, List, Optional, Union
|
|
6
6
|
|
|
7
7
|
from gptqmodel.quantization import QUANT_CONFIG_FILENAME
|
|
8
|
+
from huggingface_hub import list_repo_files
|
|
8
9
|
from transformers import AutoConfig
|
|
9
|
-
from transformers.utils.hub import cached_file
|
|
10
10
|
|
|
11
11
|
from ..utils import BACKEND
|
|
12
|
+
from ..utils.logger import setup_logger
|
|
12
13
|
from ..utils.model import check_and_get_model_type
|
|
13
14
|
from .base import BaseGPTQModel, QuantizeConfig
|
|
14
15
|
from .definitions.baichuan import BaiChuanGPTQ
|
|
@@ -23,6 +24,7 @@ from .definitions.deepseek_v2 import DeepSeekV2GPTQ
|
|
|
23
24
|
from .definitions.exaone import ExaoneGPTQ
|
|
24
25
|
from .definitions.gemma import GemmaGPTQ
|
|
25
26
|
from .definitions.gemma2 import Gemma2GPTQ
|
|
27
|
+
from .definitions.glm import GLM
|
|
26
28
|
from .definitions.gpt2 import GPT2GPTQ
|
|
27
29
|
from .definitions.gpt_bigcode import GPTBigCodeGPTQ
|
|
28
30
|
from .definitions.gpt_neox import GPTNeoXGPTQ
|
|
@@ -52,7 +54,9 @@ from .definitions.stablelmepoch import StableLMEpochGPTQ
|
|
|
52
54
|
from .definitions.starcoder2 import Starcoder2GPTQ
|
|
53
55
|
from .definitions.xverse import XverseGPTQ
|
|
54
56
|
from .definitions.yi import YiGPTQ
|
|
55
|
-
from
|
|
57
|
+
from .definitions.hymba import HymbaGPTQ
|
|
58
|
+
|
|
59
|
+
logger = setup_logger()
|
|
56
60
|
|
|
57
61
|
MODEL_MAP = {
|
|
58
62
|
"bloom": BloomGPTQ,
|
|
@@ -63,6 +67,7 @@ MODEL_MAP = {
|
|
|
63
67
|
"opt": OPTGPTQ,
|
|
64
68
|
"moss": MOSSGPTQ,
|
|
65
69
|
"chatglm": ChatGLM,
|
|
70
|
+
"glm": GLM,
|
|
66
71
|
"gpt_bigcode": GPTBigCodeGPTQ,
|
|
67
72
|
"codegen": CodeGenGPTQ,
|
|
68
73
|
"cohere": CohereGPTQ,
|
|
@@ -98,6 +103,7 @@ MODEL_MAP = {
|
|
|
98
103
|
"mllama": MLlamaGPTQ,
|
|
99
104
|
"granite": GraniteGPTQ,
|
|
100
105
|
"mobilellm": MobileLLMGPTQ,
|
|
106
|
+
"hymba": HymbaGPTQ,
|
|
101
107
|
}
|
|
102
108
|
|
|
103
109
|
|
|
@@ -152,7 +158,7 @@ class GPTQModel:
|
|
|
152
158
|
)
|
|
153
159
|
else:
|
|
154
160
|
return cls.from_pretrained(
|
|
155
|
-
|
|
161
|
+
model_id_or_path=model_id_or_path,
|
|
156
162
|
quantize_config=quantize_config,
|
|
157
163
|
trust_remote_code=trust_remote_code,
|
|
158
164
|
**kwargs,
|
|
@@ -161,14 +167,20 @@ class GPTQModel:
|
|
|
161
167
|
@classmethod
|
|
162
168
|
def from_pretrained(
|
|
163
169
|
cls,
|
|
164
|
-
|
|
170
|
+
model_id_or_path: str,
|
|
165
171
|
quantize_config: QuantizeConfig,
|
|
166
172
|
trust_remote_code: bool = False,
|
|
167
173
|
**model_init_kwargs,
|
|
168
174
|
) -> BaseGPTQModel:
|
|
169
|
-
|
|
175
|
+
if hasattr(AutoConfig.from_pretrained(model_id_or_path, trust_remote_code=trust_remote_code), "quantization_config"):
|
|
176
|
+
logger.warning("Model is already quantized, will use `from_quantized` to load quantized model.\n"
|
|
177
|
+
"If you want to quantize the model, please pass un_quantized model path or id, and use "
|
|
178
|
+
"`from_pretrained` with `quantize_config`.")
|
|
179
|
+
return cls.from_quantized(model_id_or_path, trust_remote_code=trust_remote_code)
|
|
180
|
+
|
|
181
|
+
model_type = check_and_get_model_type(model_id_or_path, trust_remote_code)
|
|
170
182
|
return MODEL_MAP[model_type].from_pretrained(
|
|
171
|
-
pretrained_model_id_or_path=
|
|
183
|
+
pretrained_model_id_or_path=model_id_or_path,
|
|
172
184
|
quantize_config=quantize_config,
|
|
173
185
|
trust_remote_code=trust_remote_code,
|
|
174
186
|
**model_init_kwargs,
|