gptqmodel 1.2.1.dev0__tar.gz → 1.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gptqmodel-1.2.1.dev0/gptqmodel.egg-info → gptqmodel-1.2.3}/PKG-INFO +39 -44
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/README.md +30 -42
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/quantizer.py +16 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/utils.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/_const.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/auto.py +16 -6
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/base.py +87 -126
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/__init__.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/chatglm.py +4 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gemma2.py +2 -9
- gptqmodel-1.2.3/gptqmodel/models/definitions/glm.py +15 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mllama.py +1 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/xverse.py +2 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/loader.py +71 -60
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/writer.py +85 -77
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py +3 -2
- gptqmodel-1.2.3/gptqmodel/nn_modules/qlinear/qlinear_cuda.py +304 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py +3 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_ipex.py +4 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py +2 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/config.py +8 -13
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/gptq.py +2 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/quantizer.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/backend.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/bitblas.py +2 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/device.py +13 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/importer.py +8 -6
- gptqmodel-1.2.3/gptqmodel/utils/logger.py +17 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/marlin.py +2 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/model.py +2 -8
- gptqmodel-1.2.3/gptqmodel/utils/plotly.py +18 -0
- gptqmodel-1.2.3/gptqmodel/version.py +1 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3/gptqmodel.egg-info}/PKG-INFO +39 -44
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/SOURCES.txt +10 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/requires.txt +10 -1
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/top_level.txt +2 -0
- gptqmodel-1.2.3/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
- gptqmodel-1.2.3/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
- gptqmodel-1.2.3/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
- gptqmodel-1.2.3/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
- gptqmodel-1.2.3/requirements.txt +13 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/setup.py +20 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_dynamic.py +5 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_estimate_vram.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_lm_eval.py +3 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_pt.py +9 -3
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_bitblas.py +1 -7
- gptqmodel-1.2.3/tests/test_q4_cuda.py +109 -0
- gptqmodel-1.2.3/tests/test_q4_ipex.py +25 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_marlin.py +4 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_save_loaded_quantized_model.py +9 -7
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_transformers_integration.py +4 -4
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_verify_hash.py +8 -2
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_vllm.py +6 -3
- gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
- gptqmodel-1.2.1.dev0/requirements.txt +0 -18
- gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/LICENSE +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/MANIFEST.in +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_base.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/data_utils.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/language_modeling_task.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/sequence_classification_task.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/text_summarization_task.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/constants.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/data.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/hf_quantizer_gptq.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/baichuan.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/bloom.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/codegen.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/cohere.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/dbrx.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/decilm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/exaone.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gemma.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt_neox.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gptj.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/granite.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/grinmoe.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/internlm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/internlm2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/llama.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/longllama.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/minicpm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/minicpm3.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mistral.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mixtral.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mobilellm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/moss.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mpt.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/opt.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/phi.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/phi3.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/rw.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/starcoder2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/yi.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/__init__.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/data.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/perplexity.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/sglang.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/vllm.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/vram.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/dependency_links.txt +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/config.h +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_repack.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_repack.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda.cpp +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_dtypes.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_repack.cu +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_repack.cuh +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/setup.cfg +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_lm_head.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_packing.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_perplexity.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_exllama_v2.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_triton.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_batch.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_formats.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_trust_remote.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_serialization.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_sglang.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_sharded.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_tgi.py +0 -0
- {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_triton.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: gptqmodel
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.3
|
|
4
4
|
Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
|
|
5
5
|
Home-page: https://github.com/ModelCloud/GPTQModel
|
|
6
6
|
Author: ModelCloud
|
|
@@ -34,64 +34,62 @@ Provides-Extra: sglang
|
|
|
34
34
|
Requires-Dist: sglang>=0.3.2; extra == "sglang"
|
|
35
35
|
Requires-Dist: flashinfer==0.1.6; extra == "sglang"
|
|
36
36
|
Provides-Extra: bitblas
|
|
37
|
-
Requires-Dist: bitblas
|
|
37
|
+
Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
|
|
38
38
|
Provides-Extra: hf
|
|
39
39
|
Requires-Dist: optimum>=1.21.2; extra == "hf"
|
|
40
40
|
Provides-Extra: ipex
|
|
41
41
|
Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
|
|
42
42
|
Provides-Extra: auto-round
|
|
43
43
|
Requires-Dist: auto_round>=0.3; extra == "auto-round"
|
|
44
|
+
Provides-Extra: logger
|
|
45
|
+
Requires-Dist: clearml; extra == "logger"
|
|
46
|
+
Requires-Dist: random_word; extra == "logger"
|
|
47
|
+
Requires-Dist: device-smi; extra == "logger"
|
|
48
|
+
Requires-Dist: plotly; extra == "logger"
|
|
49
|
+
Provides-Extra: eval
|
|
50
|
+
Requires-Dist: lm_eval>=0.4.4; extra == "eval"
|
|
44
51
|
|
|
45
52
|
<h1 align="center">GPTQModel</h1>
|
|
46
53
|
<p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
|
|
47
54
|
<p align="center">
|
|
48
55
|
<a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
|
|
49
56
|
<a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
|
|
50
|
-
<a href="https://
|
|
57
|
+
<a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
|
|
58
|
+
<a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
|
|
51
59
|
</p>
|
|
52
60
|
|
|
53
61
|
## News
|
|
54
|
-
* 11/
|
|
55
|
-
*
|
|
56
|
-
* 10/
|
|
57
|
-
* 10/
|
|
58
|
-
* 10/
|
|
59
|
-
*
|
|
60
|
-
* 09/26/2024 ✨ [
|
|
61
|
-
* 09/26/2024 ✨ [
|
|
62
|
-
* 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
63
|
-
* 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
64
|
-
* 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
62
|
+
* 11/19/2024 [1.2.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.2) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
|
|
63
|
+
* 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
|
|
64
|
+
* 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
|
|
65
|
+
* 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
|
|
66
|
+
* 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
|
|
67
|
+
* 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
|
|
68
|
+
* 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
|
|
69
|
+
* 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
|
|
65
70
|
|
|
66
71
|
<details>
|
|
67
72
|
|
|
68
73
|
<summary>Archived News:</summary>
|
|
69
|
-
*
|
|
70
|
-
|
|
71
|
-
*
|
|
72
|
-
|
|
73
|
-
*
|
|
74
|
-
|
|
75
|
-
* 07/
|
|
74
|
+
* 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
|
|
75
|
+
* 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
76
|
+
* 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
77
|
+
* 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
78
|
+
* 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
|
|
79
|
+
* 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
|
|
80
|
+
* 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
|
|
81
|
+
* 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
|
|
76
82
|
Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
|
|
77
83
|
got full end-to-end in/out features padding to enhance current/future model compatibility.
|
|
78
|
-
|
|
79
|
-
* 07/08/2024 🚀 [
|
|
80
|
-
|
|
81
|
-
* 07/
|
|
82
|
-
|
|
83
|
-
*
|
|
84
|
-
|
|
85
|
-
* 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
86
|
-
|
|
87
|
-
* 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
88
|
-
|
|
89
|
-
* 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
84
|
+
* 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
|
|
85
|
+
* 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
|
|
86
|
+
* 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
|
|
87
|
+
* 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
88
|
+
* 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
89
|
+
* 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
90
90
|
Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
|
|
91
|
-
|
|
92
|
-
* 06/
|
|
93
|
-
|
|
94
|
-
* 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
91
|
+
* 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
|
|
92
|
+
* 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
95
93
|
</details>
|
|
96
94
|
|
|
97
95
|
## Why should you use GPTQModel?
|
|
@@ -119,11 +117,12 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
119
117
|
* 🚀 Over 50% faster PPL calculations (OPT)
|
|
120
118
|
* 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
|
|
121
119
|
|
|
120
|
+
## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
|
|
121
|
+
🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
|
|
122
122
|
|
|
123
|
-
|
|
124
|
-
[🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
|
|
125
|
-
|
|
123
|
+

|
|
126
124
|
|
|
125
|
+
## Model Support: 🚀 (Added by GPTQModel)
|
|
127
126
|
| Model | | | | | | | | | | |
|
|
128
127
|
| ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
|
|
129
128
|
| Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
|
|
@@ -138,10 +137,6 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
138
137
|
| EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
|
|
139
138
|
|
|
140
139
|
|
|
141
|
-
## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
|
|
142
|
-

|
|
143
|
-
|
|
144
|
-
|
|
145
140
|
## Platform Requirements
|
|
146
141
|
|
|
147
142
|
GPTQModel is validated for Linux x86_64 with Nvidia GPUs. Windows WSL2 may work but un-tested.
|
|
@@ -3,51 +3,42 @@
|
|
|
3
3
|
<p align="center">
|
|
4
4
|
<a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
|
|
5
5
|
<a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
|
|
6
|
-
<a href="https://
|
|
6
|
+
<a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
|
|
7
|
+
<a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
|
|
7
8
|
</p>
|
|
8
9
|
|
|
9
10
|
## News
|
|
10
|
-
* 11/
|
|
11
|
-
*
|
|
12
|
-
* 10/
|
|
13
|
-
* 10/
|
|
14
|
-
* 10/
|
|
15
|
-
*
|
|
16
|
-
* 09/26/2024 ✨ [
|
|
17
|
-
* 09/26/2024 ✨ [
|
|
18
|
-
* 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
19
|
-
* 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
20
|
-
* 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
11
|
+
* 11/19/2024 [1.2.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.2) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
|
|
12
|
+
* 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
|
|
13
|
+
* 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
|
|
14
|
+
* 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
|
|
15
|
+
* 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
|
|
16
|
+
* 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
|
|
17
|
+
* 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
|
|
18
|
+
* 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
|
|
21
19
|
|
|
22
20
|
<details>
|
|
23
21
|
|
|
24
22
|
<summary>Archived News:</summary>
|
|
25
|
-
*
|
|
26
|
-
|
|
27
|
-
*
|
|
28
|
-
|
|
29
|
-
*
|
|
30
|
-
|
|
31
|
-
* 07/
|
|
23
|
+
* 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
|
|
24
|
+
* 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
|
|
25
|
+
* 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
|
|
26
|
+
* 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
|
|
27
|
+
* 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
|
|
28
|
+
* 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
|
|
29
|
+
* 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
|
|
30
|
+
* 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
|
|
32
31
|
Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
|
|
33
32
|
got full end-to-end in/out features padding to enhance current/future model compatibility.
|
|
34
|
-
|
|
35
|
-
* 07/08/2024 🚀 [
|
|
36
|
-
|
|
37
|
-
* 07/
|
|
38
|
-
|
|
39
|
-
*
|
|
40
|
-
|
|
41
|
-
* 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
42
|
-
|
|
43
|
-
* 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
44
|
-
|
|
45
|
-
* 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
33
|
+
* 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
|
|
34
|
+
* 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
|
|
35
|
+
* 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
|
|
36
|
+
* 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
|
|
37
|
+
* 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
|
|
38
|
+
* 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
|
|
46
39
|
Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
|
|
47
|
-
|
|
48
|
-
* 06/
|
|
49
|
-
|
|
50
|
-
* 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
40
|
+
* 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
|
|
41
|
+
* 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
|
|
51
42
|
</details>
|
|
52
43
|
|
|
53
44
|
## Why should you use GPTQModel?
|
|
@@ -75,11 +66,12 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
75
66
|
* 🚀 Over 50% faster PPL calculations (OPT)
|
|
76
67
|
* 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
|
|
77
68
|
|
|
69
|
+
## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
|
|
70
|
+
🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
|
|
78
71
|
|
|
79
|
-
|
|
80
|
-
[🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
|
|
81
|
-
|
|
72
|
+

|
|
82
73
|
|
|
74
|
+
## Model Support: 🚀 (Added by GPTQModel)
|
|
83
75
|
| Model | | | | | | | | | | |
|
|
84
76
|
| ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
|
|
85
77
|
| Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
|
|
@@ -94,10 +86,6 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
|
|
|
94
86
|
| EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
|
|
95
87
|
|
|
96
88
|
|
|
97
|
-
## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
|
|
98
|
-

|
|
99
|
-
|
|
100
|
-
|
|
101
89
|
## Platform Requirements
|
|
102
90
|
|
|
103
91
|
GPTQModel is validated for Linux x86_64 with Nvidia GPUs. Windows WSL2 may work but un-tested.
|
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
import json
|
|
16
16
|
import os
|
|
17
17
|
from dataclasses import dataclass
|
|
18
|
-
from logging import getLogger
|
|
19
18
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
20
19
|
|
|
21
20
|
import torch
|
|
@@ -46,9 +45,10 @@ if OPTIMUM_AVAILABLE and is_accelerate_available():
|
|
|
46
45
|
from ...quantization import FORMAT, FORMAT_FIELD_JSON, GPTQ, QuantizeConfig
|
|
47
46
|
from ...utils.backend import BACKEND
|
|
48
47
|
from ...utils.importer import select_quant_linear
|
|
48
|
+
from ...utils.logger import setup_logger
|
|
49
49
|
from ...utils.model import convert_gptq_v1_to_v2_format, convert_gptq_v2_to_v1_format, gptqmodel_post_init
|
|
50
50
|
|
|
51
|
-
logger =
|
|
51
|
+
logger = setup_logger()
|
|
52
52
|
|
|
53
53
|
OPTIMUM_INSTALL_HINT = "optimum not installed. Please install via `pip install optimum`."
|
|
54
54
|
|
|
@@ -289,6 +289,20 @@ class GPTQModelQuantizer(object):
|
|
|
289
289
|
return model
|
|
290
290
|
|
|
291
291
|
def convert_gptq_v2_to_v1(self, model: nn.Module):
|
|
292
|
+
if self.bits != 4:
|
|
293
|
+
cuda_name_modules = {}
|
|
294
|
+
from gptqmodel.nn_modules.qlinear.qlinear_cuda import CudaQuantLinear
|
|
295
|
+
for name, module in model.named_modules():
|
|
296
|
+
if isinstance(module, CudaQuantLinear):
|
|
297
|
+
cuda_name_modules[name] = module.gptqmodel_cuda
|
|
298
|
+
module.gptqmodel_cuda = None
|
|
299
|
+
|
|
300
|
+
for name, module in model.named_modules():
|
|
301
|
+
if isinstance(module, CudaQuantLinear) and name in cuda_name_modules:
|
|
302
|
+
module.gptqmodel_cuda = cuda_name_modules[name]
|
|
303
|
+
|
|
304
|
+
del cuda_name_modules
|
|
305
|
+
|
|
292
306
|
model = convert_gptq_v2_to_v1_format(
|
|
293
307
|
model, quantize_config=self.quantize_config, qlinear_kernel=self.qlinear_kernel
|
|
294
308
|
)
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import os
|
|
15
|
-
from logging import getLogger
|
|
16
15
|
from typing import Callable, Optional, Union
|
|
17
16
|
|
|
18
17
|
import torch
|
|
@@ -20,11 +19,13 @@ from torch import nn
|
|
|
20
19
|
from transformers.modeling_utils import PreTrainedModel
|
|
21
20
|
from transformers.pytorch_utils import Conv1D
|
|
22
21
|
|
|
22
|
+
from ...utils.logger import setup_logger
|
|
23
23
|
from .constants import BLOCK_PATTERNS, SEQLEN_KEYS_TRANFORMERS
|
|
24
24
|
|
|
25
25
|
ori_save_pretrained = PreTrainedModel.save_pretrained
|
|
26
26
|
|
|
27
|
-
logger =
|
|
27
|
+
logger = setup_logger()
|
|
28
|
+
|
|
28
29
|
|
|
29
30
|
"""
|
|
30
31
|
Set of utilities to get specific attributes of a model
|
|
@@ -5,10 +5,11 @@ from os.path import isdir, join
|
|
|
5
5
|
from typing import Dict, List, Optional, Union
|
|
6
6
|
|
|
7
7
|
from gptqmodel.quantization import QUANT_CONFIG_FILENAME
|
|
8
|
+
from huggingface_hub import list_repo_files
|
|
8
9
|
from transformers import AutoConfig
|
|
9
|
-
from transformers.utils.hub import cached_file
|
|
10
10
|
|
|
11
11
|
from ..utils import BACKEND
|
|
12
|
+
from ..utils.logger import setup_logger
|
|
12
13
|
from ..utils.model import check_and_get_model_type
|
|
13
14
|
from .base import BaseGPTQModel, QuantizeConfig
|
|
14
15
|
from .definitions.baichuan import BaiChuanGPTQ
|
|
@@ -23,6 +24,7 @@ from .definitions.deepseek_v2 import DeepSeekV2GPTQ
|
|
|
23
24
|
from .definitions.exaone import ExaoneGPTQ
|
|
24
25
|
from .definitions.gemma import GemmaGPTQ
|
|
25
26
|
from .definitions.gemma2 import Gemma2GPTQ
|
|
27
|
+
from .definitions.glm import GLM
|
|
26
28
|
from .definitions.gpt2 import GPT2GPTQ
|
|
27
29
|
from .definitions.gpt_bigcode import GPTBigCodeGPTQ
|
|
28
30
|
from .definitions.gpt_neox import GPTNeoXGPTQ
|
|
@@ -52,7 +54,8 @@ from .definitions.stablelmepoch import StableLMEpochGPTQ
|
|
|
52
54
|
from .definitions.starcoder2 import Starcoder2GPTQ
|
|
53
55
|
from .definitions.xverse import XverseGPTQ
|
|
54
56
|
from .definitions.yi import YiGPTQ
|
|
55
|
-
|
|
57
|
+
|
|
58
|
+
logger = setup_logger()
|
|
56
59
|
|
|
57
60
|
MODEL_MAP = {
|
|
58
61
|
"bloom": BloomGPTQ,
|
|
@@ -63,6 +66,7 @@ MODEL_MAP = {
|
|
|
63
66
|
"opt": OPTGPTQ,
|
|
64
67
|
"moss": MOSSGPTQ,
|
|
65
68
|
"chatglm": ChatGLM,
|
|
69
|
+
"glm": GLM,
|
|
66
70
|
"gpt_bigcode": GPTBigCodeGPTQ,
|
|
67
71
|
"codegen": CodeGenGPTQ,
|
|
68
72
|
"cohere": CohereGPTQ,
|
|
@@ -152,7 +156,7 @@ class GPTQModel:
|
|
|
152
156
|
)
|
|
153
157
|
else:
|
|
154
158
|
return cls.from_pretrained(
|
|
155
|
-
|
|
159
|
+
model_id_or_path=model_id_or_path,
|
|
156
160
|
quantize_config=quantize_config,
|
|
157
161
|
trust_remote_code=trust_remote_code,
|
|
158
162
|
**kwargs,
|
|
@@ -161,14 +165,20 @@ class GPTQModel:
|
|
|
161
165
|
@classmethod
|
|
162
166
|
def from_pretrained(
|
|
163
167
|
cls,
|
|
164
|
-
|
|
168
|
+
model_id_or_path: str,
|
|
165
169
|
quantize_config: QuantizeConfig,
|
|
166
170
|
trust_remote_code: bool = False,
|
|
167
171
|
**model_init_kwargs,
|
|
168
172
|
) -> BaseGPTQModel:
|
|
169
|
-
|
|
173
|
+
if hasattr(AutoConfig.from_pretrained(model_id_or_path, trust_remote_code=trust_remote_code), "quantization_config"):
|
|
174
|
+
logger.warning("Model is already quantized, will use `from_quantized` to load quantized model.\n"
|
|
175
|
+
"If you want to quantize the model, please pass un_quantized model path or id, and use "
|
|
176
|
+
"`from_pretrained` with `quantize_config`.")
|
|
177
|
+
return cls.from_quantized(model_id_or_path, trust_remote_code=trust_remote_code)
|
|
178
|
+
|
|
179
|
+
model_type = check_and_get_model_type(model_id_or_path, trust_remote_code)
|
|
170
180
|
return MODEL_MAP[model_type].from_pretrained(
|
|
171
|
-
pretrained_model_id_or_path=
|
|
181
|
+
pretrained_model_id_or_path=model_id_or_path,
|
|
172
182
|
quantize_config=quantize_config,
|
|
173
183
|
trust_remote_code=trust_remote_code,
|
|
174
184
|
**model_init_kwargs,
|