gptqmodel 1.2.1.dev0__tar.gz → 1.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. gptqmodel-1.4.1/PKG-INFO +283 -0
  2. gptqmodel-1.4.1/README.md +230 -0
  3. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/__init__.py +2 -1
  4. gptqmodel-1.4.1/gptqmodel/integration/__init__.py +2 -0
  5. gptqmodel-1.4.1/gptqmodel/integration/integration.py +104 -0
  6. gptqmodel-1.4.1/gptqmodel/integration/integration_vllm.py +67 -0
  7. {gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum → gptqmodel-1.4.1/gptqmodel/integration/src/optimum/gptq}/quantizer.py +301 -194
  8. gptqmodel-1.4.1/gptqmodel/integration/src/optimum/utils/__init__.py +0 -0
  9. gptqmodel-1.4.1/gptqmodel/integration/src/optimum/utils/import_utils.py +495 -0
  10. gptqmodel-1.4.1/gptqmodel/integration/src/optimum/utils/testing_utils.py +198 -0
  11. gptqmodel-1.4.1/gptqmodel/integration/src/peft/__init__.py +0 -0
  12. gptqmodel-1.4.1/gptqmodel/integration/src/peft/import_utils.py +131 -0
  13. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/__init__.py +0 -0
  14. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/adalora/__init__.py +0 -0
  15. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/adalora/model.py +357 -0
  16. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/lora/__init__.py +0 -0
  17. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/lora/gptq.py +122 -0
  18. gptqmodel-1.4.1/gptqmodel/integration/src/peft/tuners/lora/model.py +926 -0
  19. gptqmodel-1.4.1/gptqmodel/integration/src/peft/utils/__init__.py +35 -0
  20. gptqmodel-1.4.1/gptqmodel/integration/src/peft/utils/other.py +740 -0
  21. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/__init__.py +0 -0
  22. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/quantizers/__init__.py +0 -0
  23. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/hf_quantizer_gptq.py → gptqmodel-1.4.1/gptqmodel/integration/src/transformers/quantizers/quantizer_gptq.py +37 -28
  24. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/testing_utils.py +2680 -0
  25. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/utils/__init__.py +0 -0
  26. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/utils/import_utils.py +2216 -0
  27. gptqmodel-1.4.1/gptqmodel/integration/src/transformers/utils/quantization_config.py +1367 -0
  28. gptqmodel-1.4.1/gptqmodel/integration/src/vllm/__init__.py +0 -0
  29. gptqmodel-1.4.1/gptqmodel/integration/src/vllm/gptq_marlin.py +606 -0
  30. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/__init__.py +1 -0
  31. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/_const.py +23 -0
  32. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/auto.py +121 -12
  33. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/base.py +152 -265
  34. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/__init__.py +4 -0
  35. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/chatglm.py +4 -2
  36. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gemma2.py +2 -9
  37. gptqmodel-1.4.1/gptqmodel/models/definitions/glm.py +15 -0
  38. gptqmodel-1.4.1/gptqmodel/models/definitions/hymba.py +40 -0
  39. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/internlm.py +1 -1
  40. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/internlm2.py +1 -1
  41. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/mllama.py +1 -1
  42. gptqmodel-1.4.1/gptqmodel/models/definitions/olmo2.py +15 -0
  43. gptqmodel-1.4.1/gptqmodel/models/definitions/qwen2_vl.py +18 -0
  44. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/xverse.py +1 -1
  45. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/loader.py +119 -88
  46. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/writer.py +175 -157
  47. gptqmodel-1.4.1/gptqmodel/nn_modules/__init__.py +0 -0
  48. gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/__init__.py +166 -0
  49. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py → gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/bitblas.py +22 -3
  50. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +7 -40
  51. gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/dynamic_cuda.py +122 -0
  52. gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/exllama.py +213 -0
  53. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py → gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/exllamav2.py +16 -2
  54. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_ipex.py → gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/ipex.py +59 -34
  55. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py → gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/marlin.py +115 -4
  56. gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/torch.py +272 -0
  57. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py → gptqmodel-1.4.1/gptqmodel/nn_modules/qlinear/tritonv2.py +59 -3
  58. gptqmodel-1.4.1/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
  59. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
  60. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/quantization/config.py +43 -28
  61. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/quantization/gptq.py +33 -5
  62. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/quantization/quantizer.py +7 -9
  63. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/__init__.py +1 -0
  64. gptqmodel-1.4.1/gptqmodel/utils/backend.py +23 -0
  65. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/bitblas.py +6 -7
  66. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/device.py +13 -0
  67. gptqmodel-1.4.1/gptqmodel/utils/eval.py +215 -0
  68. gptqmodel-1.4.1/gptqmodel/utils/exllama.py +80 -0
  69. gptqmodel-1.4.1/gptqmodel/utils/importer.py +169 -0
  70. gptqmodel-1.4.1/gptqmodel/utils/logger.py +17 -0
  71. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/marlin.py +6 -6
  72. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/model.py +175 -35
  73. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/perplexity.py +2 -2
  74. gptqmodel-1.4.1/gptqmodel/utils/plotly.py +18 -0
  75. gptqmodel-1.4.1/gptqmodel/utils/progress.py +120 -0
  76. gptqmodel-1.4.1/gptqmodel/utils/vllm.py +106 -0
  77. gptqmodel-1.4.1/gptqmodel/version.py +1 -0
  78. gptqmodel-1.4.1/gptqmodel.egg-info/PKG-INFO +283 -0
  79. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel.egg-info/SOURCES.txt +73 -30
  80. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel.egg-info/requires.txt +14 -2
  81. gptqmodel-1.4.1/gptqmodel.egg-info/top_level.txt +6 -0
  82. gptqmodel-1.4.1/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
  83. gptqmodel-1.4.1/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
  84. gptqmodel-1.4.1/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
  85. gptqmodel-1.4.1/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
  86. gptqmodel-1.4.1/gptqmodel_ext/exllama/cu_compat.cuh +58 -0
  87. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_buffers.cu +75 -0
  88. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_buffers.cuh +55 -0
  89. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/column_remap.cu +63 -0
  90. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/column_remap.cuh +19 -0
  91. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/q4_matmul.cu +260 -0
  92. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/q4_matmul.cuh +43 -0
  93. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/q4_matrix.cu +225 -0
  94. gptqmodel-1.4.1/gptqmodel_ext/exllama/cuda_func/q4_matrix.cuh +53 -0
  95. gptqmodel-1.4.1/gptqmodel_ext/exllama/exllama_ext.cpp +260 -0
  96. gptqmodel-1.4.1/gptqmodel_ext/exllama/hip_compat.cuh +53 -0
  97. gptqmodel-1.4.1/gptqmodel_ext/exllama/matrix.cuh +294 -0
  98. gptqmodel-1.4.1/gptqmodel_ext/exllama/tuning.h +13 -0
  99. gptqmodel-1.4.1/gptqmodel_ext/exllama/util.cuh +33 -0
  100. gptqmodel-1.4.1/requirements.txt +10 -0
  101. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/setup.py +51 -35
  102. gptqmodel-1.4.1/tests/test_asym_gptq_v1.py +19 -0
  103. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_dynamic.py +28 -15
  104. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_estimate_vram.py +3 -2
  105. gptqmodel-1.4.1/tests/test_eval.py +46 -0
  106. gptqmodel-1.4.1/tests/test_evalplus.py +22 -0
  107. gptqmodel-1.4.1/tests/test_group_size.py +129 -0
  108. gptqmodel-1.4.1/tests/test_lm_eval.py +45 -0
  109. gptqmodel-1.4.1/tests/test_lm_head.py +26 -0
  110. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_packing.py +19 -31
  111. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_perplexity.py +0 -1
  112. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_q4_bitblas.py +1 -7
  113. gptqmodel-1.4.1/tests/test_q4_cuda.py +105 -0
  114. gptqmodel-1.2.1.dev0/tests/test_q4_exllama_v2.py → gptqmodel-1.4.1/tests/test_q4_exllama_v1.py +65 -42
  115. gptqmodel-1.4.1/tests/test_q4_exllama_v2.py +138 -0
  116. gptqmodel-1.4.1/tests/test_q4_ipex.py +24 -0
  117. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_q4_marlin.py +5 -64
  118. gptqmodel-1.4.1/tests/test_q4_torch.py +97 -0
  119. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_q4_triton.py +1 -1
  120. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_quant_batch.py +2 -2
  121. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_quant_formats.py +10 -9
  122. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_save_loaded_quantized_model.py +10 -8
  123. gptqmodel-1.4.1/tests/test_transformers_integration.py +73 -0
  124. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_verify_hash.py +8 -2
  125. gptqmodel-1.4.1/tests/test_vllm.py +165 -0
  126. gptqmodel-1.2.1.dev0/PKG-INFO +0 -256
  127. gptqmodel-1.2.1.dev0/README.md +0 -212
  128. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/__init__.py +0 -3
  129. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_base.py +0 -65
  130. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -42
  131. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/data_utils.py +0 -263
  132. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -34
  133. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/language_modeling_task.py +0 -45
  134. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/sequence_classification_task.py +0 -101
  135. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/text_summarization_task.py +0 -70
  136. gptqmodel-1.2.1.dev0/gptqmodel/integration/__init__.py +0 -1
  137. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/__init__.py +0 -2
  138. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/constants.py +0 -23
  139. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/data.py +0 -257
  140. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/utils.py +0 -210
  141. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/__init__.py +0 -109
  142. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +0 -278
  143. gptqmodel-1.2.1.dev0/gptqmodel/utils/backend.py +0 -18
  144. gptqmodel-1.2.1.dev0/gptqmodel/utils/importer.py +0 -71
  145. gptqmodel-1.2.1.dev0/gptqmodel/utils/vllm.py +0 -64
  146. gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
  147. gptqmodel-1.2.1.dev0/gptqmodel.egg-info/PKG-INFO +0 -256
  148. gptqmodel-1.2.1.dev0/gptqmodel.egg-info/top_level.txt +0 -4
  149. gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -80
  150. gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -849
  151. gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -20
  152. gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin/marlin_repack.cu +0 -93
  153. gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin/marlin_repack.cuh +0 -12
  154. gptqmodel-1.2.1.dev0/requirements.txt +0 -18
  155. gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
  156. gptqmodel-1.2.1.dev0/tests/test_lm_eval.py +0 -36
  157. gptqmodel-1.2.1.dev0/tests/test_lm_head.py +0 -49
  158. gptqmodel-1.2.1.dev0/tests/test_pt.py +0 -35
  159. gptqmodel-1.2.1.dev0/tests/test_transformers_integration.py +0 -89
  160. gptqmodel-1.2.1.dev0/tests/test_vllm.py +0 -91
  161. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/LICENSE +0 -0
  162. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/MANIFEST.in +0 -0
  163. {gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils → gptqmodel-1.4.1/gptqmodel/integration/src}/__init__.py +0 -0
  164. {gptqmodel-1.2.1.dev0/gptqmodel/nn_modules → gptqmodel-1.4.1/gptqmodel/integration/src/optimum}/__init__.py +0 -0
  165. {gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/triton_utils → gptqmodel-1.4.1/gptqmodel/integration/src/optimum/gptq}/__init__.py +0 -0
  166. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/baichuan.py +0 -0
  167. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/bloom.py +0 -0
  168. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/codegen.py +0 -0
  169. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/cohere.py +0 -0
  170. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/dbrx.py +0 -0
  171. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
  172. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/decilm.py +0 -0
  173. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
  174. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/exaone.py +0 -0
  175. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gemma.py +0 -0
  176. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gpt2.py +0 -0
  177. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
  178. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gpt_neox.py +0 -0
  179. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/gptj.py +0 -0
  180. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/granite.py +0 -0
  181. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/grinmoe.py +0 -0
  182. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/llama.py +0 -0
  183. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/longllama.py +0 -0
  184. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/minicpm.py +0 -0
  185. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/minicpm3.py +0 -0
  186. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/mistral.py +0 -0
  187. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/mixtral.py +0 -0
  188. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/mobilellm.py +0 -0
  189. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/moss.py +0 -0
  190. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/mpt.py +0 -0
  191. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/opt.py +0 -0
  192. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/phi.py +0 -0
  193. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/phi3.py +0 -0
  194. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/qwen.py +0 -0
  195. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/qwen2.py +0 -0
  196. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
  197. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/rw.py +0 -0
  198. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
  199. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/starcoder2.py +0 -0
  200. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/models/definitions/yi.py +0 -0
  201. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
  202. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
  203. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
  204. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/quantization/__init__.py +0 -0
  205. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/data.py +0 -0
  206. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/sglang.py +0 -0
  207. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel/utils/vram.py +0 -0
  208. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel.egg-info/dependency_links.txt +0 -0
  209. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/config.h +0 -0
  210. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
  211. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
  212. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
  213. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
  214. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
  215. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
  216. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
  217. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
  218. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
  219. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
  220. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
  221. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
  222. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
  223. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
  224. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
  225. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
  226. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
  227. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
  228. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
  229. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin.cuh +0 -0
  230. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_cuda.cpp +0 -0
  231. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_cuda_kernel.cu +0 -0
  232. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_cuda_kernel.cuh +0 -0
  233. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_dtypes.cuh +0 -0
  234. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_repack.cu +0 -0
  235. {gptqmodel-1.2.1.dev0/gptqmodel_ext/marlin_inference → gptqmodel-1.4.1/gptqmodel_ext/marlin}/marlin_repack.cuh +0 -0
  236. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/setup.cfg +0 -0
  237. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_quant_trust_remote.py +0 -0
  238. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_serialization.py +0 -0
  239. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_sglang.py +0 -0
  240. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_sharded.py +0 -0
  241. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_tgi.py +0 -0
  242. {gptqmodel-1.2.1.dev0 → gptqmodel-1.4.1}/tests/test_triton.py +0 -0
@@ -0,0 +1,283 @@
1
+ Metadata-Version: 2.1
2
+ Name: gptqmodel
3
+ Version: 1.4.1
4
+ Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
5
+ Home-page: https://github.com/ModelCloud/GPTQModel
6
+ Author: ModelCloud
7
+ Author-email: qubitium@modelcloud.ai
8
+ Keywords: gptq,quantization,large-language-models,transformers,4bit,llm
9
+ Platform: linux
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: C++
17
+ Classifier: Intended Audience :: Developers
18
+ Classifier: Intended Audience :: Education
19
+ Classifier: Intended Audience :: Science/Research
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Python: >=3.9.0
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Provides-Extra: test
25
+ Requires-Dist: pytest>=8.2.2; extra == "test"
26
+ Requires-Dist: parameterized; extra == "test"
27
+ Provides-Extra: quality
28
+ Requires-Dist: ruff==0.4.9; extra == "quality"
29
+ Requires-Dist: isort==5.13.2; extra == "quality"
30
+ Provides-Extra: vllm
31
+ Requires-Dist: vllm>=0.6.4; extra == "vllm"
32
+ Requires-Dist: flashinfer==0.1.6; extra == "vllm"
33
+ Provides-Extra: sglang
34
+ Requires-Dist: sglang>=0.3.2; extra == "sglang"
35
+ Requires-Dist: flashinfer==0.1.6; extra == "sglang"
36
+ Provides-Extra: bitblas
37
+ Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
38
+ Provides-Extra: hf
39
+ Requires-Dist: optimum>=1.21.2; extra == "hf"
40
+ Provides-Extra: ipex
41
+ Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
42
+ Provides-Extra: auto-round
43
+ Requires-Dist: auto_round>=0.3; extra == "auto-round"
44
+ Provides-Extra: logger
45
+ Requires-Dist: clearml; extra == "logger"
46
+ Requires-Dist: random_word; extra == "logger"
47
+ Requires-Dist: plotly; extra == "logger"
48
+ Provides-Extra: eval
49
+ Requires-Dist: lm_eval>=0.4.6; extra == "eval"
50
+ Requires-Dist: evalplus>=0.3.1; extra == "eval"
51
+ Provides-Extra: triton
52
+ Requires-Dist: triton>=2.0.0; extra == "triton"
53
+
54
+ <h1 align="center">GPTQModel</h1>
55
+ <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
56
+ <p align="center">
57
+ <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
58
+ <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
59
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
60
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
61
+ <a href="https://huggingface.co/modelcloud/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-ModelCloud-%23ff8811.svg"></a>
62
+ </p>
63
+
64
+ ## News
65
+ * 12/13/2024 [1.4.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.4.1): Added Qwen2-VL model support. `mse` quantization control exposed in `QuantizeConfig`. Monkey patch `patch_vllm()` and `patch_hf()` api added to allow Transformers/Optimum/PEFT and vLLM to correctly loaded GPTQModel quantized models while upstream PRs are in pending status.
66
+ * 12/10/2024 [1.4.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.4.0) `EvalPlus` harness integration merged upstream. We now support both `lm-eval` and `EvalPlus`. Added pure torch `Torch` kernel. Refactored `Cuda` kernel to be `DynamicCuda` kernel. `Triton` kernel now auto-padded for max model support. `Dynamic` quantization now supports both positive `+:`:default, and `-:` negative matching which allows matched modules to be skipped entirely for quantization. Fixed auto-`Marlin` kerenl selection. Added auto-kernel fallback for unsupported kernel/module pairs. Lots of internal refractor and cleanup in-preparation for transformers/optimum/peft upstream PR merge. Deprecated the saving of `Marlin` weight format since `Marlin` supports auto conversion of `gptq` format to `Marlin` during runtime.
67
+
68
+ * 11/29/2024 [1.3.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.1) Olmo2 model support. Intel XPU acceleration via IPEX. Model sharding Transformer compat fix due to api deprecation in HF. Removed triton dependency. Triton kernel now optionally dependent on triton pkg.
69
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
70
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
71
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
72
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
73
+
74
+ <details>
75
+
76
+ <summary>Archived News:</summary>
77
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
78
+
79
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
80
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
81
+
82
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
83
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
84
+
85
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
86
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
87
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
88
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
89
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
90
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
91
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
92
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
93
+ Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
94
+ got full end-to-end in/out features padding to enhance current/future model compatibility.
95
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
96
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
97
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
98
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
99
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
100
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
101
+ Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
102
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
103
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
104
+ </details>
105
+
106
+ ## Why should you use GPTQModel?
107
+
108
+ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphed into a full-stand-in replacement with cleaner api, up-to-date model support, faster inference, faster quantization, higher quality quants and a pledge that ModelCloud, together with the open-source ML community, will take every effort to bring the library up-to-date with latest advancements and model support.
109
+
110
+ ## Why GPTQ and not other low-bit quantizers?
111
+
112
+ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has the optimal blend of quality and inference speed you would want to use in a real-world production system.
113
+
114
+ ## Features
115
+ * 🚀 Extensive model support for: `Llama 1-3.3`, `Qwen2-VL`, `Olmo2`, `Hymba`, `GLM`, `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 1-4`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Qwen2MoE`, `DBRX`.
116
+ * 💯 100% CI unit-test coverage for all supported models and kernels including post-quantization quality regression.
117
+ * ✨ `Dynamic`/Mixed quantization control on a per-module basis. Each layer/module can have a unique quantization config or be excluded from quantization all together.
118
+ * 🚀 [vLLM](https://github.com/vllm-project/vllm) and [SGLang](https://github.com/sgl-project/sglang) inference integration for quantized model where format = `FORMAT.GPTQ`
119
+ * 🚀 [Intel/IPEX](https://github.com/intel/intel-extension-for-pytorch) 4bit quantization/inference support on CPU (recent Intel/AMD) and Intel/XPU.
120
+ * 🚀 [Microsoft/BITBLAS](https://github.com/microsoft/BitBLAS) format + dynamically compiled inference.
121
+ * ✨ [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization.
122
+ * ✨ Asymmetric `Sym=False` support via `FORMAT.GPTQ_V2`.
123
+ * ✨ `lm_head` module quant inference support for further VRAM reduction (auto-round only).
124
+ * 🚀 Faster quantization: More than 50% faster for TinyLlama + 4090 with batching and large calibration dataset.
125
+ * ✨ Better quality quants as measured by PPL. (Test config: defaults + `sym=True` + `FORMAT.GPTQ`, TinyLlama)
126
+ * ✨ Model weights sharding support
127
+ * ✨ Security: hash check of model weights on load
128
+ * 🚀 Over 50% faster PPL calculations (OPT)
129
+ * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
130
+
131
+ ## Quality: GPTQModel 4bit can match BF16:
132
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
133
+
134
+ ![image](https://github.com/user-attachments/assets/aab69119-f9c8-4c94-9634-a3c63e57095e)
135
+
136
+ ## Model Support: 🚀 (Added by GPTQModel)
137
+ | Model | | | | | | | | |
138
+ | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- |
139
+ | Baichuan | ✅ | Falcon | ✅ | Llama 1-3.3 | ✅ | OLMo2 | 🚀 | |
140
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Phi 1-4 | 🚀 | |
141
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen | ✅ | |
142
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | Qwen2MoE | 🚀 | |
143
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | Qwen2VL | 🚀 | |
144
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | RefinedWeb | ✅ | |
145
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | StableLM | ✅ | |
146
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | StarCoder2 | ✅ | |
147
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | XVERSE | ✅ | |
148
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | Yi | ✅ | |
149
+
150
+ ## Kernel and HW Accelerator Support
151
+
152
+ GPTQModel is validated for Linux x86_64 with the following devices:
153
+
154
+ | Device | | Optimized Arch | Kernels |
155
+ | ---------------- | --- | -------------- | -------------- |
156
+ | Nvidia GPU | ✅ | Ampere or Higher | Marlin, Exllama V2, Exallma V1, Triton, DyanamicCuda, Torch |
157
+ | Intel/AMD CPU | ✅ | `avx512` or `amx` | IPEX, Torch |
158
+ | Intel XPU | ✅ | Intel Arc + Datacenter Max | IPEX, Torch |
159
+
160
+ ## Install
161
+
162
+ ### PIP/UV
163
+
164
+ ```bash
165
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
166
+ # Example: pip install -v --no-build-isolation gptqmodel[vllm,sglang,bitblas,ipex,auto_round]
167
+ pip install -v gptqmodel --no-build-isolation
168
+ uv pip install -v gptqmodel --no-build-isolation
169
+ ```
170
+
171
+ ### Install from source
172
+
173
+ ```bash
174
+ # clone repo
175
+ git clone https://github.com/ModelCloud/GPTQModel.git && cd GPTQModel
176
+
177
+ # pip: compile and install
178
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
179
+ # Example: pip install -v --no-build-isolation .[vllm,sglang,bitblas,ipex,auto_round]
180
+ pip install -v . --no-build-isolation
181
+ ```
182
+
183
+ ### Quantization and Inference
184
+
185
+ Below is a basic sample using `GPTQModel` to quantize a llm model and perform post-quantization inference:
186
+
187
+ ```py
188
+ from datasets import load_dataset
189
+ from transformers import AutoTokenizer
190
+ from gptqmodel import GPTQModel, QuantizeConfig
191
+
192
+ model_id = "meta-llama/Llama-3.2-1B-Instruct"
193
+ quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit"
194
+
195
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
196
+
197
+ calibration_dataset = [
198
+ tokenizer(example["text"])
199
+ for example in load_dataset(
200
+ "allenai/c4",
201
+ data_files="en/c4-train.00001-of-01024.json.gz",
202
+ split="train"
203
+ ).select(range(1024))
204
+ ]
205
+
206
+ quant_config = QuantizeConfig(bits=4, group_size=128)
207
+
208
+ model = GPTQModel.load(model_id, quant_config)
209
+
210
+ model.quantize(calibration_dataset)
211
+
212
+ model.save(quant_path)
213
+
214
+ model = GPTQModel.load(quant_path)
215
+
216
+ result = model.generate(
217
+ **tokenizer(
218
+ "Uncovering deep insights begins with", return_tensors="pt"
219
+ ).to(model.device)
220
+ )[0]
221
+ ```
222
+
223
+ For more advanced features of model quantization, please reference to [this script](https://github.com/ModelCloud/GPTQModel/blob/main/examples/quantization/basic_usage_wikitext2.py)
224
+
225
+ ### How to Add Support for a New Model
226
+
227
+ Read the [`gptqmodel/models/llama.py`](https://github.com/ModelCloud/GPTQModel/blob/5627f5ffeb3f19b1a2a97e3b6de6fbe668b0dc42/gptqmodel/models/llama.py) code which explains in detail via comments how the model support is defined. Use it as guide to PR for to new models. Most models follow the same pattern.
228
+
229
+ ### Evaluation and Quality Benchmarks
230
+
231
+ GPTQModel inference is integrated into both [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness) and [evalplus](https://github.com/evalplus/evalplus)
232
+ We highly recommend avoid using `ppl` and use `lm-eval`/`evalplus` to validate post-quantization model quality. `ppl` should only be used for regression tests and is not a good indicator of model output quality.
233
+
234
+ ```
235
+ # gptqmodel is integrated into lm-eval >= v0.4.6
236
+ pip install lm-eval>=0.4.6
237
+ ```
238
+
239
+ ```
240
+ # gptqmodel is integrated into evalplus[main]
241
+ pip install -U "evalplus @ git+https://github.com/evalplus/evalplus"
242
+ ```
243
+
244
+ Below is a basic sample using `GPTQModel.eval` API
245
+
246
+ ```py
247
+ from gptqmodel import GPTQModel
248
+ from gptqmodel.utils import EVAL
249
+
250
+ model_id = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"
251
+
252
+ # Use `lm-eval` as framework to evaluate the model
253
+ lm_eval_results = GPTQModel.eval(model_id, framework=EVAL.LM_EVAL, tasks=[EVAL.LM_EVAL.ARC_CHALLENGE], output_file='lm-eval_result.json')
254
+
255
+ # Use `evalplus` as framework to evaluate the model
256
+ evalplus_results = GPTQModel.eval(model_id, framework=EVAL.EVALPLUS, tasks=[EVAL.EVALPLUS.HUMAN], output_file='evalplus_result.json')
257
+ ```
258
+
259
+ ## Citation
260
+ ```
261
+ @misc{gptqmodel,
262
+ author = {ModelCloud.ai},
263
+ title = {GPTQModel},
264
+ year = {2024},
265
+ publisher = {GitHub},
266
+ journal = {GitHub repository},
267
+ howpublished = {\url{https://github.com/modelcloud/gptqmodel}},
268
+ }
269
+
270
+ @article{frantar-gptq,
271
+ title={{GPTQ}: Accurate Post-training Compression for Generative Pretrained Transformers},
272
+ author={Elias Frantar and Saleh Ashkboos and Torsten Hoefler and Dan Alistarh},
273
+ year={2022},
274
+ journal={arXiv preprint arXiv:2210.17323}
275
+ }
276
+
277
+ @article{frantar2024marlin,
278
+ title={MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models},
279
+ author={Frantar, Elias and Castro, Roberto L and Chen, Jiale and Hoefler, Torsten and Alistarh, Dan},
280
+ journal={arXiv preprint arXiv:2408.11743},
281
+ year={2024}
282
+ }
283
+ ```
@@ -0,0 +1,230 @@
1
+ <h1 align="center">GPTQModel</h1>
2
+ <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
3
+ <p align="center">
4
+ <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
5
+ <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
6
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
7
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
8
+ <a href="https://huggingface.co/modelcloud/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-ModelCloud-%23ff8811.svg"></a>
9
+ </p>
10
+
11
+ ## News
12
+ * 12/13/2024 [1.4.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.4.1): Added Qwen2-VL model support. `mse` quantization control exposed in `QuantizeConfig`. Monkey patch `patch_vllm()` and `patch_hf()` api added to allow Transformers/Optimum/PEFT and vLLM to correctly loaded GPTQModel quantized models while upstream PRs are in pending status.
13
+ * 12/10/2024 [1.4.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.4.0) `EvalPlus` harness integration merged upstream. We now support both `lm-eval` and `EvalPlus`. Added pure torch `Torch` kernel. Refactored `Cuda` kernel to be `DynamicCuda` kernel. `Triton` kernel now auto-padded for max model support. `Dynamic` quantization now supports both positive `+:`:default, and `-:` negative matching which allows matched modules to be skipped entirely for quantization. Fixed auto-`Marlin` kerenl selection. Added auto-kernel fallback for unsupported kernel/module pairs. Lots of internal refractor and cleanup in-preparation for transformers/optimum/peft upstream PR merge. Deprecated the saving of `Marlin` weight format since `Marlin` supports auto conversion of `gptq` format to `Marlin` during runtime.
14
+
15
+ * 11/29/2024 [1.3.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.1) Olmo2 model support. Intel XPU acceleration via IPEX. Model sharding Transformer compat fix due to api deprecation in HF. Removed triton dependency. Triton kernel now optionally dependent on triton pkg.
16
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
17
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
18
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
19
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
20
+
21
+ <details>
22
+
23
+ <summary>Archived News:</summary>
24
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
25
+
26
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
27
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
28
+
29
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
30
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
31
+
32
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
33
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
34
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
35
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
36
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
37
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
38
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
39
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
40
+ Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
41
+ got full end-to-end in/out features padding to enhance current/future model compatibility.
42
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
43
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
44
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
45
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
46
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
47
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
48
+ Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
49
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
50
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
51
+ </details>
52
+
53
+ ## Why should you use GPTQModel?
54
+
55
+ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphed into a full-stand-in replacement with cleaner api, up-to-date model support, faster inference, faster quantization, higher quality quants and a pledge that ModelCloud, together with the open-source ML community, will take every effort to bring the library up-to-date with latest advancements and model support.
56
+
57
+ ## Why GPTQ and not other low-bit quantizers?
58
+
59
+ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has the optimal blend of quality and inference speed you would want to use in a real-world production system.
60
+
61
+ ## Features
62
+ * 🚀 Extensive model support for: `Llama 1-3.3`, `Qwen2-VL`, `Olmo2`, `Hymba`, `GLM`, `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 1-4`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Qwen2MoE`, `DBRX`.
63
+ * 💯 100% CI unit-test coverage for all supported models and kernels including post-quantization quality regression.
64
+ * ✨ `Dynamic`/Mixed quantization control on a per-module basis. Each layer/module can have a unique quantization config or be excluded from quantization all together.
65
+ * 🚀 [vLLM](https://github.com/vllm-project/vllm) and [SGLang](https://github.com/sgl-project/sglang) inference integration for quantized model where format = `FORMAT.GPTQ`
66
+ * 🚀 [Intel/IPEX](https://github.com/intel/intel-extension-for-pytorch) 4bit quantization/inference support on CPU (recent Intel/AMD) and Intel/XPU.
67
+ * 🚀 [Microsoft/BITBLAS](https://github.com/microsoft/BitBLAS) format + dynamically compiled inference.
68
+ * ✨ [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization.
69
+ * ✨ Asymmetric `Sym=False` support via `FORMAT.GPTQ_V2`.
70
+ * ✨ `lm_head` module quant inference support for further VRAM reduction (auto-round only).
71
+ * 🚀 Faster quantization: More than 50% faster for TinyLlama + 4090 with batching and large calibration dataset.
72
+ * ✨ Better quality quants as measured by PPL. (Test config: defaults + `sym=True` + `FORMAT.GPTQ`, TinyLlama)
73
+ * ✨ Model weights sharding support
74
+ * ✨ Security: hash check of model weights on load
75
+ * 🚀 Over 50% faster PPL calculations (OPT)
76
+ * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
77
+
78
+ ## Quality: GPTQModel 4bit can match BF16:
79
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
80
+
81
+ ![image](https://github.com/user-attachments/assets/aab69119-f9c8-4c94-9634-a3c63e57095e)
82
+
83
+ ## Model Support: 🚀 (Added by GPTQModel)
84
+ | Model | | | | | | | | |
85
+ | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- |
86
+ | Baichuan | ✅ | Falcon | ✅ | Llama 1-3.3 | ✅ | OLMo2 | 🚀 | |
87
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Phi 1-4 | 🚀 | |
88
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen | ✅ | |
89
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | Qwen2MoE | 🚀 | |
90
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | Qwen2VL | 🚀 | |
91
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | RefinedWeb | ✅ | |
92
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | StableLM | ✅ | |
93
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | StarCoder2 | ✅ | |
94
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | XVERSE | ✅ | |
95
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | Yi | ✅ | |
96
+
97
+ ## Kernel and HW Accelerator Support
98
+
99
+ GPTQModel is validated for Linux x86_64 with the following devices:
100
+
101
+ | Device | | Optimized Arch | Kernels |
102
+ | ---------------- | --- | -------------- | -------------- |
103
+ | Nvidia GPU | ✅ | Ampere or Higher | Marlin, Exllama V2, Exallma V1, Triton, DyanamicCuda, Torch |
104
+ | Intel/AMD CPU | ✅ | `avx512` or `amx` | IPEX, Torch |
105
+ | Intel XPU | ✅ | Intel Arc + Datacenter Max | IPEX, Torch |
106
+
107
+ ## Install
108
+
109
+ ### PIP/UV
110
+
111
+ ```bash
112
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
113
+ # Example: pip install -v --no-build-isolation gptqmodel[vllm,sglang,bitblas,ipex,auto_round]
114
+ pip install -v gptqmodel --no-build-isolation
115
+ uv pip install -v gptqmodel --no-build-isolation
116
+ ```
117
+
118
+ ### Install from source
119
+
120
+ ```bash
121
+ # clone repo
122
+ git clone https://github.com/ModelCloud/GPTQModel.git && cd GPTQModel
123
+
124
+ # pip: compile and install
125
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
126
+ # Example: pip install -v --no-build-isolation .[vllm,sglang,bitblas,ipex,auto_round]
127
+ pip install -v . --no-build-isolation
128
+ ```
129
+
130
+ ### Quantization and Inference
131
+
132
+ Below is a basic sample using `GPTQModel` to quantize a llm model and perform post-quantization inference:
133
+
134
+ ```py
135
+ from datasets import load_dataset
136
+ from transformers import AutoTokenizer
137
+ from gptqmodel import GPTQModel, QuantizeConfig
138
+
139
+ model_id = "meta-llama/Llama-3.2-1B-Instruct"
140
+ quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit"
141
+
142
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
143
+
144
+ calibration_dataset = [
145
+ tokenizer(example["text"])
146
+ for example in load_dataset(
147
+ "allenai/c4",
148
+ data_files="en/c4-train.00001-of-01024.json.gz",
149
+ split="train"
150
+ ).select(range(1024))
151
+ ]
152
+
153
+ quant_config = QuantizeConfig(bits=4, group_size=128)
154
+
155
+ model = GPTQModel.load(model_id, quant_config)
156
+
157
+ model.quantize(calibration_dataset)
158
+
159
+ model.save(quant_path)
160
+
161
+ model = GPTQModel.load(quant_path)
162
+
163
+ result = model.generate(
164
+ **tokenizer(
165
+ "Uncovering deep insights begins with", return_tensors="pt"
166
+ ).to(model.device)
167
+ )[0]
168
+ ```
169
+
170
+ For more advanced features of model quantization, please reference to [this script](https://github.com/ModelCloud/GPTQModel/blob/main/examples/quantization/basic_usage_wikitext2.py)
171
+
172
+ ### How to Add Support for a New Model
173
+
174
+ Read the [`gptqmodel/models/llama.py`](https://github.com/ModelCloud/GPTQModel/blob/5627f5ffeb3f19b1a2a97e3b6de6fbe668b0dc42/gptqmodel/models/llama.py) code which explains in detail via comments how the model support is defined. Use it as guide to PR for to new models. Most models follow the same pattern.
175
+
176
+ ### Evaluation and Quality Benchmarks
177
+
178
+ GPTQModel inference is integrated into both [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness) and [evalplus](https://github.com/evalplus/evalplus)
179
+ We highly recommend avoid using `ppl` and use `lm-eval`/`evalplus` to validate post-quantization model quality. `ppl` should only be used for regression tests and is not a good indicator of model output quality.
180
+
181
+ ```
182
+ # gptqmodel is integrated into lm-eval >= v0.4.6
183
+ pip install lm-eval>=0.4.6
184
+ ```
185
+
186
+ ```
187
+ # gptqmodel is integrated into evalplus[main]
188
+ pip install -U "evalplus @ git+https://github.com/evalplus/evalplus"
189
+ ```
190
+
191
+ Below is a basic sample using `GPTQModel.eval` API
192
+
193
+ ```py
194
+ from gptqmodel import GPTQModel
195
+ from gptqmodel.utils import EVAL
196
+
197
+ model_id = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"
198
+
199
+ # Use `lm-eval` as framework to evaluate the model
200
+ lm_eval_results = GPTQModel.eval(model_id, framework=EVAL.LM_EVAL, tasks=[EVAL.LM_EVAL.ARC_CHALLENGE], output_file='lm-eval_result.json')
201
+
202
+ # Use `evalplus` as framework to evaluate the model
203
+ evalplus_results = GPTQModel.eval(model_id, framework=EVAL.EVALPLUS, tasks=[EVAL.EVALPLUS.HUMAN], output_file='evalplus_result.json')
204
+ ```
205
+
206
+ ## Citation
207
+ ```
208
+ @misc{gptqmodel,
209
+ author = {ModelCloud.ai},
210
+ title = {GPTQModel},
211
+ year = {2024},
212
+ publisher = {GitHub},
213
+ journal = {GitHub repository},
214
+ howpublished = {\url{https://github.com/modelcloud/gptqmodel}},
215
+ }
216
+
217
+ @article{frantar-gptq,
218
+ title={{GPTQ}: Accurate Post-training Compression for Generative Pretrained Transformers},
219
+ author={Elias Frantar and Saleh Ashkboos and Torsten Hoefler and Dan Alistarh},
220
+ year={2022},
221
+ journal={arXiv preprint arXiv:2210.17323}
222
+ }
223
+
224
+ @article{frantar2024marlin,
225
+ title={MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models},
226
+ author={Frantar, Elias and Castro, Roberto L and Chen, Jiale and Hoefler, Torsten and Alistarh, Dan},
227
+ journal={arXiv preprint arXiv:2408.11743},
228
+ year={2024}
229
+ }
230
+ ```
@@ -1,4 +1,5 @@
1
- from .models import GPTQModel
1
+ from .models import GPTQModel, get_best_device
2
2
  from .quantization import BaseQuantizeConfig, QuantizeConfig
3
3
  from .utils import BACKEND, get_backend
4
+ from .utils.exllama import exllama_set_max_input_length
4
5
  from .version import __version__
@@ -0,0 +1,2 @@
1
+ from .integration import patch_hf
2
+ from .integration_vllm import patch_vllm
@@ -0,0 +1,104 @@
1
+ HAS_OPTIMUM = True
2
+ try:
3
+ import optimum.gptq as optimum_gptq
4
+ from optimum.gptq import quantizer as optimum_quantizer
5
+ from optimum.utils import import_utils as optimum_import_utils
6
+ from optimum.utils import testing_utils as optimum_testing_utils
7
+
8
+ from .src.optimum.gptq import quantizer as patched_optimum_quantizer
9
+ from .src.optimum.utils import import_utils as patched_optimum_import_utils
10
+ from .src.optimum.utils import testing_utils as patched_optimum_testing_utils
11
+ except BaseException:
12
+ HAS_OPTIMUM = False
13
+
14
+ HAS_PEFT = True
15
+ try:
16
+ from peft import import_utils as peft_import_utils
17
+ from peft.tuners.adalora.model import AdaLoraModel as peft_AdaLoraModel
18
+ from peft.tuners.lora import gptq as peft_gptq
19
+ from peft.tuners.lora import model as peft_model
20
+ from peft.utils import other as peft_other
21
+
22
+ from .src.peft import import_utils as patched_peft_import_utils
23
+ from .src.peft.tuners.adalora.model import AdaLoraModel as patched_peft_AdaLoraModel
24
+ from .src.peft.tuners.lora import gptq as patched_peft_gptq
25
+ from .src.peft.tuners.lora import model as patched_peft_model
26
+ from .src.peft.utils import other as patched_peft_other
27
+ except BaseException:
28
+ HAS_PEFT = False
29
+
30
+ import transformers.testing_utils as transformers_testing_utils # noqa: E402
31
+ from transformers.quantizers import quantizer_gptq as transformers_quantizer_gptq # noqa: E402
32
+ from transformers.utils import import_utils as transformers_import_utils # noqa: E402
33
+ from transformers.utils import quantization_config as transformers_quantization_config # noqa: E402
34
+
35
+ from .src.transformers import testing_utils as patched_transformers_testing_utils # noqa: E402
36
+ from .src.transformers.quantizers import quantizer_gptq as patched_transformers_quantizer_gptq # noqa: E402
37
+ from .src.transformers.utils import import_utils as patched_transformers_import_utils # noqa: E402
38
+ from .src.transformers.utils import quantization_config as patched_transformers_quantization_config # noqa: E402
39
+
40
+
41
+ def patch_hf():
42
+ _patch_peft()
43
+ _patch_optimum()
44
+ _patch_transformers()
45
+
46
+
47
+ def _patch_peft():
48
+ if not HAS_PEFT:
49
+ return
50
+
51
+ peft_import_utils.is_gptqmodel_available = patched_peft_import_utils.is_gptqmodel_available
52
+
53
+ peft_AdaLoraModel._create_and_replace = patched_peft_AdaLoraModel._create_and_replace
54
+
55
+ peft_gptq.dispatch_gptq = patched_peft_gptq.dispatch_gptq
56
+
57
+ peft_model.LoraModel = patched_peft_model.LoraModel
58
+
59
+ peft_other.get_auto_gptq_quant_linear = patched_peft_other.get_auto_gptq_quant_linear
60
+ peft_other.get_gptqmodel_quant_linear = patched_peft_other.get_gptqmodel_quant_linear
61
+
62
+
63
+ def _patch_optimum():
64
+ if not HAS_OPTIMUM:
65
+ return
66
+
67
+ optimum_gptq.GPTQQuantizer = patched_optimum_quantizer.GPTQQuantizer
68
+ optimum_quantizer.is_gptqmodel_available = patched_optimum_quantizer.is_gptqmodel_available
69
+ optimum_quantizer.has_device_more_than_cpu = patched_optimum_quantizer.has_device_more_than_cpu
70
+ optimum_quantizer.ExllamaVersion = patched_optimum_quantizer.ExllamaVersion
71
+
72
+ optimum_import_utils._gptqmodel_available = patched_optimum_import_utils._gptqmodel_available
73
+ optimum_import_utils.is_gptqmodel_available = patched_optimum_import_utils.is_gptqmodel_available
74
+ optimum_testing_utils.require_gptq = patched_optimum_testing_utils.require_gptq
75
+
76
+
77
+ def _patch_transformers():
78
+ transformers_quantizer_gptq.GptqHfQuantizer.required_packages = patched_transformers_quantizer_gptq.GptqHfQuantizer.required_packages
79
+ transformers_quantizer_gptq.GptqHfQuantizer.validate_environment = patched_transformers_quantizer_gptq.GptqHfQuantizer.validate_environment
80
+ transformers_quantizer_gptq.GptqHfQuantizer._process_model_before_weight_loading = patched_transformers_quantizer_gptq.GptqHfQuantizer._process_model_before_weight_loading
81
+
82
+ transformers_import_utils._gptqmodel_available = patched_transformers_import_utils._gptqmodel_available
83
+ transformers_import_utils.is_gptqmodel_available = patched_transformers_import_utils.is_gptqmodel_available
84
+
85
+ transformers_quantization_config.GPTQConfig.__init__ = patched_transformers_quantization_config.GPTQConfig.__init__
86
+ transformers_quantization_config.GPTQConfig.post_init = patched_transformers_quantization_config.GPTQConfig.post_init
87
+
88
+ transformers_testing_utils.require_gptq = patched_transformers_testing_utils.require_gptq
89
+
90
+ # if 'transformers.quantizers.quantizer_gptq' in sys.modules:
91
+ # del sys.modules['transformers.quantizers.quantizer_gptq']
92
+ # sys.modules['transformers.quantizers.quantizer_gptq'] = patched_transformers_quantizer_gptq
93
+ #
94
+ # if 'transformers.utils.import_utils' in sys.modules:
95
+ # del sys.modules['transformers.utils.import_utils']
96
+ # sys.modules['transformers.utils.import_utils'] = patched_transformers_import_utils
97
+ #
98
+ # if 'transformers.utils.quantization_config' in sys.modules:
99
+ # del sys.modules['transformers.utils.quantization_config']
100
+ # sys.modules['transformers.utils.quantization_config'] = patched_transformers_quantization_config
101
+ #
102
+ # if 'transformers.testing_utils' in sys.modules:
103
+ # del sys.modules['transformers.testing_utils']
104
+ # sys.modules['transformers.testing_utils'] = patched_transformers_testing_utils