gptqmodel 1.2.1.dev0__tar.gz → 1.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. {gptqmodel-1.2.1.dev0/gptqmodel.egg-info → gptqmodel-1.3.1}/PKG-INFO +68 -63
  2. gptqmodel-1.3.1/README.md +208 -0
  3. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/__init__.py +2 -1
  4. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/__init__.py +1 -0
  5. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/_const.py +22 -0
  6. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/auto.py +25 -10
  7. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/base.py +115 -256
  8. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/__init__.py +3 -0
  9. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/chatglm.py +4 -2
  10. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gemma2.py +2 -9
  11. gptqmodel-1.3.1/gptqmodel/models/definitions/glm.py +15 -0
  12. gptqmodel-1.3.1/gptqmodel/models/definitions/hymba.py +39 -0
  13. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/internlm.py +1 -1
  14. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/internlm2.py +1 -1
  15. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/mllama.py +1 -1
  16. gptqmodel-1.3.1/gptqmodel/models/definitions/olmo2.py +15 -0
  17. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/xverse.py +1 -1
  18. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/loader.py +110 -83
  19. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/writer.py +160 -155
  20. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +3 -2
  21. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py +3 -2
  22. gptqmodel-1.3.1/gptqmodel/nn_modules/qlinear/qlinear_cuda.py +304 -0
  23. gptqmodel-1.3.1/gptqmodel/nn_modules/qlinear/qlinear_exllama.py +199 -0
  24. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py +3 -2
  25. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_ipex.py +25 -13
  26. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +3 -3
  27. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py +17 -3
  28. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
  29. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/quantization/config.py +8 -13
  30. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/quantization/gptq.py +2 -2
  31. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/quantization/quantizer.py +3 -3
  32. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/backend.py +9 -7
  33. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/bitblas.py +5 -6
  34. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/device.py +13 -0
  35. gptqmodel-1.3.1/gptqmodel/utils/exllama.py +80 -0
  36. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/importer.py +40 -11
  37. gptqmodel-1.3.1/gptqmodel/utils/lm_eval.py +125 -0
  38. gptqmodel-1.3.1/gptqmodel/utils/logger.py +17 -0
  39. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/marlin.py +4 -4
  40. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/model.py +90 -12
  41. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/perplexity.py +2 -2
  42. gptqmodel-1.3.1/gptqmodel/utils/plotly.py +18 -0
  43. gptqmodel-1.3.1/gptqmodel/utils/progress.py +120 -0
  44. gptqmodel-1.3.1/gptqmodel/version.py +1 -0
  45. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1/gptqmodel.egg-info}/PKG-INFO +68 -63
  46. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel.egg-info/SOURCES.txt +31 -19
  47. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel.egg-info/requires.txt +13 -1
  48. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel.egg-info/top_level.txt +3 -0
  49. gptqmodel-1.3.1/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
  50. gptqmodel-1.3.1/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
  51. gptqmodel-1.3.1/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
  52. gptqmodel-1.3.1/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
  53. gptqmodel-1.3.1/gptqmodel_ext/exllama/cu_compat.cuh +58 -0
  54. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_buffers.cu +75 -0
  55. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_buffers.cuh +55 -0
  56. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/column_remap.cu +63 -0
  57. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/column_remap.cuh +19 -0
  58. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/q4_matmul.cu +260 -0
  59. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/q4_matmul.cuh +43 -0
  60. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/q4_matrix.cu +225 -0
  61. gptqmodel-1.3.1/gptqmodel_ext/exllama/cuda_func/q4_matrix.cuh +53 -0
  62. gptqmodel-1.3.1/gptqmodel_ext/exllama/exllama_ext.cpp +260 -0
  63. gptqmodel-1.3.1/gptqmodel_ext/exllama/hip_compat.cuh +53 -0
  64. gptqmodel-1.3.1/gptqmodel_ext/exllama/matrix.cuh +294 -0
  65. gptqmodel-1.3.1/gptqmodel_ext/exllama/tuning.h +13 -0
  66. gptqmodel-1.3.1/gptqmodel_ext/exllama/util.cuh +33 -0
  67. gptqmodel-1.3.1/requirements.txt +10 -0
  68. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/setup.py +36 -1
  69. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_dynamic.py +5 -4
  70. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_estimate_vram.py +3 -2
  71. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_lm_eval.py +6 -3
  72. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_lm_head.py +1 -1
  73. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_packing.py +48 -0
  74. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_q4_bitblas.py +1 -7
  75. gptqmodel-1.3.1/tests/test_q4_cuda.py +107 -0
  76. gptqmodel-1.2.1.dev0/tests/test_q4_exllama_v2.py → gptqmodel-1.3.1/tests/test_q4_exllama_v1.py +65 -42
  77. gptqmodel-1.3.1/tests/test_q4_exllama_v2.py +138 -0
  78. gptqmodel-1.3.1/tests/test_q4_ipex.py +24 -0
  79. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_q4_marlin.py +5 -4
  80. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_quant_batch.py +1 -1
  81. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_quant_formats.py +3 -3
  82. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_save_loaded_quantized_model.py +10 -8
  83. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_verify_hash.py +8 -2
  84. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_vllm.py +4 -3
  85. gptqmodel-1.2.1.dev0/README.md +0 -212
  86. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/__init__.py +0 -3
  87. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_base.py +0 -65
  88. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -42
  89. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/data_utils.py +0 -263
  90. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -34
  91. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/language_modeling_task.py +0 -45
  92. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/sequence_classification_task.py +0 -101
  93. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/text_summarization_task.py +0 -70
  94. gptqmodel-1.2.1.dev0/gptqmodel/integration/__init__.py +0 -1
  95. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/__init__.py +0 -2
  96. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/constants.py +0 -23
  97. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/data.py +0 -257
  98. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/hf_quantizer_gptq.py +0 -99
  99. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/quantizer.py +0 -813
  100. gptqmodel-1.2.1.dev0/gptqmodel/integration/optimum/utils.py +0 -210
  101. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
  102. gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
  103. gptqmodel-1.2.1.dev0/requirements.txt +0 -18
  104. gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
  105. gptqmodel-1.2.1.dev0/tests/test_pt.py +0 -35
  106. gptqmodel-1.2.1.dev0/tests/test_transformers_integration.py +0 -89
  107. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/LICENSE +0 -0
  108. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/MANIFEST.in +0 -0
  109. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/baichuan.py +0 -0
  110. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/bloom.py +0 -0
  111. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/codegen.py +0 -0
  112. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/cohere.py +0 -0
  113. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/dbrx.py +0 -0
  114. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
  115. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/decilm.py +0 -0
  116. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
  117. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/exaone.py +0 -0
  118. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gemma.py +0 -0
  119. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gpt2.py +0 -0
  120. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
  121. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gpt_neox.py +0 -0
  122. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/gptj.py +0 -0
  123. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/granite.py +0 -0
  124. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/grinmoe.py +0 -0
  125. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/llama.py +0 -0
  126. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/longllama.py +0 -0
  127. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/minicpm.py +0 -0
  128. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/minicpm3.py +0 -0
  129. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/mistral.py +0 -0
  130. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/mixtral.py +0 -0
  131. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/mobilellm.py +0 -0
  132. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/moss.py +0 -0
  133. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/mpt.py +0 -0
  134. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/opt.py +0 -0
  135. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/phi.py +0 -0
  136. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/phi3.py +0 -0
  137. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/qwen.py +0 -0
  138. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/qwen2.py +0 -0
  139. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
  140. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/rw.py +0 -0
  141. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
  142. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/starcoder2.py +0 -0
  143. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/models/definitions/yi.py +0 -0
  144. {gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils → gptqmodel-1.3.1/gptqmodel/nn_modules}/__init__.py +0 -0
  145. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/__init__.py +0 -0
  146. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py +0 -0
  147. {gptqmodel-1.2.1.dev0/gptqmodel/nn_modules → gptqmodel-1.3.1/gptqmodel/nn_modules/triton_utils}/__init__.py +0 -0
  148. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
  149. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
  150. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
  151. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/quantization/__init__.py +0 -0
  152. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/__init__.py +0 -0
  153. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/data.py +0 -0
  154. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/sglang.py +0 -0
  155. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/vllm.py +0 -0
  156. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel/utils/vram.py +0 -0
  157. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel.egg-info/dependency_links.txt +0 -0
  158. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/config.h +0 -0
  159. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
  160. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
  161. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
  162. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
  163. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
  164. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
  165. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
  166. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
  167. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
  168. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
  169. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
  170. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
  171. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
  172. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
  173. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
  174. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
  175. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
  176. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
  177. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
  178. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -0
  179. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -0
  180. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -0
  181. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin/marlin_repack.cu +0 -0
  182. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin/marlin_repack.cuh +0 -0
  183. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin.cuh +0 -0
  184. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_cuda.cpp +0 -0
  185. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cu +0 -0
  186. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cuh +0 -0
  187. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_dtypes.cuh +0 -0
  188. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_repack.cu +0 -0
  189. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/gptqmodel_ext/marlin_inference/marlin_repack.cuh +0 -0
  190. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/setup.cfg +0 -0
  191. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_perplexity.py +0 -0
  192. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_q4_triton.py +0 -0
  193. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_quant_trust_remote.py +0 -0
  194. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_serialization.py +0 -0
  195. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_sglang.py +0 -0
  196. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_sharded.py +0 -0
  197. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_tgi.py +0 -0
  198. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.1}/tests/test_triton.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: gptqmodel
3
- Version: 1.2.1.dev0
3
+ Version: 1.3.1
4
4
  Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
5
5
  Home-page: https://github.com/ModelCloud/GPTQModel
6
6
  Author: ModelCloud
@@ -34,64 +34,66 @@ Provides-Extra: sglang
34
34
  Requires-Dist: sglang>=0.3.2; extra == "sglang"
35
35
  Requires-Dist: flashinfer==0.1.6; extra == "sglang"
36
36
  Provides-Extra: bitblas
37
- Requires-Dist: bitblas>=0.0.1.dev13; extra == "bitblas"
37
+ Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
38
38
  Provides-Extra: hf
39
39
  Requires-Dist: optimum>=1.21.2; extra == "hf"
40
40
  Provides-Extra: ipex
41
41
  Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
42
42
  Provides-Extra: auto-round
43
43
  Requires-Dist: auto_round>=0.3; extra == "auto-round"
44
+ Provides-Extra: logger
45
+ Requires-Dist: clearml; extra == "logger"
46
+ Requires-Dist: random_word; extra == "logger"
47
+ Requires-Dist: device-smi; extra == "logger"
48
+ Requires-Dist: plotly; extra == "logger"
49
+ Provides-Extra: eval
50
+ Requires-Dist: lm_eval>=0.4.4; extra == "eval"
51
+ Provides-Extra: triton
52
+ Requires-Dist: triton>=2.0.0; extra == "triton"
44
53
 
45
54
  <h1 align="center">GPTQModel</h1>
46
55
  <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
47
56
  <p align="center">
48
57
  <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
49
58
  <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
50
- <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/gptqmodel"></a>
59
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
60
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
51
61
  </p>
52
62
 
53
63
  ## News
54
- * 11/11/2024 🚀 [1.2.0] Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
55
- * 10/29/2024 🚀 [v1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
56
- * 10/12/2024 ✨ [v1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
57
- * 10/11/2024 ✨ [v1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
58
- * 10/08/2024 ✨ [v1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
59
- * 09/26/2024 ✨ [v1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
60
- * 09/26/2024 ✨ [v1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
61
- * 09/26/2024 ✨ [v1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
62
- * 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
63
- * 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
64
- * 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
64
+ * 11/29/2024 [1.3.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.1) Olmo2 model support. Intel XPU acceleration via IPEX. Model sharding Transformer compat fix due to api deprecation in HF. Removed triton dependency. Triton kernel now optionally dependent on triton pkg.
65
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
66
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
67
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
68
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
69
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
70
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
71
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
72
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
73
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
65
74
 
66
75
  <details>
67
76
 
68
77
  <summary>Archived News:</summary>
69
- * 08/10/2024 🚀 [v0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
70
-
71
- * 07/31/2024 🚀 [v0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
72
-
73
- * 07/25/2024 🚀 [v0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
74
-
75
- * 07/13/2024 🚀 [v0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
78
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
79
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
80
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
81
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
82
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
83
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
84
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
85
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
76
86
  Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
77
87
  got full end-to-end in/out features padding to enhance current/future model compatibility.
78
-
79
- * 07/08/2024 🚀 [v0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
80
-
81
- * 07/08/2024 🚀 [v0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
82
-
83
- * 07/05/2024 🚀 [v0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
84
-
85
- * 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
86
-
87
- * 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
88
-
89
- * 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
88
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
89
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
90
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
91
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
92
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
93
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
90
94
  Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
91
-
92
- * 06/29/2024 🚀 [v0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
93
-
94
- * 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
95
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
96
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
95
97
  </details>
96
98
 
97
99
  ## Why should you use GPTQModel?
@@ -103,7 +105,7 @@ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphe
103
105
  Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
104
106
 
105
107
  ## Features
106
- * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted).
108
+ * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
107
109
  * ✨ 100% CI coverage for all supported models including quality/ppl regression.
108
110
  * 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
109
111
  * 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
@@ -119,32 +121,34 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
119
121
  * 🚀 Over 50% faster PPL calculations (OPT)
120
122
  * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
121
123
 
124
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
125
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
122
126
 
123
- ## Model Support: 🚀 (Added by GPTQModel)
124
- [🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
125
-
126
-
127
- | Model | | | | | | | | | | |
128
- | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
129
- | Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
130
- | Bloom | ✅ | Gemma 2 | 🚀 | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
131
- | ChatGLM | 🚀 | GPTBigCod | ✅ | MiniCPM3 | 🚀 | RefinedWeb | ✅ | | | |
132
- | CodeGen | ✅ | GPTNeoX | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
133
- | Cohere | ✅ | GPT-2 | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
134
- | DBRX Converted | 🚀 | GPT-J | ✅ | MobileLLM | 🚀 | XVERSE | ✅ | | | |
135
- | Deci | ✅ | Granite | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
136
- | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MPT | ✅ | | | | | |
137
- | DeepSeek-V2-Lite | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |
138
- | EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
127
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
139
128
 
140
-
141
- ## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
142
- ![plus-v2 5v3@2x](https://github.com/user-attachments/assets/dd223d08-f790-41ff-814d-4c2a98e4a3de)
129
+ ## Model Support: 🚀 (Added by GPTQModel)
130
+ | Model | | | | | | | |
131
+ | ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- |
132
+ | Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | OLMo2 | 🚀 |
133
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Phi/Phi-3 | 🚀 |
134
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen | ✅ |
135
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | Qwen2MoE | 🚀 |
136
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | RefinedWeb | ✅ |
137
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StableLM | ✅ |
138
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | StarCoder2 | ✅ |
139
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | XVERSE | ✅ |
140
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | Yi | ✅ |
141
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | |
143
142
 
144
143
 
145
144
  ## Platform Requirements
146
145
 
147
- GPTQModel is validated for Linux x86_64 with Nvidia GPUs. Windows WSL2 may work but un-tested.
146
+ GPTQModel is validated for Linux x86_64 with the following devices:
147
+ | NV GPU | ✅ |
148
+ | Intel CPU | ✅ |
149
+ | Intel GPU | ✅ |
150
+
151
+ Windows WSL2 may work but un-tested.
148
152
 
149
153
  ## Install
150
154
 
@@ -165,7 +169,7 @@ git clone https://github.com/ModelCloud/GPTQModel.git && cd GPTQModel
165
169
 
166
170
  # pip: compile and install
167
171
  # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
168
- # Example: pip install -v --no-build-isolation gptqmodel[vllm,sglang,bitblas,ipex,auto_round]
172
+ # Example: pip install -v --no-build-isolation .[vllm,sglang,bitblas,ipex,auto_round]
169
173
  pip install -v . --no-build-isolation
170
174
  ```
171
175
 
@@ -176,7 +180,7 @@ Below is a basic sample using `GPTQModel` to quantize a llm model and perform po
176
180
  ```py
177
181
  from datasets import load_dataset
178
182
  from transformers import AutoTokenizer
179
- from gptqmodel import GPTQModel, QuantizeConfig
183
+ from gptqmodel import GPTQModel, QuantizeConfig, get_best_device
180
184
 
181
185
  model_id = "meta-llama/Llama-3.2-1B-Instruct"
182
186
  quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit"
@@ -200,7 +204,7 @@ model.quantize(calibration_dataset)
200
204
 
201
205
  model.save(quant_path)
202
206
 
203
- model = GPTQModel.load(quant_path)
207
+ model = GPTQModel.load(quant_path, device=get_best_device())
204
208
 
205
209
  result = model.generate(
206
210
  **tokenizer(
@@ -226,8 +230,9 @@ pip install lm-eval[gptqmodel]
226
230
 
227
231
  ### Which kernel is used by default?
228
232
 
229
- * `GPU`: Marlin, Exllama v2, Triton kernels in that order for maximum inference performance. Optional Microsoft/BITBLAS kernel can be toggled.
230
- * `CPU`: Intel/IPEX kernel
233
+ * `GPU`: Marlin, Exllama v2, Triton kernels in that order for maximum inference performance. Optional Microsoft/BITBLAS kernel can be toggled.
234
+ * `CPU`: Intel/IPEX kernel
235
+ * `XPU`: Intel/IPEX kernel
231
236
 
232
237
  ## Citation
233
238
  ```
@@ -0,0 +1,208 @@
1
+ <h1 align="center">GPTQModel</h1>
2
+ <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
3
+ <p align="center">
4
+ <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
5
+ <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
6
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
7
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
8
+ </p>
9
+
10
+ ## News
11
+ * 11/29/2024 [1.3.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.1) Olmo2 model support. Intel XPU acceleration via IPEX. Model sharding Transformer compat fix due to api deprecation in HF. Removed triton dependency. Triton kernel now optionally dependent on triton pkg.
12
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
13
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
14
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
15
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
16
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
17
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
18
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
19
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
20
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
21
+
22
+ <details>
23
+
24
+ <summary>Archived News:</summary>
25
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
26
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
27
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
28
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
29
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
30
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
31
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
32
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
33
+ Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
34
+ got full end-to-end in/out features padding to enhance current/future model compatibility.
35
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
36
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
37
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
38
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
39
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
40
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
41
+ Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
42
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
43
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
44
+ </details>
45
+
46
+ ## Why should you use GPTQModel?
47
+
48
+ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphed into a full-stand-in replacement with cleaner api, up-to-date model support, faster inference, faster quantization, higher quality quants and a pledge that ModelCloud, together with the open-source ML community, will take every effort to bring the library up-to-date with latest advancements and model support.
49
+
50
+ ## Why GPTQ specifically and not the dozens of other low-bit quantizers?
51
+
52
+ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
53
+
54
+ ## Features
55
+ * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
56
+ * ✨ 100% CI coverage for all supported models including quality/ppl regression.
57
+ * 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
58
+ * 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
59
+ * 🚀 [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
60
+ * 🚀 [Intel/IPEX](https://github.com/intel/intel-extension-for-pytorch) support added for 4 bit quantization/inference on CPU.
61
+ * 🚀 [BITBLAS](https://github.com/microsoft/BitBLAS) format/inference support from Microsoft
62
+ * 🚀`Sym=False` Support. AutoGPTQ has unusable `sym=false`. (Re-quant required)
63
+ * 🚀`lm_head` module quant inference support for further VRAM reduction.
64
+ * 🚀 Faster quantization: More than 50% faster for TinyLlama + 4090 with batching and large calibration dataset.
65
+ * 🚀 Better quality quants as measured by PPL. (Test config: defaults + `sym=True` + `FORMAT.GPTQ`, TinyLlama)
66
+ * 🚀 Model weights sharding support
67
+ * 🚀 Security: hash check of model weights on load
68
+ * 🚀 Over 50% faster PPL calculations (OPT)
69
+ * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
70
+
71
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
72
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
73
+
74
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
75
+
76
+ ## Model Support: 🚀 (Added by GPTQModel)
77
+ | Model | | | | | | | |
78
+ | ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- |
79
+ | Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | OLMo2 | 🚀 |
80
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Phi/Phi-3 | 🚀 |
81
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen | ✅ |
82
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | Qwen2MoE | 🚀 |
83
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | RefinedWeb | ✅ |
84
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StableLM | ✅ |
85
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | StarCoder2 | ✅ |
86
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | XVERSE | ✅ |
87
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | Yi | ✅ |
88
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | |
89
+
90
+
91
+ ## Platform Requirements
92
+
93
+ GPTQModel is validated for Linux x86_64 with the following devices:
94
+ | NV GPU | ✅ |
95
+ | Intel CPU | ✅ |
96
+ | Intel GPU | ✅ |
97
+
98
+ Windows WSL2 may work but un-tested.
99
+
100
+ ## Install
101
+
102
+ ### PIP/UV
103
+
104
+ ```bash
105
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
106
+ # Example: pip install -v --no-build-isolation gptqmodel[vllm,sglang,bitblas,ipex,auto_round]
107
+ pip install -v gptqmodel --no-build-isolation
108
+ uv pip install -v gptqmodel --no-build-isolation
109
+ ```
110
+
111
+ ### Install from source
112
+
113
+ ```bash
114
+ # clone repo
115
+ git clone https://github.com/ModelCloud/GPTQModel.git && cd GPTQModel
116
+
117
+ # pip: compile and install
118
+ # You can install optional modules like autoround, ipex, vllm, sglang, bitblas, and ipex.
119
+ # Example: pip install -v --no-build-isolation .[vllm,sglang,bitblas,ipex,auto_round]
120
+ pip install -v . --no-build-isolation
121
+ ```
122
+
123
+ ### Quantization and Inference
124
+
125
+ Below is a basic sample using `GPTQModel` to quantize a llm model and perform post-quantization inference:
126
+
127
+ ```py
128
+ from datasets import load_dataset
129
+ from transformers import AutoTokenizer
130
+ from gptqmodel import GPTQModel, QuantizeConfig, get_best_device
131
+
132
+ model_id = "meta-llama/Llama-3.2-1B-Instruct"
133
+ quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit"
134
+
135
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
136
+
137
+ calibration_dataset = [
138
+ tokenizer(example["text"])
139
+ for example in load_dataset(
140
+ "allenai/c4",
141
+ data_files="en/c4-train.00001-of-01024.json.gz",
142
+ split="train"
143
+ ).select(range(1024))
144
+ ]
145
+
146
+ quant_config = QuantizeConfig(bits=4, group_size=128)
147
+
148
+ model = GPTQModel.load(model_id, quant_config)
149
+
150
+ model.quantize(calibration_dataset)
151
+
152
+ model.save(quant_path)
153
+
154
+ model = GPTQModel.load(quant_path, device=get_best_device())
155
+
156
+ result = model.generate(
157
+ **tokenizer(
158
+ "Uncovering deep insights begins with", return_tensors="pt"
159
+ ).to(model.device)
160
+ )[0]
161
+ ```
162
+
163
+ For more advanced features of model quantization, please reference to [this script](https://github.com/ModelCloud/GPTQModel/blob/main/examples/quantization/basic_usage_wikitext2.py)
164
+
165
+ ### How to Add Support for a New Model
166
+
167
+ Read the [`gptqmodel/models/llama.py`](https://github.com/ModelCloud/GPTQModel/blob/5627f5ffeb3f19b1a2a97e3b6de6fbe668b0dc42/gptqmodel/models/llama.py) code which explains in detail via comments how the model support is defined. Use it as guide to PR for to new models. Most models follow the same pattern.
168
+
169
+ ### Evaluation and Quality Benchmarks
170
+
171
+ GPTQModel inference is integrated into [lm-evaluation-hardness](https://github.com/EleutherAI/lm-evaluation-harness) and we highly recommend avoid using PPL and use `lm-eval` to validate post-quantization model quality.
172
+
173
+ ```
174
+ # currently gptqmodel is merged into lm-eval main but not yet released on pypi
175
+ pip install lm-eval[gptqmodel]
176
+ ```
177
+
178
+ ### Which kernel is used by default?
179
+
180
+ * `GPU`: Marlin, Exllama v2, Triton kernels in that order for maximum inference performance. Optional Microsoft/BITBLAS kernel can be toggled.
181
+ * `CPU`: Intel/IPEX kernel
182
+ * `XPU`: Intel/IPEX kernel
183
+
184
+ ## Citation
185
+ ```
186
+ @misc{gptqmodel,
187
+ author = {ModelCloud.ai},
188
+ title = {GPTQModel},
189
+ year = {2024},
190
+ publisher = {GitHub},
191
+ journal = {GitHub repository},
192
+ howpublished = {\url{https://github.com/modelcloud/gptqmodel}},
193
+ }
194
+
195
+ @article{frantar-gptq,
196
+ title={{GPTQ}: Accurate Post-training Compression for Generative Pretrained Transformers},
197
+ author={Elias Frantar and Saleh Ashkboos and Torsten Hoefler and Dan Alistarh},
198
+ year={2022},
199
+ journal={arXiv preprint arXiv:2210.17323}
200
+ }
201
+
202
+ @article{frantar2024marlin,
203
+ title={MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models},
204
+ author={Frantar, Elias and Castro, Roberto L and Chen, Jiale and Hoefler, Torsten and Alistarh, Dan},
205
+ journal={arXiv preprint arXiv:2408.11743},
206
+ year={2024}
207
+ }
208
+ ```
@@ -1,4 +1,5 @@
1
- from .models import GPTQModel
1
+ from .models import GPTQModel, get_best_device
2
2
  from .quantization import BaseQuantizeConfig, QuantizeConfig
3
3
  from .utils import BACKEND, get_backend
4
+ from .utils.exllama import exllama_set_max_input_length
4
5
  from .version import __version__
@@ -1,3 +1,4 @@
1
+ from ._const import get_best_device
1
2
  from .auto import MODEL_MAP, GPTQModel
2
3
  from .base import BaseGPTQModel
3
4
  from .definitions import *
@@ -1,14 +1,24 @@
1
1
  from enum import Enum
2
2
 
3
+ import torch
3
4
  from torch import device
4
5
 
6
+ from ..utils import BACKEND
7
+
5
8
  CPU = device("cpu")
6
9
  CUDA = device("cuda")
7
10
  CUDA_0 = device("cuda:0")
11
+ XPU = device("xpu")
12
+ XPU_0 = device("xpu:0")
8
13
 
9
14
  class DEVICE(Enum):
10
15
  CPU = "cpu"
11
16
  CUDA = "cuda"
17
+ XPU = "xpu"
18
+
19
+
20
+ def is_torch_support_xpu():
21
+ return hasattr(torch, "xpu") and torch.xpu.is_available()
12
22
 
13
23
 
14
24
  def get_device_by_type(type_value: str):
@@ -17,6 +27,15 @@ def get_device_by_type(type_value: str):
17
27
  return enum_constant
18
28
  raise ValueError(f"Invalid type_value str: {type_value}")
19
29
 
30
+
31
+ def get_best_device(beckend=BACKEND.AUTO):
32
+ if torch.cuda.is_available() and beckend != BACKEND.IPEX:
33
+ return CUDA_0
34
+ elif is_torch_support_xpu():
35
+ return XPU_0
36
+ else:
37
+ return CPU
38
+
20
39
  SUPPORTED_MODELS = [
21
40
  "bloom",
22
41
  "gptj",
@@ -27,6 +46,7 @@ SUPPORTED_MODELS = [
27
46
  "gpt_bigcode",
28
47
  "codegen",
29
48
  "chatglm",
49
+ "glm",
30
50
  "RefinedWebModel",
31
51
  "RefinedWeb",
32
52
  "baichuan",
@@ -60,6 +80,8 @@ SUPPORTED_MODELS = [
60
80
  "mllama",
61
81
  "granite",
62
82
  "mobilellm",
83
+ "hymba",
84
+ "olmo2",
63
85
  ]
64
86
 
65
87
  EXLLAMA_DEFAULT_MAX_INPUT_LENGTH = 2048