gptqmodel 1.2.1.dev0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. {gptqmodel-1.2.1.dev0/gptqmodel.egg-info → gptqmodel-1.3.0}/PKG-INFO +52 -56
  2. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/README.md +43 -54
  3. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/quantizer.py +19 -5
  4. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/utils.py +3 -2
  5. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/_const.py +2 -0
  6. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/auto.py +18 -6
  7. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/base.py +102 -248
  8. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/__init__.py +2 -0
  9. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/chatglm.py +4 -2
  10. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gemma2.py +2 -9
  11. gptqmodel-1.3.0/gptqmodel/models/definitions/glm.py +15 -0
  12. gptqmodel-1.3.0/gptqmodel/models/definitions/hymba.py +31 -0
  13. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mllama.py +1 -1
  14. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/xverse.py +2 -1
  15. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/loader.py +71 -60
  16. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/writer.py +85 -77
  17. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +3 -2
  18. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py +3 -2
  19. gptqmodel-1.3.0/gptqmodel/nn_modules/qlinear/qlinear_cuda.py +304 -0
  20. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py +3 -2
  21. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_ipex.py +4 -3
  22. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +3 -3
  23. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py +2 -2
  24. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
  25. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/config.py +8 -13
  26. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/gptq.py +2 -2
  27. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/quantizer.py +3 -3
  28. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/backend.py +1 -0
  29. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/bitblas.py +5 -6
  30. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/device.py +13 -0
  31. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/importer.py +8 -6
  32. gptqmodel-1.3.0/gptqmodel/utils/lm_eval.py +125 -0
  33. gptqmodel-1.3.0/gptqmodel/utils/logger.py +17 -0
  34. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/marlin.py +4 -4
  35. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/model.py +7 -13
  36. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/perplexity.py +3 -2
  37. gptqmodel-1.3.0/gptqmodel/utils/plotly.py +18 -0
  38. gptqmodel-1.3.0/gptqmodel/utils/progress.py +123 -0
  39. gptqmodel-1.3.0/gptqmodel/version.py +1 -0
  40. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0/gptqmodel.egg-info}/PKG-INFO +52 -56
  41. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/SOURCES.txt +13 -10
  42. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/requires.txt +10 -1
  43. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/top_level.txt +2 -0
  44. gptqmodel-1.3.0/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
  45. gptqmodel-1.3.0/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
  46. gptqmodel-1.3.0/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
  47. gptqmodel-1.3.0/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
  48. gptqmodel-1.3.0/requirements.txt +11 -0
  49. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/setup.py +20 -3
  50. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_dynamic.py +5 -4
  51. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_estimate_vram.py +3 -2
  52. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_lm_eval.py +7 -3
  53. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_pt.py +9 -3
  54. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_bitblas.py +1 -7
  55. gptqmodel-1.3.0/tests/test_q4_cuda.py +108 -0
  56. gptqmodel-1.3.0/tests/test_q4_ipex.py +23 -0
  57. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_marlin.py +4 -3
  58. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_save_loaded_quantized_model.py +9 -7
  59. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_transformers_integration.py +4 -4
  60. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_verify_hash.py +8 -2
  61. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_vllm.py +4 -3
  62. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/__init__.py +0 -3
  63. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_base.py +0 -65
  64. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -42
  65. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/data_utils.py +0 -263
  66. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -34
  67. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/language_modeling_task.py +0 -45
  68. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/sequence_classification_task.py +0 -101
  69. gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/text_summarization_task.py +0 -70
  70. gptqmodel-1.2.1.dev0/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
  71. gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
  72. gptqmodel-1.2.1.dev0/requirements.txt +0 -18
  73. gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
  74. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/LICENSE +0 -0
  75. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/MANIFEST.in +0 -0
  76. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/__init__.py +0 -0
  77. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/__init__.py +0 -0
  78. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/__init__.py +0 -0
  79. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/constants.py +0 -0
  80. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/data.py +0 -0
  81. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/integration/optimum/hf_quantizer_gptq.py +0 -0
  82. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/__init__.py +0 -0
  83. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/baichuan.py +0 -0
  84. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/bloom.py +0 -0
  85. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/codegen.py +0 -0
  86. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/cohere.py +0 -0
  87. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/dbrx.py +0 -0
  88. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
  89. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/decilm.py +0 -0
  90. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
  91. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/exaone.py +0 -0
  92. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gemma.py +0 -0
  93. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt2.py +0 -0
  94. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
  95. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gpt_neox.py +0 -0
  96. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/gptj.py +0 -0
  97. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/granite.py +0 -0
  98. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/grinmoe.py +0 -0
  99. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/internlm.py +0 -0
  100. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/internlm2.py +0 -0
  101. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/llama.py +0 -0
  102. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/longllama.py +0 -0
  103. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/minicpm.py +0 -0
  104. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/minicpm3.py +0 -0
  105. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mistral.py +0 -0
  106. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mixtral.py +0 -0
  107. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mobilellm.py +0 -0
  108. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/moss.py +0 -0
  109. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/mpt.py +0 -0
  110. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/opt.py +0 -0
  111. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/phi.py +0 -0
  112. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/phi3.py +0 -0
  113. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen.py +0 -0
  114. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen2.py +0 -0
  115. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
  116. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/rw.py +0 -0
  117. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
  118. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/starcoder2.py +0 -0
  119. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/models/definitions/yi.py +0 -0
  120. {gptqmodel-1.2.1.dev0/gptqmodel/eval_tasks/_utils → gptqmodel-1.3.0/gptqmodel/nn_modules}/__init__.py +0 -0
  121. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/__init__.py +0 -0
  122. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py +0 -0
  123. {gptqmodel-1.2.1.dev0/gptqmodel/nn_modules → gptqmodel-1.3.0/gptqmodel/nn_modules/triton_utils}/__init__.py +0 -0
  124. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
  125. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
  126. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
  127. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/quantization/__init__.py +0 -0
  128. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/__init__.py +0 -0
  129. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/data.py +0 -0
  130. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/sglang.py +0 -0
  131. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/vllm.py +0 -0
  132. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel/utils/vram.py +0 -0
  133. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel.egg-info/dependency_links.txt +0 -0
  134. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/config.h +0 -0
  135. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
  136. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
  137. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
  138. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
  139. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
  140. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
  141. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
  142. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
  143. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
  144. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
  145. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
  146. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
  147. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
  148. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
  149. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
  150. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
  151. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
  152. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
  153. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
  154. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -0
  155. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -0
  156. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -0
  157. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_repack.cu +0 -0
  158. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin/marlin_repack.cuh +0 -0
  159. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin.cuh +0 -0
  160. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda.cpp +0 -0
  161. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cu +0 -0
  162. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cuh +0 -0
  163. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_dtypes.cuh +0 -0
  164. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_repack.cu +0 -0
  165. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/gptqmodel_ext/marlin_inference/marlin_repack.cuh +0 -0
  166. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/setup.cfg +0 -0
  167. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_lm_head.py +0 -0
  168. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_packing.py +0 -0
  169. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_perplexity.py +0 -0
  170. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_exllama_v2.py +0 -0
  171. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_q4_triton.py +0 -0
  172. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_batch.py +0 -0
  173. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_formats.py +0 -0
  174. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_quant_trust_remote.py +0 -0
  175. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_serialization.py +0 -0
  176. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_sglang.py +0 -0
  177. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_sharded.py +0 -0
  178. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_tgi.py +0 -0
  179. {gptqmodel-1.2.1.dev0 → gptqmodel-1.3.0}/tests/test_triton.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: gptqmodel
3
- Version: 1.2.1.dev0
3
+ Version: 1.3.0
4
4
  Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
5
5
  Home-page: https://github.com/ModelCloud/GPTQModel
6
6
  Author: ModelCloud
@@ -34,64 +34,63 @@ Provides-Extra: sglang
34
34
  Requires-Dist: sglang>=0.3.2; extra == "sglang"
35
35
  Requires-Dist: flashinfer==0.1.6; extra == "sglang"
36
36
  Provides-Extra: bitblas
37
- Requires-Dist: bitblas>=0.0.1.dev13; extra == "bitblas"
37
+ Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
38
38
  Provides-Extra: hf
39
39
  Requires-Dist: optimum>=1.21.2; extra == "hf"
40
40
  Provides-Extra: ipex
41
41
  Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
42
42
  Provides-Extra: auto-round
43
43
  Requires-Dist: auto_round>=0.3; extra == "auto-round"
44
+ Provides-Extra: logger
45
+ Requires-Dist: clearml; extra == "logger"
46
+ Requires-Dist: random_word; extra == "logger"
47
+ Requires-Dist: device-smi; extra == "logger"
48
+ Requires-Dist: plotly; extra == "logger"
49
+ Provides-Extra: eval
50
+ Requires-Dist: lm_eval>=0.4.4; extra == "eval"
44
51
 
45
52
  <h1 align="center">GPTQModel</h1>
46
53
  <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
47
54
  <p align="center">
48
55
  <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
49
56
  <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
50
- <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/gptqmodel"></a>
57
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
58
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
51
59
  </p>
52
60
 
53
61
  ## News
54
- * 11/11/2024 🚀 [1.2.0] Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
55
- * 10/29/2024 🚀 [v1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
56
- * 10/12/2024 ✨ [v1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
57
- * 10/11/2024 ✨ [v1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
58
- * 10/08/2024 ✨ [v1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
59
- * 09/26/2024 ✨ [v1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
60
- * 09/26/2024 ✨ [v1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
61
- * 09/26/2024 ✨ [v1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
62
- * 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
63
- * 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
64
- * 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
62
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
63
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
64
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
65
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
66
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
67
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
68
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
69
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
70
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
65
71
 
66
72
  <details>
67
73
 
68
74
  <summary>Archived News:</summary>
69
- * 08/10/2024 🚀 [v0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
70
-
71
- * 07/31/2024 🚀 [v0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
72
-
73
- * 07/25/2024 🚀 [v0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
74
-
75
- * 07/13/2024 🚀 [v0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
75
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
76
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
77
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
78
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
79
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
80
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
81
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
82
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
76
83
  Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
77
84
  got full end-to-end in/out features padding to enhance current/future model compatibility.
78
-
79
- * 07/08/2024 🚀 [v0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
80
-
81
- * 07/08/2024 🚀 [v0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
82
-
83
- * 07/05/2024 🚀 [v0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
84
-
85
- * 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
86
-
87
- * 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
88
-
89
- * 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
85
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
86
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
87
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
88
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
89
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
90
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
90
91
  Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
91
-
92
- * 06/29/2024 🚀 [v0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
93
-
94
- * 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
92
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
93
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
95
94
  </details>
96
95
 
97
96
  ## Why should you use GPTQModel?
@@ -103,7 +102,7 @@ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphe
103
102
  Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
104
103
 
105
104
  ## Features
106
- * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted).
105
+ * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
107
106
  * ✨ 100% CI coverage for all supported models including quality/ppl regression.
108
107
  * 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
109
108
  * 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
@@ -119,27 +118,24 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
119
118
  * 🚀 Over 50% faster PPL calculations (OPT)
120
119
  * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
121
120
 
121
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
122
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
122
123
 
123
- ## Model Support: 🚀 (Added by GPTQModel)
124
- [🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
125
-
124
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
126
125
 
126
+ ## Model Support: 🚀 (Added by GPTQModel)
127
127
  | Model | | | | | | | | | | |
128
- | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
129
- | Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
130
- | Bloom | ✅ | Gemma 2 | 🚀 | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
131
- | ChatGLM | 🚀 | GPTBigCod | ✅ | MiniCPM3 | 🚀 | RefinedWeb | ✅ | | | |
132
- | CodeGen | ✅ | GPTNeoX | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
133
- | Cohere | ✅ | GPT-2 | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
134
- | DBRX Converted | 🚀 | GPT-J | ✅ | MobileLLM | 🚀 | XVERSE | ✅ | | | |
135
- | Deci | ✅ | Granite | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
136
- | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MPT | ✅ | | | | | |
137
- | DeepSeek-V2-Lite | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |
138
- | EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
139
-
140
-
141
- ## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
142
- ![plus-v2 5v3@2x](https://github.com/user-attachments/assets/dd223d08-f790-41ff-814d-4c2a98e4a3de)
128
+ | ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- | --- | --- | --- |
129
+ | Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | |
130
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
131
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
132
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | RefinedWeb | ✅ | | | |
133
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
134
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
135
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | XVERSE | ✅ | | | |
136
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
137
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | | | | | |
138
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |## Compatiblity
143
139
 
144
140
 
145
141
  ## Platform Requirements
@@ -3,51 +3,43 @@
3
3
  <p align="center">
4
4
  <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
5
5
  <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
6
- <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/gptqmodel"></a>
6
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
7
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
7
8
  </p>
8
9
 
9
10
  ## News
10
- * 11/11/2024 🚀 [1.2.0] Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
11
- * 10/29/2024 🚀 [v1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
12
- * 10/12/2024 ✨ [v1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
13
- * 10/11/2024 ✨ [v1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
14
- * 10/08/2024 ✨ [v1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
15
- * 09/26/2024 ✨ [v1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
16
- * 09/26/2024 ✨ [v1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
17
- * 09/26/2024 ✨ [v1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
18
- * 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
19
- * 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
20
- * 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
11
+ * 11/26/2024 [1.3.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.3.0) Zero-Day Hymba model support. Removed `tqdm` and `rogue` dependency.
12
+ * 11/24/2024 [1.2.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.3) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
13
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
14
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
15
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
16
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
17
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
18
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
19
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
21
20
 
22
21
  <details>
23
22
 
24
23
  <summary>Archived News:</summary>
25
- * 08/10/2024 🚀 [v0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
26
-
27
- * 07/31/2024 🚀 [v0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
28
-
29
- * 07/25/2024 🚀 [v0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
30
-
31
- * 07/13/2024 🚀 [v0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
24
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
25
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
26
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
27
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
28
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
29
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
30
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
31
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
32
32
  Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
33
33
  got full end-to-end in/out features padding to enhance current/future model compatibility.
34
-
35
- * 07/08/2024 🚀 [v0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
36
-
37
- * 07/08/2024 🚀 [v0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
38
-
39
- * 07/05/2024 🚀 [v0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
40
-
41
- * 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
42
-
43
- * 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
44
-
45
- * 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
34
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
35
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
36
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
37
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
38
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
39
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
46
40
  Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
47
-
48
- * 06/29/2024 🚀 [v0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
49
-
50
- * 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
41
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
42
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
51
43
  </details>
52
44
 
53
45
  ## Why should you use GPTQModel?
@@ -59,7 +51,7 @@ GPTQModel started out as a major refractor (fork) of AutoGTQP but has now morphe
59
51
  Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-par and/or exceeds other 4bit quantization methods in terms of both quality recovery and production level inference speed in both token latency and rps. GPTQ has currently the optimal blend of quality and inference speed you would want to use in a real-world production system.
60
52
 
61
53
  ## Features
62
- * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted).
54
+ * 🚀 Extensive model support for: `IBM Granite`, `Llama 3.2 Vision`, `MiniCPM3`, `GRIN-Moe`, `Phi 3.5`, `EXAONE 3.0`, `InternLM 2.5`, `Gemma 2`, `DeepSeek-V2`, `DeepSeek-V2-Lite`, `ChatGLM`, `MiniCPM`, `Phi-3`, `Qwen2MoE`, `DBRX` (Converted), `Hymba`.
63
55
  * ✨ 100% CI coverage for all supported models including quality/ppl regression.
64
56
  * 🚀 vLLM inference integration for quantized model where format = `FORMAT.GPTQ`
65
57
  * 🚀 SGLang inference integration for quantized model where format = `FORMAT.GPTQ`
@@ -75,27 +67,24 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
75
67
  * 🚀 Over 50% faster PPL calculations (OPT)
76
68
  * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
77
69
 
70
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
71
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
78
72
 
79
- ## Model Support: 🚀 (Added by GPTQModel)
80
- [🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
81
-
73
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
82
74
 
75
+ ## Model Support: 🚀 (Added by GPTQModel)
83
76
  | Model | | | | | | | | | | |
84
- | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
85
- | Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
86
- | Bloom | ✅ | Gemma 2 | 🚀 | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
87
- | ChatGLM | 🚀 | GPTBigCod | ✅ | MiniCPM3 | 🚀 | RefinedWeb | ✅ | | | |
88
- | CodeGen | ✅ | GPTNeoX | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
89
- | Cohere | ✅ | GPT-2 | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
90
- | DBRX Converted | 🚀 | GPT-J | ✅ | MobileLLM | 🚀 | XVERSE | ✅ | | | |
91
- | Deci | ✅ | Granite | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
92
- | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MPT | ✅ | | | | | |
93
- | DeepSeek-V2-Lite | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |
94
- | EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
95
-
96
-
97
- ## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
98
- ![plus-v2 5v3@2x](https://github.com/user-attachments/assets/dd223d08-f790-41ff-814d-4c2a98e4a3de)
77
+ | ---------------- | --- | -------------- | --- | ---------------- | --- |------------| --- | --- | --- | --- |
78
+ | Baichuan | ✅ | Falon | ✅ | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | |
79
+ | Bloom | ✅ | Gemma 2 | 🚀 | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
80
+ | ChatGLM | 🚀 | GPTBigCod | ✅ | LongLLaMA | ✅ | Qwen2MoE | 🚀 | | | |
81
+ | CodeGen | ✅ | GPTNeoX | ✅ | MiniCPM3 | ✅ | RefinedWeb | ✅ | | | |
82
+ | Cohere | ✅ | GPT-2 | ✅ | Mistral | ✅ | StableLM | ✅ | | | |
83
+ | DBRX Converted | 🚀 | GPT-J | ✅ | Mixtral | ✅ | StarCoder2 | ✅ | | | |
84
+ | Deci | ✅ | Granite | 🚀 | MobileLLM | 🚀 | XVERSE | ✅ | | | |
85
+ | DeepSeek-V2 | 🚀 | GRIN-MoE | 🚀 | MOSS | ✅ | Yi | ✅ | | | |
86
+ | DeepSeek-V2-Lite | 🚀 | Hymba | 🚀 | MPT | ✅ | | | | | |
87
+ | EXAONE 3.0 | 🚀 | InternLM 1/2.5 | 🚀 | OPT | ✅ | | | | | |## Compatiblity
99
88
 
100
89
 
101
90
  ## Platform Requirements
@@ -15,12 +15,10 @@
15
15
  import json
16
16
  import os
17
17
  from dataclasses import dataclass
18
- from logging import getLogger
19
18
  from typing import Any, Dict, List, Optional, Tuple, Union
20
19
 
21
20
  import torch
22
21
  from torch import nn
23
- from tqdm.auto import tqdm
24
22
  from transformers import AutoTokenizer
25
23
  from transformers.pytorch_utils import Conv1D
26
24
  from transformers.utils.quantization_config import GPTQConfig, QuantizationMethod
@@ -28,6 +26,7 @@ from transformers.utils.quantization_config import GPTQConfig, QuantizationMetho
28
26
  from .constants import GPTQ_CONFIG
29
27
  from .data import get_dataset, prepare_dataset
30
28
  from .utils import get_block_name_with_pattern, get_device, get_layers, get_preceding_modules, get_seqlen
29
+ from ...utils.progress import ProgressBar
31
30
 
32
31
  try:
33
32
  from optimum.utils import is_accelerate_available
@@ -46,9 +45,10 @@ if OPTIMUM_AVAILABLE and is_accelerate_available():
46
45
  from ...quantization import FORMAT, FORMAT_FIELD_JSON, GPTQ, QuantizeConfig
47
46
  from ...utils.backend import BACKEND
48
47
  from ...utils.importer import select_quant_linear
48
+ from ...utils.logger import setup_logger
49
49
  from ...utils.model import convert_gptq_v1_to_v2_format, convert_gptq_v2_to_v1_format, gptqmodel_post_init
50
50
 
51
- logger = getLogger(__name__)
51
+ logger = setup_logger()
52
52
 
53
53
  OPTIMUM_INSTALL_HINT = "optimum not installed. Please install via `pip install optimum`."
54
54
 
@@ -289,6 +289,20 @@ class GPTQModelQuantizer(object):
289
289
  return model
290
290
 
291
291
  def convert_gptq_v2_to_v1(self, model: nn.Module):
292
+ if self.bits != 4:
293
+ cuda_name_modules = {}
294
+ from gptqmodel.nn_modules.qlinear.qlinear_cuda import CudaQuantLinear
295
+ for name, module in model.named_modules():
296
+ if isinstance(module, CudaQuantLinear):
297
+ cuda_name_modules[name] = module.gptqmodel_cuda
298
+ module.gptqmodel_cuda = None
299
+
300
+ for name, module in model.named_modules():
301
+ if isinstance(module, CudaQuantLinear) and name in cuda_name_modules:
302
+ module.gptqmodel_cuda = cuda_name_modules[name]
303
+
304
+ del cuda_name_modules
305
+
292
306
  model = convert_gptq_v2_to_v1_format(
293
307
  model, quantize_config=self.quantize_config, qlinear_kernel=self.qlinear_kernel
294
308
  )
@@ -501,7 +515,7 @@ class GPTQModelQuantizer(object):
501
515
 
502
516
  # Step 3: Quantize the blocks
503
517
  quantizers = {}
504
- for i, block in enumerate(tqdm(blocks, desc=f"Quantizing {self.block_name_to_quantize} blocks ")):
518
+ for i, block in enumerate(ProgressBar(blocks, desc=f"Quantizing {self.block_name_to_quantize} blocks ")):
505
519
  logger.info(f"Start quantizing block {self.block_name_to_quantize} {i + 1}/{len(blocks)}")
506
520
 
507
521
  if not self.cache_block_outputs:
@@ -533,7 +547,7 @@ class GPTQModelQuantizer(object):
533
547
  else:
534
548
  layers_name_list = [list(layers.keys())]
535
549
  logger.info(f"Module to quantize {layers_name_list}")
536
- for subset_name_list in tqdm(layers_name_list, leave=False, desc="Quantizing layers inside the block"):
550
+ for subset_name_list in ProgressBar(layers_name_list, desc="Quantizing layers inside the block"):
537
551
  subset_layers = {name: layers[name] for name in subset_name_list}
538
552
  gptq = {}
539
553
  handles = []
@@ -12,7 +12,6 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import os
15
- from logging import getLogger
16
15
  from typing import Callable, Optional, Union
17
16
 
18
17
  import torch
@@ -20,11 +19,13 @@ from torch import nn
20
19
  from transformers.modeling_utils import PreTrainedModel
21
20
  from transformers.pytorch_utils import Conv1D
22
21
 
22
+ from ...utils.logger import setup_logger
23
23
  from .constants import BLOCK_PATTERNS, SEQLEN_KEYS_TRANFORMERS
24
24
 
25
25
  ori_save_pretrained = PreTrainedModel.save_pretrained
26
26
 
27
- logger = getLogger(__name__)
27
+ logger = setup_logger()
28
+
28
29
 
29
30
  """
30
31
  Set of utilities to get specific attributes of a model
@@ -27,6 +27,7 @@ SUPPORTED_MODELS = [
27
27
  "gpt_bigcode",
28
28
  "codegen",
29
29
  "chatglm",
30
+ "glm",
30
31
  "RefinedWebModel",
31
32
  "RefinedWeb",
32
33
  "baichuan",
@@ -60,6 +61,7 @@ SUPPORTED_MODELS = [
60
61
  "mllama",
61
62
  "granite",
62
63
  "mobilellm",
64
+ "hymba"
63
65
  ]
64
66
 
65
67
  EXLLAMA_DEFAULT_MAX_INPUT_LENGTH = 2048
@@ -5,10 +5,11 @@ from os.path import isdir, join
5
5
  from typing import Dict, List, Optional, Union
6
6
 
7
7
  from gptqmodel.quantization import QUANT_CONFIG_FILENAME
8
+ from huggingface_hub import list_repo_files
8
9
  from transformers import AutoConfig
9
- from transformers.utils.hub import cached_file
10
10
 
11
11
  from ..utils import BACKEND
12
+ from ..utils.logger import setup_logger
12
13
  from ..utils.model import check_and_get_model_type
13
14
  from .base import BaseGPTQModel, QuantizeConfig
14
15
  from .definitions.baichuan import BaiChuanGPTQ
@@ -23,6 +24,7 @@ from .definitions.deepseek_v2 import DeepSeekV2GPTQ
23
24
  from .definitions.exaone import ExaoneGPTQ
24
25
  from .definitions.gemma import GemmaGPTQ
25
26
  from .definitions.gemma2 import Gemma2GPTQ
27
+ from .definitions.glm import GLM
26
28
  from .definitions.gpt2 import GPT2GPTQ
27
29
  from .definitions.gpt_bigcode import GPTBigCodeGPTQ
28
30
  from .definitions.gpt_neox import GPTNeoXGPTQ
@@ -52,7 +54,9 @@ from .definitions.stablelmepoch import StableLMEpochGPTQ
52
54
  from .definitions.starcoder2 import Starcoder2GPTQ
53
55
  from .definitions.xverse import XverseGPTQ
54
56
  from .definitions.yi import YiGPTQ
55
- from huggingface_hub import list_repo_files
57
+ from .definitions.hymba import HymbaGPTQ
58
+
59
+ logger = setup_logger()
56
60
 
57
61
  MODEL_MAP = {
58
62
  "bloom": BloomGPTQ,
@@ -63,6 +67,7 @@ MODEL_MAP = {
63
67
  "opt": OPTGPTQ,
64
68
  "moss": MOSSGPTQ,
65
69
  "chatglm": ChatGLM,
70
+ "glm": GLM,
66
71
  "gpt_bigcode": GPTBigCodeGPTQ,
67
72
  "codegen": CodeGenGPTQ,
68
73
  "cohere": CohereGPTQ,
@@ -98,6 +103,7 @@ MODEL_MAP = {
98
103
  "mllama": MLlamaGPTQ,
99
104
  "granite": GraniteGPTQ,
100
105
  "mobilellm": MobileLLMGPTQ,
106
+ "hymba": HymbaGPTQ,
101
107
  }
102
108
 
103
109
 
@@ -152,7 +158,7 @@ class GPTQModel:
152
158
  )
153
159
  else:
154
160
  return cls.from_pretrained(
155
- pretrained_model_id_or_path=model_id_or_path,
161
+ model_id_or_path=model_id_or_path,
156
162
  quantize_config=quantize_config,
157
163
  trust_remote_code=trust_remote_code,
158
164
  **kwargs,
@@ -161,14 +167,20 @@ class GPTQModel:
161
167
  @classmethod
162
168
  def from_pretrained(
163
169
  cls,
164
- pretrained_model_id_or_path: str,
170
+ model_id_or_path: str,
165
171
  quantize_config: QuantizeConfig,
166
172
  trust_remote_code: bool = False,
167
173
  **model_init_kwargs,
168
174
  ) -> BaseGPTQModel:
169
- model_type = check_and_get_model_type(pretrained_model_id_or_path, trust_remote_code)
175
+ if hasattr(AutoConfig.from_pretrained(model_id_or_path, trust_remote_code=trust_remote_code), "quantization_config"):
176
+ logger.warning("Model is already quantized, will use `from_quantized` to load quantized model.\n"
177
+ "If you want to quantize the model, please pass un_quantized model path or id, and use "
178
+ "`from_pretrained` with `quantize_config`.")
179
+ return cls.from_quantized(model_id_or_path, trust_remote_code=trust_remote_code)
180
+
181
+ model_type = check_and_get_model_type(model_id_or_path, trust_remote_code)
170
182
  return MODEL_MAP[model_type].from_pretrained(
171
- pretrained_model_id_or_path=pretrained_model_id_or_path,
183
+ pretrained_model_id_or_path=model_id_or_path,
172
184
  quantize_config=quantize_config,
173
185
  trust_remote_code=trust_remote_code,
174
186
  **model_init_kwargs,