gptqmodel 1.2.1.dev0__tar.gz → 1.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. {gptqmodel-1.2.1.dev0/gptqmodel.egg-info → gptqmodel-1.2.3}/PKG-INFO +39 -44
  2. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/README.md +30 -42
  3. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/quantizer.py +16 -2
  4. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/utils.py +3 -2
  5. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/_const.py +1 -0
  6. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/auto.py +16 -6
  7. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/base.py +87 -126
  8. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/__init__.py +1 -0
  9. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/chatglm.py +4 -2
  10. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gemma2.py +2 -9
  11. gptqmodel-1.2.3/gptqmodel/models/definitions/glm.py +15 -0
  12. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mllama.py +1 -1
  13. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/xverse.py +2 -1
  14. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/loader.py +71 -60
  15. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/writer.py +85 -77
  16. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/bitblas_target_detector.py +3 -2
  17. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_bitblas.py +3 -2
  18. gptqmodel-1.2.3/gptqmodel/nn_modules/qlinear/qlinear_cuda.py +304 -0
  19. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_exllamav2.py +3 -2
  20. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_ipex.py +4 -3
  21. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_marlin.py +3 -3
  22. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_tritonv2.py +2 -2
  23. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/kernels.py +2 -3
  24. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/config.py +8 -13
  25. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/gptq.py +2 -2
  26. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/quantizer.py +3 -3
  27. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/backend.py +1 -0
  28. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/bitblas.py +2 -3
  29. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/device.py +13 -0
  30. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/importer.py +8 -6
  31. gptqmodel-1.2.3/gptqmodel/utils/logger.py +17 -0
  32. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/marlin.py +2 -2
  33. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/model.py +2 -8
  34. gptqmodel-1.2.3/gptqmodel/utils/plotly.py +18 -0
  35. gptqmodel-1.2.3/gptqmodel/version.py +1 -0
  36. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3/gptqmodel.egg-info}/PKG-INFO +39 -44
  37. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/SOURCES.txt +10 -1
  38. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/requires.txt +10 -1
  39. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/top_level.txt +2 -0
  40. gptqmodel-1.2.3/gptqmodel_ext/cuda_256/gptqmodel_cuda_256.cpp +70 -0
  41. gptqmodel-1.2.3/gptqmodel_ext/cuda_256/gptqmodel_cuda_kernel_256.cu +554 -0
  42. gptqmodel-1.2.3/gptqmodel_ext/cuda_64/gptqmodel_cuda_64.cpp +70 -0
  43. gptqmodel-1.2.3/gptqmodel_ext/cuda_64/gptqmodel_cuda_kernel_64.cu +554 -0
  44. gptqmodel-1.2.3/requirements.txt +13 -0
  45. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/setup.py +20 -3
  46. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_dynamic.py +5 -4
  47. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_estimate_vram.py +3 -3
  48. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_lm_eval.py +3 -3
  49. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_pt.py +9 -3
  50. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_bitblas.py +1 -7
  51. gptqmodel-1.2.3/tests/test_q4_cuda.py +109 -0
  52. gptqmodel-1.2.3/tests/test_q4_ipex.py +25 -0
  53. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_marlin.py +4 -4
  54. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_save_loaded_quantized_model.py +9 -7
  55. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_transformers_integration.py +4 -4
  56. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_verify_hash.py +8 -2
  57. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_vllm.py +6 -3
  58. gptqmodel-1.2.1.dev0/gptqmodel/version.py +0 -1
  59. gptqmodel-1.2.1.dev0/requirements.txt +0 -18
  60. gptqmodel-1.2.1.dev0/tests/test_ipex.py +0 -37
  61. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/LICENSE +0 -0
  62. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/MANIFEST.in +0 -0
  63. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/__init__.py +0 -0
  64. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/__init__.py +0 -0
  65. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_base.py +0 -0
  66. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/__init__.py +0 -0
  67. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/classification_utils.py +0 -0
  68. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/data_utils.py +0 -0
  69. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/_utils/generation_utils.py +0 -0
  70. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/language_modeling_task.py +0 -0
  71. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/sequence_classification_task.py +0 -0
  72. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/eval_tasks/text_summarization_task.py +0 -0
  73. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/__init__.py +0 -0
  74. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/__init__.py +0 -0
  75. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/constants.py +0 -0
  76. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/data.py +0 -0
  77. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/integration/optimum/hf_quantizer_gptq.py +0 -0
  78. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/__init__.py +0 -0
  79. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/baichuan.py +0 -0
  80. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/bloom.py +0 -0
  81. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/codegen.py +0 -0
  82. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/cohere.py +0 -0
  83. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/dbrx.py +0 -0
  84. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/dbrx_converted.py +0 -0
  85. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/decilm.py +0 -0
  86. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/deepseek_v2.py +0 -0
  87. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/exaone.py +0 -0
  88. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gemma.py +0 -0
  89. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt2.py +0 -0
  90. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt_bigcode.py +0 -0
  91. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gpt_neox.py +0 -0
  92. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/gptj.py +0 -0
  93. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/granite.py +0 -0
  94. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/grinmoe.py +0 -0
  95. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/internlm.py +0 -0
  96. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/internlm2.py +0 -0
  97. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/llama.py +0 -0
  98. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/longllama.py +0 -0
  99. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/minicpm.py +0 -0
  100. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/minicpm3.py +0 -0
  101. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mistral.py +0 -0
  102. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mixtral.py +0 -0
  103. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mobilellm.py +0 -0
  104. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/moss.py +0 -0
  105. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/mpt.py +0 -0
  106. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/opt.py +0 -0
  107. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/phi.py +0 -0
  108. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/phi3.py +0 -0
  109. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen.py +0 -0
  110. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen2.py +0 -0
  111. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/qwen2_moe.py +0 -0
  112. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/rw.py +0 -0
  113. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/stablelmepoch.py +0 -0
  114. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/starcoder2.py +0 -0
  115. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/models/definitions/yi.py +0 -0
  116. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/__init__.py +0 -0
  117. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/__init__.py +0 -0
  118. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/qlinear/qlinear_marlin_inference.py +0 -0
  119. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/__init__.py +0 -0
  120. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/custom_autotune.py +0 -0
  121. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/dequant.py +0 -0
  122. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/nn_modules/triton_utils/mixin.py +0 -0
  123. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/quantization/__init__.py +0 -0
  124. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/__init__.py +0 -0
  125. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/data.py +0 -0
  126. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/perplexity.py +0 -0
  127. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/sglang.py +0 -0
  128. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/vllm.py +0 -0
  129. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel/utils/vram.py +0 -0
  130. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel.egg-info/dependency_links.txt +0 -0
  131. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/config.h +0 -0
  132. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cpp/util.h +0 -0
  133. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/compat.cuh +0 -0
  134. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/compat_gemm.cuh +0 -0
  135. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/matrix_view.cuh +0 -0
  136. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm.cu +0 -0
  137. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm.cuh +0 -0
  138. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel.cuh +0 -0
  139. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_gemm_kernel_gptq.cuh +0 -0
  140. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_matrix.cu +0 -0
  141. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/q_matrix.cuh +0 -0
  142. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_2.cuh +0 -0
  143. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_3.cuh +0 -0
  144. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_4.cuh +0 -0
  145. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_5.cuh +0 -0
  146. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_6.cuh +0 -0
  147. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_8.cuh +0 -0
  148. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/quant/qdq_util.cuh +0 -0
  149. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/cuda/util.cuh +0 -0
  150. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/exllamav2/ext.cpp +0 -0
  151. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda.cpp +0 -0
  152. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda_kernel.cu +0 -0
  153. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_cuda_kernel.cuh +0 -0
  154. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_repack.cu +0 -0
  155. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin/marlin_repack.cuh +0 -0
  156. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin.cuh +0 -0
  157. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda.cpp +0 -0
  158. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cu +0 -0
  159. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_cuda_kernel.cuh +0 -0
  160. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_dtypes.cuh +0 -0
  161. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_repack.cu +0 -0
  162. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/gptqmodel_ext/marlin_inference/marlin_repack.cuh +0 -0
  163. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/setup.cfg +0 -0
  164. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_lm_head.py +0 -0
  165. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_packing.py +0 -0
  166. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_perplexity.py +0 -0
  167. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_exllama_v2.py +0 -0
  168. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_q4_triton.py +0 -0
  169. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_batch.py +0 -0
  170. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_formats.py +0 -0
  171. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_quant_trust_remote.py +0 -0
  172. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_serialization.py +0 -0
  173. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_sglang.py +0 -0
  174. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_sharded.py +0 -0
  175. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_tgi.py +0 -0
  176. {gptqmodel-1.2.1.dev0 → gptqmodel-1.2.3}/tests/test_triton.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: gptqmodel
3
- Version: 1.2.1.dev0
3
+ Version: 1.2.3
4
4
  Summary: A LLM quantization package with user-friendly apis. Based on GPTQ algorithm.
5
5
  Home-page: https://github.com/ModelCloud/GPTQModel
6
6
  Author: ModelCloud
@@ -34,64 +34,62 @@ Provides-Extra: sglang
34
34
  Requires-Dist: sglang>=0.3.2; extra == "sglang"
35
35
  Requires-Dist: flashinfer==0.1.6; extra == "sglang"
36
36
  Provides-Extra: bitblas
37
- Requires-Dist: bitblas>=0.0.1.dev13; extra == "bitblas"
37
+ Requires-Dist: bitblas==0.0.1.dev13; extra == "bitblas"
38
38
  Provides-Extra: hf
39
39
  Requires-Dist: optimum>=1.21.2; extra == "hf"
40
40
  Provides-Extra: ipex
41
41
  Requires-Dist: intel_extension_for_pytorch>=2.5.0; extra == "ipex"
42
42
  Provides-Extra: auto-round
43
43
  Requires-Dist: auto_round>=0.3; extra == "auto-round"
44
+ Provides-Extra: logger
45
+ Requires-Dist: clearml; extra == "logger"
46
+ Requires-Dist: random_word; extra == "logger"
47
+ Requires-Dist: device-smi; extra == "logger"
48
+ Requires-Dist: plotly; extra == "logger"
49
+ Provides-Extra: eval
50
+ Requires-Dist: lm_eval>=0.4.4; extra == "eval"
44
51
 
45
52
  <h1 align="center">GPTQModel</h1>
46
53
  <p align="center">Production ready LLM model compression/quantization toolkit with accelerated inference support for both cpu/gpu via HF, vLLM, and SGLang.</p>
47
54
  <p align="center">
48
55
  <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
49
56
  <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
50
- <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/gptqmodel"></a>
57
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
58
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
51
59
  </p>
52
60
 
53
61
  ## News
54
- * 11/11/2024 🚀 [1.2.0] Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
55
- * 10/29/2024 🚀 [v1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
56
- * 10/12/2024 ✨ [v1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
57
- * 10/11/2024 ✨ [v1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
58
- * 10/08/2024 ✨ [v1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
59
- * 09/26/2024 ✨ [v1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
60
- * 09/26/2024 ✨ [v1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
61
- * 09/26/2024 ✨ [v1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
62
- * 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
63
- * 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
64
- * 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
62
+ * 11/19/2024 [1.2.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.2) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
63
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
64
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
65
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
66
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
67
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
68
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
69
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
65
70
 
66
71
  <details>
67
72
 
68
73
  <summary>Archived News:</summary>
69
- * 08/10/2024 🚀 [v0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
70
-
71
- * 07/31/2024 🚀 [v0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
72
-
73
- * 07/25/2024 🚀 [v0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
74
-
75
- * 07/13/2024 🚀 [v0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
74
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
75
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
76
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
77
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
78
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
79
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
80
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
81
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
76
82
  Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
77
83
  got full end-to-end in/out features padding to enhance current/future model compatibility.
78
-
79
- * 07/08/2024 🚀 [v0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
80
-
81
- * 07/08/2024 🚀 [v0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
82
-
83
- * 07/05/2024 🚀 [v0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
84
-
85
- * 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
86
-
87
- * 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
88
-
89
- * 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
84
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
85
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
86
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
87
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
88
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
89
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
90
90
  Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
91
-
92
- * 06/29/2024 🚀 [v0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
93
-
94
- * 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
91
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
92
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
95
93
  </details>
96
94
 
97
95
  ## Why should you use GPTQModel?
@@ -119,11 +117,12 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
119
117
  * 🚀 Over 50% faster PPL calculations (OPT)
120
118
  * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
121
119
 
120
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
121
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
122
122
 
123
- ## Model Support: 🚀 (Added by GPTQModel)
124
- [🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
125
-
123
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
126
124
 
125
+ ## Model Support: 🚀 (Added by GPTQModel)
127
126
  | Model | | | | | | | | | | |
128
127
  | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
129
128
  | Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
@@ -138,10 +137,6 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
138
137
  | EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
139
138
 
140
139
 
141
- ## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
142
- ![plus-v2 5v3@2x](https://github.com/user-attachments/assets/dd223d08-f790-41ff-814d-4c2a98e4a3de)
143
-
144
-
145
140
  ## Platform Requirements
146
141
 
147
142
  GPTQModel is validated for Linux x86_64 with Nvidia GPUs. Windows WSL2 may work but un-tested.
@@ -3,51 +3,42 @@
3
3
  <p align="center">
4
4
  <a href="https://github.com/ModelCloud/GPTQModel/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/GPTQModel.svg"></a>
5
5
  <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/gptqmodel"></a>
6
- <a href="https://pypi.org/project/gptqmodel/" style="text-decoration:none;"><img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/gptqmodel"></a>
6
+ <a href="https://pepy.tech/projects/gptqmodel" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/gptqmodel" alt="PyPI Downloads"></a>
7
+ <a href="https://github.com/ModelCloud/GPTQModel/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/gptqmodel"></a>
7
8
  </p>
8
9
 
9
10
  ## News
10
- * 11/11/2024 🚀 [1.2.0] Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
11
- * 10/29/2024 🚀 [v1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
12
- * 10/12/2024 ✨ [v1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
13
- * 10/11/2024 ✨ [v1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
14
- * 10/08/2024 ✨ [v1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
15
- * 09/26/2024 ✨ [v1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
16
- * 09/26/2024 ✨ [v1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
17
- * 09/26/2024 ✨ [v1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
18
- * 09/18/2024 ✨ [v1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
19
- * 08/16/2024 ✨ [v1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
20
- * 08/14/2024 ✨ [v1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
11
+ * 11/19/2024 [1.2.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.2) HF GLM model support. ClearML logging integration. Use `device-smi` and replace `gputil` + `psutil` depends. Fixed model unit tests.
12
+ * 11/11/2024 🚀 [1.2.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.2.1) Meta MobileLLM model support added. `lm-eval[gptqmodel]` integration merged upstream. Intel/IPEX cpu inference merged replacing QBits (deprecated). Auto-fix/patch ChatGLM-3/GLM-4 compat with latest transformers. New `.load()` and `.save()` api.
13
+ * 10/29/2024 🚀 [1.1.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.1.0) IBM Granite model support. Full auto-buildless wheel install from pypi. Reduce max cpu memory usage by >20% during quantization. 100% CI model/feature coverage.
14
+ * 10/12/2024 ✨ [1.0.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.9) Move AutoRound to optional and fix pip install regression in v1.0.8.
15
+ * 10/11/2024 ✨ [1.0.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.8) Add wheel for python 3.12 and cuda 11.8.
16
+ * 10/08/2024 ✨ [1.0.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.7) Fixed marlin (faster) kernel was not auto-selected for some models.
17
+ * 09/26/2024 ✨ [1.0.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.6) Fixed quantized Llama 3.2 vision quantized loader.
18
+ * 09/26/2024 ✨ [1.0.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.5) Partial Llama 3.2 Vision model support (mllama): only text-layer quantization layers are supported for now.
21
19
 
22
20
  <details>
23
21
 
24
22
  <summary>Archived News:</summary>
25
- * 08/10/2024 🚀 [v0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
26
-
27
- * 07/31/2024 🚀 [v0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
28
-
29
- * 07/25/2024 🚀 [v0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
30
-
31
- * 07/13/2024 🚀 [v0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
23
+ * 09/26/2024 ✨ [1.0.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.4) Integrated Liger Kernel support for ~1/2 memory reduction on some models during quantization. Added control toggle disable parallel packing.
24
+ * 09/18/2024 ✨ [1.0.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.3) Added Microsoft GRIN-MoE and MiniCPM3 support.
25
+ * 08/16/2024 ✨ [1.0.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.2) Support Intel/AutoRound v0.3, pre-built whl packages, and PyPI release.
26
+ * 08/14/2024 ✨ [1.0.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v1.0.0) 40% faster `packing`, Fixed Python 3.9 compat, added `lm_eval` api.
27
+ * 08/10/2024 🚀 [0.9.11](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.11) Added LG EXAONE 3.0 model support. New `dynamic` per layer/module flexible quantization where each layer/module may have different bits/params. Added proper sharding support to `backend.BITBLAS`. Auto-heal quantization errors due to small damp values.
28
+ * 07/31/2024 🚀 [0.9.10](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.10) Ported vllm/nm `gptq_marlin` inference kernel with expanded bits (8bits), group_size (64,32), and desc_act support for all GPTQ models with `FORMAT.GPTQ`. Auto calculate auto-round nsamples/seglen parameters based on calibration dataset. Fixed save_quantized() called on pre-quantized models with non-supported backends. HF transformers depend updated to ensure Llama 3.1 fixes are correctly applied to both quant and inference.
29
+ * 07/25/2024 🚀 [0.9.9](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.9): Added Llama-3.1 support, Gemma2 27B quant inference support via vLLM, auto pad_token normalization, fixed auto-round quant compat for vLLM/SGLang, and more.
30
+ * 07/13/2024 🚀 [0.9.8](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.8):
32
31
  Run quantized models directly using GPTQModel using fast `vLLM` or `SGLang` backend! Both vLLM and SGLang are optimized for dyanamic batching inference for maximum `TPS` (check usage under examples). Marlin backend also
33
32
  got full end-to-end in/out features padding to enhance current/future model compatibility.
34
-
35
- * 07/08/2024 🚀 [v0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
36
-
37
- * 07/08/2024 🚀 [v0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
38
-
39
- * 07/05/2024 🚀 [v0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
40
-
41
- * 07/03/2024 🚀 [v0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
42
-
43
- * 07/02/2024 🚀 [v0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
44
-
45
- * 06/30/2024 🚀 [v0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
33
+ * 07/08/2024 🚀 [0.9.7](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.7): InternLM 2.5 model support added.
34
+ * 07/08/2024 🚀 [0.9.6](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.6): [Intel/AutoRound](https://github.com/intel/auto-round) QUANT_METHOD support added for a potentially higher quality quantization with `lm_head` module quantization support for even more vram reduction: format export to `FORMAT.GPTQ` for max inference compatibility.
35
+ * 07/05/2024 🚀 [0.9.5](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.5): Cuda kernels have been fully deprecated in favor of Exllama(v1/v2)/Marlin/Triton.
36
+ * 07/03/2024 🚀 [0.9.4](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.4): HF Transformers integration added and bug fixed Gemma 2 support.
37
+ * 07/02/2024 🚀 [0.9.3](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.3): Added Gemma 2 support, faster PPL calculations on gpu, and more code/arg refractor.
38
+ * 06/30/2024 🚀 [0.9.2](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.2): Added auto-padding of model in/out-features for exllama and exllama v2.
46
39
  Fixed quantization of OPT and DeepSeek V2-Lite models. Fixed inference for DeepSeek V2-Lite.
47
-
48
- * 06/29/2024 🚀 [v0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
49
-
50
- * 06/20/2924 ✨ GPTQModel [v0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
40
+ * 06/29/2024 🚀 [0.9.1](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.1): With 3 new models (DeepSeek-V2, DeepSeek-V2-Lite, DBRX Converted), BITBLAS new format/kernel, proper batching of calibration dataset resulting > 50% quantization speedup, security hash check of loaded model weights, tons of refractor/usability improvements, bugs fixes and much more.
41
+ * 06/20/2924 ✨ [0.9.0](https://github.com/ModelCloud/GPTQModel/releases/tag/v0.9.0): Thanks for all the work from ModelCloud team and the opensource ML community for their contributions!
51
42
  </details>
52
43
 
53
44
  ## Why should you use GPTQModel?
@@ -75,11 +66,12 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
75
66
  * 🚀 Over 50% faster PPL calculations (OPT)
76
67
  * 🚀 Over 40% faster `packing` stage in quantization (Llama 3.1 8B)
77
68
 
69
+ ## Quality: GPTQModel 4Bit Quantized models can match and sometimes exceed BF16:
70
+ 🤗 [ModelCloud quantized ultra-high recovery vortex-series models on HF](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2)
78
71
 
79
- ## Model Support: 🚀 (Added by GPTQModel)
80
- [🤗 Pre-quantized models on HF](https://hf.co/ModelCloud)
81
-
72
+ ![image](https://github.com/user-attachments/assets/e19dcabc-ba47-467c-ac86-3b411d31df2d)
82
73
 
74
+ ## Model Support: 🚀 (Added by GPTQModel)
83
75
  | Model | | | | | | | | | | |
84
76
  | ---------------- | --- | -------------- | --- | ---------------- | --- | ---------- | --- | --- | --- | --- |
85
77
  | Baichuan | ✅ | Falon | ✅ | Llama 3.2 Vision | 🚀 | Qwen | ✅ | | | |
@@ -94,10 +86,6 @@ Public tests/papers and ModelCloud's internal tests have shown that GPTQ is on-p
94
86
  | EXAONE 3.0 | 🚀 | Llama 1/2/3 | ✅ | Phi/Phi-3 | 🚀 | | | | | |## Compatiblity
95
87
 
96
88
 
97
- ## Quality: Quantized Llama-3.2-Instruct models with 100% avg recovery:
98
- ![plus-v2 5v3@2x](https://github.com/user-attachments/assets/dd223d08-f790-41ff-814d-4c2a98e4a3de)
99
-
100
-
101
89
  ## Platform Requirements
102
90
 
103
91
  GPTQModel is validated for Linux x86_64 with Nvidia GPUs. Windows WSL2 may work but un-tested.
@@ -15,7 +15,6 @@
15
15
  import json
16
16
  import os
17
17
  from dataclasses import dataclass
18
- from logging import getLogger
19
18
  from typing import Any, Dict, List, Optional, Tuple, Union
20
19
 
21
20
  import torch
@@ -46,9 +45,10 @@ if OPTIMUM_AVAILABLE and is_accelerate_available():
46
45
  from ...quantization import FORMAT, FORMAT_FIELD_JSON, GPTQ, QuantizeConfig
47
46
  from ...utils.backend import BACKEND
48
47
  from ...utils.importer import select_quant_linear
48
+ from ...utils.logger import setup_logger
49
49
  from ...utils.model import convert_gptq_v1_to_v2_format, convert_gptq_v2_to_v1_format, gptqmodel_post_init
50
50
 
51
- logger = getLogger(__name__)
51
+ logger = setup_logger()
52
52
 
53
53
  OPTIMUM_INSTALL_HINT = "optimum not installed. Please install via `pip install optimum`."
54
54
 
@@ -289,6 +289,20 @@ class GPTQModelQuantizer(object):
289
289
  return model
290
290
 
291
291
  def convert_gptq_v2_to_v1(self, model: nn.Module):
292
+ if self.bits != 4:
293
+ cuda_name_modules = {}
294
+ from gptqmodel.nn_modules.qlinear.qlinear_cuda import CudaQuantLinear
295
+ for name, module in model.named_modules():
296
+ if isinstance(module, CudaQuantLinear):
297
+ cuda_name_modules[name] = module.gptqmodel_cuda
298
+ module.gptqmodel_cuda = None
299
+
300
+ for name, module in model.named_modules():
301
+ if isinstance(module, CudaQuantLinear) and name in cuda_name_modules:
302
+ module.gptqmodel_cuda = cuda_name_modules[name]
303
+
304
+ del cuda_name_modules
305
+
292
306
  model = convert_gptq_v2_to_v1_format(
293
307
  model, quantize_config=self.quantize_config, qlinear_kernel=self.qlinear_kernel
294
308
  )
@@ -12,7 +12,6 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import os
15
- from logging import getLogger
16
15
  from typing import Callable, Optional, Union
17
16
 
18
17
  import torch
@@ -20,11 +19,13 @@ from torch import nn
20
19
  from transformers.modeling_utils import PreTrainedModel
21
20
  from transformers.pytorch_utils import Conv1D
22
21
 
22
+ from ...utils.logger import setup_logger
23
23
  from .constants import BLOCK_PATTERNS, SEQLEN_KEYS_TRANFORMERS
24
24
 
25
25
  ori_save_pretrained = PreTrainedModel.save_pretrained
26
26
 
27
- logger = getLogger(__name__)
27
+ logger = setup_logger()
28
+
28
29
 
29
30
  """
30
31
  Set of utilities to get specific attributes of a model
@@ -27,6 +27,7 @@ SUPPORTED_MODELS = [
27
27
  "gpt_bigcode",
28
28
  "codegen",
29
29
  "chatglm",
30
+ "glm",
30
31
  "RefinedWebModel",
31
32
  "RefinedWeb",
32
33
  "baichuan",
@@ -5,10 +5,11 @@ from os.path import isdir, join
5
5
  from typing import Dict, List, Optional, Union
6
6
 
7
7
  from gptqmodel.quantization import QUANT_CONFIG_FILENAME
8
+ from huggingface_hub import list_repo_files
8
9
  from transformers import AutoConfig
9
- from transformers.utils.hub import cached_file
10
10
 
11
11
  from ..utils import BACKEND
12
+ from ..utils.logger import setup_logger
12
13
  from ..utils.model import check_and_get_model_type
13
14
  from .base import BaseGPTQModel, QuantizeConfig
14
15
  from .definitions.baichuan import BaiChuanGPTQ
@@ -23,6 +24,7 @@ from .definitions.deepseek_v2 import DeepSeekV2GPTQ
23
24
  from .definitions.exaone import ExaoneGPTQ
24
25
  from .definitions.gemma import GemmaGPTQ
25
26
  from .definitions.gemma2 import Gemma2GPTQ
27
+ from .definitions.glm import GLM
26
28
  from .definitions.gpt2 import GPT2GPTQ
27
29
  from .definitions.gpt_bigcode import GPTBigCodeGPTQ
28
30
  from .definitions.gpt_neox import GPTNeoXGPTQ
@@ -52,7 +54,8 @@ from .definitions.stablelmepoch import StableLMEpochGPTQ
52
54
  from .definitions.starcoder2 import Starcoder2GPTQ
53
55
  from .definitions.xverse import XverseGPTQ
54
56
  from .definitions.yi import YiGPTQ
55
- from huggingface_hub import list_repo_files
57
+
58
+ logger = setup_logger()
56
59
 
57
60
  MODEL_MAP = {
58
61
  "bloom": BloomGPTQ,
@@ -63,6 +66,7 @@ MODEL_MAP = {
63
66
  "opt": OPTGPTQ,
64
67
  "moss": MOSSGPTQ,
65
68
  "chatglm": ChatGLM,
69
+ "glm": GLM,
66
70
  "gpt_bigcode": GPTBigCodeGPTQ,
67
71
  "codegen": CodeGenGPTQ,
68
72
  "cohere": CohereGPTQ,
@@ -152,7 +156,7 @@ class GPTQModel:
152
156
  )
153
157
  else:
154
158
  return cls.from_pretrained(
155
- pretrained_model_id_or_path=model_id_or_path,
159
+ model_id_or_path=model_id_or_path,
156
160
  quantize_config=quantize_config,
157
161
  trust_remote_code=trust_remote_code,
158
162
  **kwargs,
@@ -161,14 +165,20 @@ class GPTQModel:
161
165
  @classmethod
162
166
  def from_pretrained(
163
167
  cls,
164
- pretrained_model_id_or_path: str,
168
+ model_id_or_path: str,
165
169
  quantize_config: QuantizeConfig,
166
170
  trust_remote_code: bool = False,
167
171
  **model_init_kwargs,
168
172
  ) -> BaseGPTQModel:
169
- model_type = check_and_get_model_type(pretrained_model_id_or_path, trust_remote_code)
173
+ if hasattr(AutoConfig.from_pretrained(model_id_or_path, trust_remote_code=trust_remote_code), "quantization_config"):
174
+ logger.warning("Model is already quantized, will use `from_quantized` to load quantized model.\n"
175
+ "If you want to quantize the model, please pass un_quantized model path or id, and use "
176
+ "`from_pretrained` with `quantize_config`.")
177
+ return cls.from_quantized(model_id_or_path, trust_remote_code=trust_remote_code)
178
+
179
+ model_type = check_and_get_model_type(model_id_or_path, trust_remote_code)
170
180
  return MODEL_MAP[model_type].from_pretrained(
171
- pretrained_model_id_or_path=pretrained_model_id_or_path,
181
+ pretrained_model_id_or_path=model_id_or_path,
172
182
  quantize_config=quantize_config,
173
183
  trust_remote_code=trust_remote_code,
174
184
  **model_init_kwargs,