onecomp 1.4.0__tar.gz → 1.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {onecomp-1.4.0/onecomp.egg-info → onecomp-1.4.2}/PKG-INFO +4 -3
- {onecomp-1.4.0 → onecomp-1.4.2}/README.md +3 -2
- onecomp-1.4.2/benchmark/jointq_benchmark/2bit_g32/quant_benchmark.py +94 -0
- onecomp-1.4.2/benchmark/jointq_benchmark/actorder_mse/quant_benchmark.py +116 -0
- onecomp-1.4.2/benchmark/jointq_benchmark/calibration_stability/quant_benchmark.py +204 -0
- onecomp-1.4.2/benchmark/jointq_benchmark/main/quant_benchmark.py +122 -0
- onecomp-1.4.2/benchmark/jointq_benchmark/unregularization/quant_benchmark.py +101 -0
- {onecomp-1.4.0/benchmark/qwen3-8b-jointq → onecomp-1.4.2/benchmark/llama2-7b-gptq}/quant_benchmark.py +31 -39
- onecomp-1.4.2/benchmark/llama2-7b-gptq/quant_benchmark_2bit_g32.py +115 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/llama3-8b-gptq/quant_benchmark.py +10 -0
- onecomp-1.4.2/benchmark/llama3-8b-gptq/quant_benchmark_2bit_g32.py +122 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/qwen3-14b-gptq/quant_benchmark.py +10 -0
- onecomp-1.4.2/benchmark/qwen3-14b-gptq/quant_benchmark_2bit_g32.py +122 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/qwen3-8b-gptq/quant_benchmark.py +10 -0
- onecomp-1.4.2/benchmark/qwen3-8b-gptq/quant_benchmark_2bit_g32.py +122 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/llamacpp_plugins/gptq/llamacpp_plugin.py +42 -6
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/__version__.py +1 -1
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/analyzer/__init__.py +9 -1
- onecomp-1.4.2/onecomp/analyzer/output_squared_error.py +321 -0
- onecomp-1.4.2/onecomp/analyzer/quantization_error.py +809 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/c4.py +33 -6
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/calibration_config.py +6 -1
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/calibration_data_loader.py +7 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/custom.py +84 -11
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/wikitext.py +18 -2
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cli.py +3 -1
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/dequantize.py +36 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/direct.py +32 -6
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/fallback.py +5 -2
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/skeleton.py +3 -2
- onecomp-1.4.2/onecomp/cpu/export/transforms/__init__.py +0 -0
- onecomp-1.4.2/onecomp/cpu/export/transforms/qwen35.py +258 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/llama_tooling.py +48 -5
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/hub.py +1 -1
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/model_card.py +38 -4
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/_jointq.py +490 -428
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/clip.py +2 -2
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/error_propagation/quantize_advanced.py +2 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/error_propagation/quantizer_advanced.py +3 -0
- onecomp-1.4.2/onecomp/quantizer/jointq/core/group_swap.py +795 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/quantize.py +3 -4
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/rtn/_rtn.py +0 -10
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/rtn/quantizer.py +20 -9
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/runner.py +6 -2
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/unfuse_moe.py +91 -18
- {onecomp-1.4.0 → onecomp-1.4.2/onecomp.egg-info}/PKG-INFO +4 -3
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp.egg-info/SOURCES.txt +14 -3
- onecomp-1.4.0/benchmark/llama3-8b-jointq/quant_benchmark.py +0 -114
- onecomp-1.4.0/benchmark/qwen3-14b-jointq/quant_benchmark.py +0 -114
- onecomp-1.4.0/onecomp/analyzer/quantization_error.py +0 -185
- {onecomp-1.4.0 → onecomp-1.4.2}/LICENSE +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/llama3-8b-lpcd-gptq/quant_benchmark.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/llama3-8b-qep-gptq/quant_benchmark.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/benchmark/llama3-8b-various/quant_benchmark.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/api/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/api/jobs.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/constants.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/core/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/core/config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/core/database.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/main.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/models/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/models/job.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/schemas/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/schemas/job.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/services/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/services/huggingface.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/services/inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/services/job_store.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/worker/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/worker/celery_app.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/app/worker/tasks.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/cpu_patch.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/start_backend.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/dashboard/backend/start_worker.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/envs/openvino/example_export_openvino.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/envs/openvino/openvino_export_utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/envs/vllm/v0_24_0_rocm/src/onecomp_vllm_v0_24_0_rocm/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/envs/vllm/v0_24_0_rocm/src/onecomp_vllm_v0_24_0_rocm/patch.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/cpu_inference/example_gptq_gguf_cpu.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/cpu_inference/example_mixed_gptq_gguf_cpu.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/cpu_inference/example_serve_cpu.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_auto_run.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_autobit.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_custom_calibration.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_jointq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_lpcd_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_mdbf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_qep_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/example_save_load.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_blockwise_global_ptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_blockwise_global_ptq_staged.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_blockwise_ptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_global_ptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_global_ptq_dbf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_global_ptq_distributed.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_global_ptq_mdbf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_lora_gptq_vllm_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_lora_sft.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_lora_sft_knowledge.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_lora_sft_knowledge_jointq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_reload_post_process_resave.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/post_process/example_router_fine_tuning.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/pre_process/example_llama_preprocess_rtn.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/pre_process/example_preprocess_save_load.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_autobit_vllm_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_dbf_vllm_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_gptq_vllm_gptoss_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_gptq_vllm_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_gptq_vllm_qwen36_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/example/vllm_inference/example_jointq_vllm_inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/llamacpp_plugins/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/llamacpp_plugins/gptq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/llamacpp_plugins/gptq/constants.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/autobit/validate_autobit.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/autobit_qep/validate_autobit.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/gptq/validate_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/gptq/validate_load.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/gptq/validate_vllm.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/jointq/validate_jointq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/model_validation/qep_gptq/validate_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/__main__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/analyzer/cumulative_error.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/analyzer/weight_outlier.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/_cache.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/calibration/chunking.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/cli.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/eval/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/eval/benchmark.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/eval/inspect_gguf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/eval/parity.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/eval/perplexity.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/auto.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/blocks.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/checkpoint.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/export/rotation.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/inference.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/cpu/serve.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/__main__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/conf/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/conf/eval_config.yaml +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/base.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/adapter.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/data.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/gen_answer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/judge.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/radar_chart.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/run.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/mt_bench/show_result.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/throughput/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/throughput/adapter.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/throughput/bench.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/evals/throughput/run.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/orchestrator/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/orchestrator/aggregator.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/orchestrator/runner.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/orchestrator/server.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/orchestrator/subprocess_runner.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/run_evaluate.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/schema.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/utils/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/utils/model_utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/utils/ports.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/utils/resources.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/eval/utils/secrets.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/gguf_export.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/gguf_reader.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/export/gguf_writer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/log.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/_gradient_solver.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/_lpcd_config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/_lpcd_runner.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/_metric.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/_refiner.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/arch/_llama.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/arch/_llama_cf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/lpcd/arch/_qwen3.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/model_config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_base.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/dbf_block_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/dbf_cbq_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/generic_block_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/gptq_block_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/gptq_cbq_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/helpers.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/onebit_block_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_blockwise/onebit_cbq_optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/core.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/dbf_adapter.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/gptq_adapter.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/helpers.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/losses.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/mdbf_adapter.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_global_ptq/trainer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/_runtime.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/blockwise_ptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/global_ptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/global_ptq_distributed.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/post_process_lora_sft.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/post_process/router_fine_tuning.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/hadamard_utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/modeling_llama.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/modeling_qwen3.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/optimizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/prepare_rotated_model.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/preprocess_args.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/quant_models.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/rotation_utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/pre_process/train_rotation.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/qep/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/qep/_qep_config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/qep/_quantize_with_qep.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/qep/_quantize_with_qep_arch.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantized_model_loader.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/_quantizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/arb/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/arb/_arb.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/arb/arb_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/_autobit.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/activation_stats.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/dbf_fallback.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/ilp.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/manual.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/autobit/visualize.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/cq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/cq/_cq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/cq/cq_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/_dbf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/admm_extended.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/balance.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/dbf_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/dbf_layer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/dbf_original.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/fine_tune.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/dbf/middle.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/gemlite.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/gptq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/gptq/_gptq.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/gptq/config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/gptq/gptq_layer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/__version__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/error_propagation/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/error_propagation/local_search_advanced.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/local_search.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/quantize_multi_gpu.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/quantizer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/jointq/core/solution.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/_mdbf.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/admm.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/gradient_refine.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/initialize.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/mdbf_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/mdbf_layer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/mdbf/utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/onebit/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/onebit/_onebit.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/onebit/onebit_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/onebit/onebit_layer.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/qbb/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/qbb/_qbb.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/qbb/qbb_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/_quip.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/quant_quip.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/quip_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/utils.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/utils_had.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/quip/vector_balance.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/rtn/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/quantizer/rtn/rtn_impl.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/rotated_model_config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/runner_methods/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/runner_methods/chunked_quantization.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/runner_methods/jointq_error_propagation.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/accuracy.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/activation_capture.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/activation_check.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/blockwise.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/device.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/dtype.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/lora.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/model_inputs.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/mxfp4_compat.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/perplexity.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/quant_config.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/quantization_progress.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp/utils/vram_estimator.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp.egg-info/dependency_links.txt +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp.egg-info/entry_points.txt +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp.egg-info/requires.txt +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/onecomp.egg-info/top_level.txt +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/pyproject.toml +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/scripts/check_copyright_header.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/scripts/check_no_japanese.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/scripts/check_readme_examples.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/scripts/prepare_calibration_cache.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/setup.cfg +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/dbf/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/dbf/modules/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/dbf/modules/gemlite_linear.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/dbf/modules/naive.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/dbf/vllm_plugin.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/gptq/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/gptq/constants.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/gptq/gptoss_wna16_moe.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/gptq/vllm_plugin.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/patches/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/patches/_paths.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/patches/apply_all.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/patches/gpt_oss_gptq_moe.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/patches/gpt_oss_wna16_bias.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/utils/__init__.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/utils/module.py +0 -0
- {onecomp-1.4.0 → onecomp-1.4.2}/vllm_plugins/utils/rotation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: onecomp
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.2
|
|
4
4
|
Summary: Python package for LLM compression
|
|
5
5
|
Author: Keiji Kimura
|
|
6
6
|
License: MIT License
|
|
@@ -171,10 +171,11 @@ Other Hugging Face-compatible models may work but are currently untested.
|
|
|
171
171
|
| # | Architecture | Verified Models | Status |
|
|
172
172
|
|---|-------------|-----------------|--------|
|
|
173
173
|
| 1 | Llama | TinyLlama, Llama-2, Llama-3 | ✅ Verified |
|
|
174
|
-
| 2 | Qwen3 | Qwen3-0.6B ~ 32B | ✅ Verified |
|
|
175
|
-
| 3 | Gemma | Gemma 2, Gemma 3, Gemma 4 | ✅ Verified |
|
|
174
|
+
| 2 | Qwen3 | Qwen3-0.6B ~ 32B, Qwen3-30B-A3B | ✅ Verified |
|
|
175
|
+
| 3 | Gemma | Gemma 2, Gemma 3, Gemma 4, Gemma 4-26B-A4B | ✅ Verified |
|
|
176
176
|
| 4 | [GPT-OSS](docs/user-guide/gptoss.md) | openai/gpt-oss-20b, gpt-oss-120b | ✅ Verified |
|
|
177
177
|
| 5 | Qwen3.6 | Qwen3.6-27B, Qwen3.6-35B-A3B | ✅ Verified |
|
|
178
|
+
| 6 | Qwen3.8 | Qwen3.8-27B | ✅ Verified |
|
|
178
179
|
|
|
179
180
|
|
|
180
181
|
> **Note:** Support for additional architectures is planned. Contributions and test reports are welcome.
|
|
@@ -57,10 +57,11 @@ Other Hugging Face-compatible models may work but are currently untested.
|
|
|
57
57
|
| # | Architecture | Verified Models | Status |
|
|
58
58
|
|---|-------------|-----------------|--------|
|
|
59
59
|
| 1 | Llama | TinyLlama, Llama-2, Llama-3 | ✅ Verified |
|
|
60
|
-
| 2 | Qwen3 | Qwen3-0.6B ~ 32B | ✅ Verified |
|
|
61
|
-
| 3 | Gemma | Gemma 2, Gemma 3, Gemma 4 | ✅ Verified |
|
|
60
|
+
| 2 | Qwen3 | Qwen3-0.6B ~ 32B, Qwen3-30B-A3B | ✅ Verified |
|
|
61
|
+
| 3 | Gemma | Gemma 2, Gemma 3, Gemma 4, Gemma 4-26B-A4B | ✅ Verified |
|
|
62
62
|
| 4 | [GPT-OSS](docs/user-guide/gptoss.md) | openai/gpt-oss-20b, gpt-oss-120b | ✅ Verified |
|
|
63
63
|
| 5 | Qwen3.6 | Qwen3.6-27B, Qwen3.6-35B-A3B | ✅ Verified |
|
|
64
|
+
| 6 | Qwen3.8 | Qwen3.8-27B | ✅ Verified |
|
|
64
65
|
|
|
65
66
|
|
|
66
67
|
> **Note:** Support for additional architectures is planned. Contributions and test reports are welcome.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Run the JointQ 2-bit/group-size-32 benchmark configuration.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python quant_benchmark.py model_path=path/to/model output_dir=path/to/output_dir
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hydra
|
|
8
|
+
from omegaconf import DictConfig, OmegaConf
|
|
9
|
+
|
|
10
|
+
from onecomp import CalibrationConfig, JointQ, ModelConfig, Runner
|
|
11
|
+
|
|
12
|
+
ACCURACY_TASKS = [
|
|
13
|
+
"arc_easy",
|
|
14
|
+
"arc_challenge",
|
|
15
|
+
"piqa",
|
|
16
|
+
"winogrande",
|
|
17
|
+
"hellaswag",
|
|
18
|
+
"boolq",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_quantizers(cfg: DictConfig):
|
|
23
|
+
"""Create the selected JointQ_2bit_g32 configuration."""
|
|
24
|
+
regularization_lambda_by_layer = {
|
|
25
|
+
"self_attn.q_proj": 0.1,
|
|
26
|
+
"self_attn.k_proj": 0.1,
|
|
27
|
+
"self_attn.v_proj": 0.1,
|
|
28
|
+
"self_attn.o_proj": 0.0,
|
|
29
|
+
"mlp.gate_proj": 0.0,
|
|
30
|
+
"mlp.up_proj": 0.0,
|
|
31
|
+
"mlp.down_proj": 0.0,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return [
|
|
35
|
+
JointQ(
|
|
36
|
+
num_layers=cfg.jointq.num_layers,
|
|
37
|
+
bits=2,
|
|
38
|
+
symmetric=cfg.jointq.symmetric,
|
|
39
|
+
group_size=32,
|
|
40
|
+
regularization_lambda_by_layer=regularization_lambda_by_layer,
|
|
41
|
+
regularization_retry_lambda=None,
|
|
42
|
+
calc_quant_error=True,
|
|
43
|
+
name="JointQ_2bit_g32",
|
|
44
|
+
)
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@hydra.main(version_base=None, config_path="conf", config_name="benchmark")
|
|
49
|
+
def main(cfg: DictConfig):
|
|
50
|
+
print(OmegaConf.to_yaml(cfg))
|
|
51
|
+
|
|
52
|
+
quantizers = create_quantizers(cfg)
|
|
53
|
+
runner = Runner(
|
|
54
|
+
model_config=ModelConfig(
|
|
55
|
+
path=cfg.model_path,
|
|
56
|
+
device=cfg.model_device,
|
|
57
|
+
dtype="bfloat16",
|
|
58
|
+
),
|
|
59
|
+
quantizers=quantizers,
|
|
60
|
+
calibration_config=CalibrationConfig(
|
|
61
|
+
max_length=cfg.max_length,
|
|
62
|
+
num_calibration_samples=cfg.num_calibration_samples,
|
|
63
|
+
strategy=cfg.calibration_strategy,
|
|
64
|
+
seed=cfg.calibration_seed,
|
|
65
|
+
batch_size=cfg.calibration_batch_size,
|
|
66
|
+
),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
runner.run()
|
|
70
|
+
|
|
71
|
+
for quantizer in quantizers:
|
|
72
|
+
runner.save_quantization_statistics(
|
|
73
|
+
f"quantization_statistics_{quantizer.name}.json",
|
|
74
|
+
quantizer=quantizer,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
if cfg.calc_ppl:
|
|
78
|
+
runner.benchmark_perplexity(
|
|
79
|
+
original_model=cfg.calc_original_ppl,
|
|
80
|
+
dequantized_model=True,
|
|
81
|
+
quantized_model=False,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
if cfg.calc_acc:
|
|
85
|
+
runner.benchmark_accuracy(
|
|
86
|
+
tasks=ACCURACY_TASKS,
|
|
87
|
+
original_model=cfg.calc_original_acc,
|
|
88
|
+
dequantized_model=True,
|
|
89
|
+
quantized_model=False,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
if __name__ == "__main__":
|
|
94
|
+
main()
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Run the JointQ activation-ordering and GPTQ MSE ablation.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python quant_benchmark.py model_path=path/to/model output_dir=path/to/output_dir
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hydra
|
|
8
|
+
from omegaconf import DictConfig, OmegaConf
|
|
9
|
+
|
|
10
|
+
from onecomp import GPTQ, CalibrationConfig, JointQ, ModelConfig, Runner
|
|
11
|
+
|
|
12
|
+
ACCURACY_TASKS = [
|
|
13
|
+
"arc_easy",
|
|
14
|
+
"arc_challenge",
|
|
15
|
+
"piqa",
|
|
16
|
+
"winogrande",
|
|
17
|
+
"hellaswag",
|
|
18
|
+
"boolq",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_quantizers(cfg: DictConfig):
|
|
23
|
+
"""Create the selected 4-bit and 3-bit actorder/MSE configurations."""
|
|
24
|
+
settings = [
|
|
25
|
+
(
|
|
26
|
+
"JointQ_4bit_g128_actorder_mse",
|
|
27
|
+
4,
|
|
28
|
+
[0.0, 0.0, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
29
|
+
),
|
|
30
|
+
(
|
|
31
|
+
"JointQ_3bit_g128_actorder_mse",
|
|
32
|
+
3,
|
|
33
|
+
[0.01, 0.01, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
34
|
+
),
|
|
35
|
+
]
|
|
36
|
+
layer_names = [
|
|
37
|
+
"self_attn.q_proj",
|
|
38
|
+
"self_attn.k_proj",
|
|
39
|
+
"self_attn.v_proj",
|
|
40
|
+
"self_attn.o_proj",
|
|
41
|
+
"mlp.gate_proj",
|
|
42
|
+
"mlp.up_proj",
|
|
43
|
+
"mlp.down_proj",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
return [
|
|
47
|
+
JointQ(
|
|
48
|
+
num_layers=cfg.jointq.num_layers,
|
|
49
|
+
bits=bits,
|
|
50
|
+
symmetric=cfg.jointq.symmetric,
|
|
51
|
+
group_size=128,
|
|
52
|
+
regularization_lambda_by_layer=dict(zip(layer_names, lambdas)),
|
|
53
|
+
regularization_retry_lambda=0.01,
|
|
54
|
+
regularization_retry_threshold=0.2,
|
|
55
|
+
actorder=True,
|
|
56
|
+
gptq=GPTQ(
|
|
57
|
+
wbits=bits,
|
|
58
|
+
groupsize=128,
|
|
59
|
+
sym=cfg.jointq.symmetric,
|
|
60
|
+
mse=True,
|
|
61
|
+
),
|
|
62
|
+
enable_group_swap=False,
|
|
63
|
+
calc_quant_error=True,
|
|
64
|
+
name=name,
|
|
65
|
+
)
|
|
66
|
+
for name, bits, lambdas in settings
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@hydra.main(version_base=None, config_path="conf", config_name="benchmark")
|
|
71
|
+
def main(cfg: DictConfig):
|
|
72
|
+
print(OmegaConf.to_yaml(cfg))
|
|
73
|
+
|
|
74
|
+
quantizers = create_quantizers(cfg)
|
|
75
|
+
runner = Runner(
|
|
76
|
+
model_config=ModelConfig(
|
|
77
|
+
path=cfg.model_path,
|
|
78
|
+
device=cfg.model_device,
|
|
79
|
+
dtype="bfloat16",
|
|
80
|
+
),
|
|
81
|
+
quantizers=quantizers,
|
|
82
|
+
calibration_config=CalibrationConfig(
|
|
83
|
+
max_length=cfg.max_length,
|
|
84
|
+
num_calibration_samples=cfg.num_calibration_samples,
|
|
85
|
+
strategy=cfg.calibration_strategy,
|
|
86
|
+
seed=cfg.calibration_seed,
|
|
87
|
+
batch_size=cfg.calibration_batch_size,
|
|
88
|
+
),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
runner.run()
|
|
92
|
+
|
|
93
|
+
for quantizer in quantizers:
|
|
94
|
+
runner.save_quantization_statistics(
|
|
95
|
+
f"quantization_statistics_{quantizer.name}.json",
|
|
96
|
+
quantizer=quantizer,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
if cfg.calc_ppl:
|
|
100
|
+
runner.benchmark_perplexity(
|
|
101
|
+
original_model=cfg.calc_original_ppl,
|
|
102
|
+
dequantized_model=True,
|
|
103
|
+
quantized_model=False,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
if cfg.calc_acc:
|
|
107
|
+
runner.benchmark_accuracy(
|
|
108
|
+
tasks=ACCURACY_TASKS,
|
|
109
|
+
original_model=cfg.calc_original_acc,
|
|
110
|
+
dequantized_model=True,
|
|
111
|
+
quantized_model=False,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
main()
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Benchmark quantization stability with small calibration datasets.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python quant_benchmark.py model_path=path/to/model output_dir=path/to/output_dir
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import hydra
|
|
11
|
+
from omegaconf import DictConfig, OmegaConf
|
|
12
|
+
|
|
13
|
+
from onecomp import GPTQ, CalibrationConfig, JointQ, ModelConfig, Runner
|
|
14
|
+
|
|
15
|
+
ACCURACY_TASKS = [
|
|
16
|
+
"arc_easy",
|
|
17
|
+
"arc_challenge",
|
|
18
|
+
"piqa",
|
|
19
|
+
"winogrande",
|
|
20
|
+
"hellaswag",
|
|
21
|
+
"boolq",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
ACCURACY_METRICS = {
|
|
25
|
+
"arc_easy": "acc_norm,none",
|
|
26
|
+
"arc_challenge": "acc_norm,none",
|
|
27
|
+
"piqa": "acc_norm,none",
|
|
28
|
+
"winogrande": "acc,none",
|
|
29
|
+
"hellaswag": "acc_norm,none",
|
|
30
|
+
"boolq": "acc,none",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
REGULARIZED_LAMBDAS = {
|
|
34
|
+
3: [0.01, 0.01, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
35
|
+
4: [0.0, 0.0, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def create_quantizers(cfg: DictConfig):
|
|
40
|
+
"""Create the five group-size-128 benchmark configurations."""
|
|
41
|
+
bits = int(cfg.bits)
|
|
42
|
+
if bits not in REGULARIZED_LAMBDAS:
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"Unsupported bits: {bits}. Expected one of {sorted(REGULARIZED_LAMBDAS)}"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
layer_names = [
|
|
48
|
+
"self_attn.q_proj",
|
|
49
|
+
"self_attn.k_proj",
|
|
50
|
+
"self_attn.v_proj",
|
|
51
|
+
"self_attn.o_proj",
|
|
52
|
+
"mlp.gate_proj",
|
|
53
|
+
"mlp.up_proj",
|
|
54
|
+
"mlp.down_proj",
|
|
55
|
+
]
|
|
56
|
+
regularized_lambdas = REGULARIZED_LAMBDAS[bits]
|
|
57
|
+
unregularized_lambdas = [0.0] * len(layer_names)
|
|
58
|
+
|
|
59
|
+
quantizers = [
|
|
60
|
+
JointQ(
|
|
61
|
+
num_layers=cfg.jointq.num_layers,
|
|
62
|
+
bits=bits,
|
|
63
|
+
symmetric=cfg.jointq.symmetric,
|
|
64
|
+
group_size=128,
|
|
65
|
+
regularization_lambda_by_layer=dict(zip(layer_names, regularized_lambdas)),
|
|
66
|
+
regularization_retry_lambda=0.01,
|
|
67
|
+
regularization_retry_threshold=0.2,
|
|
68
|
+
calc_quant_error=True,
|
|
69
|
+
name=f"JointQ_{bits}bit_g128",
|
|
70
|
+
),
|
|
71
|
+
JointQ(
|
|
72
|
+
num_layers=cfg.jointq.num_layers,
|
|
73
|
+
bits=bits,
|
|
74
|
+
symmetric=cfg.jointq.symmetric,
|
|
75
|
+
group_size=128,
|
|
76
|
+
regularization_lambda_by_layer=dict(zip(layer_names, unregularized_lambdas)),
|
|
77
|
+
regularization_retry_lambda=None,
|
|
78
|
+
calc_quant_error=True,
|
|
79
|
+
name=f"JointQ_{bits}bit_g128_unregularized",
|
|
80
|
+
),
|
|
81
|
+
JointQ(
|
|
82
|
+
num_layers=cfg.jointq.num_layers,
|
|
83
|
+
bits=bits,
|
|
84
|
+
symmetric=cfg.jointq.symmetric,
|
|
85
|
+
group_size=128,
|
|
86
|
+
regularization_lambda_by_layer=dict(zip(layer_names, regularized_lambdas)),
|
|
87
|
+
regularization_retry_lambda=0.01,
|
|
88
|
+
regularization_retry_threshold=0.2,
|
|
89
|
+
actorder=True,
|
|
90
|
+
gptq=GPTQ(
|
|
91
|
+
wbits=bits,
|
|
92
|
+
groupsize=128,
|
|
93
|
+
sym=cfg.jointq.symmetric,
|
|
94
|
+
mse=True,
|
|
95
|
+
),
|
|
96
|
+
enable_group_swap=False,
|
|
97
|
+
calc_quant_error=True,
|
|
98
|
+
name=f"JointQ_{bits}bit_g128_actorder_mse",
|
|
99
|
+
),
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
for name, mse, actorder in [
|
|
103
|
+
(f"GPTQ_{bits}bit_g128_default", False, False),
|
|
104
|
+
(f"GPTQ_{bits}bit_g128_mse_actorder", True, True),
|
|
105
|
+
]:
|
|
106
|
+
quantizer = GPTQ(
|
|
107
|
+
num_layers=cfg.gptq.num_layers,
|
|
108
|
+
wbits=bits,
|
|
109
|
+
sym=cfg.gptq.symmetric,
|
|
110
|
+
groupsize=128,
|
|
111
|
+
blocksize=cfg.gptq.blocksize,
|
|
112
|
+
percdamp=cfg.gptq.percdamp,
|
|
113
|
+
actorder=actorder,
|
|
114
|
+
mse=mse,
|
|
115
|
+
q_grid=cfg.gptq.q_grid,
|
|
116
|
+
q_norm=cfg.gptq.q_norm,
|
|
117
|
+
calc_quant_error=True,
|
|
118
|
+
name=name,
|
|
119
|
+
)
|
|
120
|
+
quantizer.name = name
|
|
121
|
+
quantizers.append(quantizer)
|
|
122
|
+
|
|
123
|
+
return quantizers
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def calculate_mean_accuracy(task_results: dict) -> float:
|
|
127
|
+
"""Calculate the arithmetic mean using the benchmark's task metrics."""
|
|
128
|
+
values = [task_results[task][metric] for task, metric in ACCURACY_METRICS.items()]
|
|
129
|
+
return float(sum(values) / len(values))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def build_results(quantizers, ppl_results: dict, accuracy_results: dict) -> dict:
|
|
133
|
+
"""Build the per-quantizer PPL and mean accuracy result mapping."""
|
|
134
|
+
results = {}
|
|
135
|
+
for quantizer in quantizers:
|
|
136
|
+
result_key = f"{quantizer.name}_dequantized"
|
|
137
|
+
results[quantizer.name] = {
|
|
138
|
+
"ppl": float(ppl_results[result_key]),
|
|
139
|
+
"mean_accuracy": calculate_mean_accuracy(accuracy_results[result_key]),
|
|
140
|
+
}
|
|
141
|
+
return results
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def result_filename(num_calibration_samples: int, calibration_seed: int) -> str:
|
|
145
|
+
"""Return the benchmark result filename for one calibration run."""
|
|
146
|
+
return f"numsamples{num_calibration_samples}_seed{calibration_seed}.json"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
@hydra.main(version_base=None, config_path="conf", config_name="benchmark")
|
|
150
|
+
def main(cfg: DictConfig):
|
|
151
|
+
print(OmegaConf.to_yaml(cfg))
|
|
152
|
+
|
|
153
|
+
quantizers = create_quantizers(cfg)
|
|
154
|
+
runner = Runner(
|
|
155
|
+
model_config=ModelConfig(
|
|
156
|
+
path=cfg.model_path,
|
|
157
|
+
device=cfg.model_device,
|
|
158
|
+
dtype="bfloat16",
|
|
159
|
+
),
|
|
160
|
+
quantizers=quantizers,
|
|
161
|
+
calibration_config=CalibrationConfig(
|
|
162
|
+
max_length=cfg.max_length,
|
|
163
|
+
num_calibration_samples=cfg.num_calibration_samples,
|
|
164
|
+
strategy=cfg.calibration_strategy,
|
|
165
|
+
seed=cfg.calibration_seed,
|
|
166
|
+
shuffle_documents=cfg.shuffle_documents,
|
|
167
|
+
batch_size=cfg.calibration_batch_size,
|
|
168
|
+
),
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
runner.run()
|
|
172
|
+
|
|
173
|
+
for quantizer in quantizers:
|
|
174
|
+
runner.save_quantization_statistics(
|
|
175
|
+
f"quantization_statistics_{quantizer.name}.json",
|
|
176
|
+
quantizer=quantizer,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
ppl_results = runner.benchmark_perplexity(
|
|
180
|
+
original_model=False,
|
|
181
|
+
dequantized_model=True,
|
|
182
|
+
quantized_model=False,
|
|
183
|
+
)
|
|
184
|
+
accuracy_results = runner.benchmark_accuracy(
|
|
185
|
+
tasks=ACCURACY_TASKS,
|
|
186
|
+
original_model=False,
|
|
187
|
+
dequantized_model=True,
|
|
188
|
+
quantized_model=False,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
result = {
|
|
192
|
+
"bits": int(cfg.bits),
|
|
193
|
+
"max_length": int(cfg.max_length),
|
|
194
|
+
"num_calibration_samples": int(cfg.num_calibration_samples),
|
|
195
|
+
"calibration_seed": int(cfg.calibration_seed),
|
|
196
|
+
"shuffle_documents": bool(cfg.shuffle_documents),
|
|
197
|
+
"results": build_results(quantizers, ppl_results, accuracy_results),
|
|
198
|
+
}
|
|
199
|
+
output_path = Path(result_filename(cfg.num_calibration_samples, cfg.calibration_seed))
|
|
200
|
+
output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
if __name__ == "__main__":
|
|
204
|
+
main()
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Run the selected 4-bit and 3-bit JointQ benchmark configurations.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python quant_benchmark.py model_path=path/to/model output_dir=path/to/output_dir
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hydra
|
|
8
|
+
from omegaconf import DictConfig, OmegaConf
|
|
9
|
+
|
|
10
|
+
from onecomp import CalibrationConfig, JointQ, ModelConfig, Runner
|
|
11
|
+
|
|
12
|
+
ACCURACY_TASKS = [
|
|
13
|
+
"arc_easy",
|
|
14
|
+
"arc_challenge",
|
|
15
|
+
"piqa",
|
|
16
|
+
"winogrande",
|
|
17
|
+
"hellaswag",
|
|
18
|
+
"boolq",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_quantizers(cfg: DictConfig):
|
|
23
|
+
"""Create the four selected JointQ configurations."""
|
|
24
|
+
settings = [
|
|
25
|
+
(
|
|
26
|
+
"JointQ_4bit_g128",
|
|
27
|
+
4,
|
|
28
|
+
128,
|
|
29
|
+
[0.0, 0.0, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
30
|
+
),
|
|
31
|
+
(
|
|
32
|
+
"JointQ_4bit_pc",
|
|
33
|
+
4,
|
|
34
|
+
None,
|
|
35
|
+
[0.0, 0.0, 0.15, 0.0, 0.15, 0.15, 0.01],
|
|
36
|
+
),
|
|
37
|
+
(
|
|
38
|
+
"JointQ_3bit_g128",
|
|
39
|
+
3,
|
|
40
|
+
128,
|
|
41
|
+
[0.01, 0.01, 0.01, 0.0, 0.01, 0.01, 0.0],
|
|
42
|
+
),
|
|
43
|
+
(
|
|
44
|
+
"JointQ_3bit_pc",
|
|
45
|
+
3,
|
|
46
|
+
None,
|
|
47
|
+
[0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.01],
|
|
48
|
+
),
|
|
49
|
+
]
|
|
50
|
+
layer_names = [
|
|
51
|
+
"self_attn.q_proj",
|
|
52
|
+
"self_attn.k_proj",
|
|
53
|
+
"self_attn.v_proj",
|
|
54
|
+
"self_attn.o_proj",
|
|
55
|
+
"mlp.gate_proj",
|
|
56
|
+
"mlp.up_proj",
|
|
57
|
+
"mlp.down_proj",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
return [
|
|
61
|
+
JointQ(
|
|
62
|
+
num_layers=cfg.jointq.num_layers,
|
|
63
|
+
bits=bits,
|
|
64
|
+
symmetric=cfg.jointq.symmetric,
|
|
65
|
+
group_size=group_size,
|
|
66
|
+
regularization_lambda_by_layer=dict(zip(layer_names, lambdas)),
|
|
67
|
+
regularization_retry_lambda=0.01,
|
|
68
|
+
regularization_retry_threshold=0.2,
|
|
69
|
+
calc_quant_error=True,
|
|
70
|
+
name=name,
|
|
71
|
+
)
|
|
72
|
+
for name, bits, group_size, lambdas in settings
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@hydra.main(version_base=None, config_path="conf", config_name="benchmark")
|
|
77
|
+
def main(cfg: DictConfig):
|
|
78
|
+
print(OmegaConf.to_yaml(cfg))
|
|
79
|
+
|
|
80
|
+
quantizers = create_quantizers(cfg)
|
|
81
|
+
runner = Runner(
|
|
82
|
+
model_config=ModelConfig(
|
|
83
|
+
path=cfg.model_path,
|
|
84
|
+
device=cfg.model_device,
|
|
85
|
+
dtype="bfloat16",
|
|
86
|
+
),
|
|
87
|
+
quantizers=quantizers,
|
|
88
|
+
calibration_config=CalibrationConfig(
|
|
89
|
+
max_length=cfg.max_length,
|
|
90
|
+
num_calibration_samples=cfg.num_calibration_samples,
|
|
91
|
+
strategy=cfg.calibration_strategy,
|
|
92
|
+
seed=cfg.calibration_seed,
|
|
93
|
+
batch_size=cfg.calibration_batch_size,
|
|
94
|
+
),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
runner.run()
|
|
98
|
+
|
|
99
|
+
for quantizer in quantizers:
|
|
100
|
+
runner.save_quantization_statistics(
|
|
101
|
+
f"quantization_statistics_{quantizer.name}.json",
|
|
102
|
+
quantizer=quantizer,
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if cfg.calc_ppl:
|
|
106
|
+
runner.benchmark_perplexity(
|
|
107
|
+
original_model=cfg.calc_original_ppl,
|
|
108
|
+
dequantized_model=True,
|
|
109
|
+
quantized_model=False,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if cfg.calc_acc:
|
|
113
|
+
runner.benchmark_accuracy(
|
|
114
|
+
tasks=ACCURACY_TASKS,
|
|
115
|
+
original_model=cfg.calc_original_acc,
|
|
116
|
+
dequantized_model=True,
|
|
117
|
+
quantized_model=False,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
if __name__ == "__main__":
|
|
122
|
+
main()
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Run the JointQ ablation with regularization disabled.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
python quant_benchmark.py model_path=path/to/model output_dir=path/to/output_dir
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import hydra
|
|
8
|
+
from omegaconf import DictConfig, OmegaConf
|
|
9
|
+
|
|
10
|
+
from onecomp import CalibrationConfig, JointQ, ModelConfig, Runner
|
|
11
|
+
|
|
12
|
+
ACCURACY_TASKS = [
|
|
13
|
+
"arc_easy",
|
|
14
|
+
"arc_challenge",
|
|
15
|
+
"piqa",
|
|
16
|
+
"winogrande",
|
|
17
|
+
"hellaswag",
|
|
18
|
+
"boolq",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_quantizers(cfg: DictConfig):
|
|
23
|
+
"""Create four JointQ configurations with regularization disabled."""
|
|
24
|
+
regularization_lambda_by_layer = {
|
|
25
|
+
"self_attn.q_proj": 0.0,
|
|
26
|
+
"self_attn.k_proj": 0.0,
|
|
27
|
+
"self_attn.v_proj": 0.0,
|
|
28
|
+
"self_attn.o_proj": 0.0,
|
|
29
|
+
"mlp.gate_proj": 0.0,
|
|
30
|
+
"mlp.up_proj": 0.0,
|
|
31
|
+
"mlp.down_proj": 0.0,
|
|
32
|
+
}
|
|
33
|
+
settings = [
|
|
34
|
+
("JointQ_4bit_g128_unregularized", 4, 128),
|
|
35
|
+
("JointQ_4bit_pc_unregularized", 4, None),
|
|
36
|
+
("JointQ_3bit_g128_unregularized", 3, 128),
|
|
37
|
+
("JointQ_3bit_pc_unregularized", 3, None),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
return [
|
|
41
|
+
JointQ(
|
|
42
|
+
num_layers=cfg.jointq.num_layers,
|
|
43
|
+
bits=bits,
|
|
44
|
+
symmetric=cfg.jointq.symmetric,
|
|
45
|
+
group_size=group_size,
|
|
46
|
+
regularization_lambda_by_layer=dict(regularization_lambda_by_layer),
|
|
47
|
+
regularization_retry_lambda=None,
|
|
48
|
+
calc_quant_error=True,
|
|
49
|
+
name=name,
|
|
50
|
+
)
|
|
51
|
+
for name, bits, group_size in settings
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@hydra.main(version_base=None, config_path="conf", config_name="benchmark")
|
|
56
|
+
def main(cfg: DictConfig):
|
|
57
|
+
print(OmegaConf.to_yaml(cfg))
|
|
58
|
+
|
|
59
|
+
quantizers = create_quantizers(cfg)
|
|
60
|
+
runner = Runner(
|
|
61
|
+
model_config=ModelConfig(
|
|
62
|
+
path=cfg.model_path,
|
|
63
|
+
device=cfg.model_device,
|
|
64
|
+
dtype="bfloat16",
|
|
65
|
+
),
|
|
66
|
+
quantizers=quantizers,
|
|
67
|
+
calibration_config=CalibrationConfig(
|
|
68
|
+
max_length=cfg.max_length,
|
|
69
|
+
num_calibration_samples=cfg.num_calibration_samples,
|
|
70
|
+
strategy=cfg.calibration_strategy,
|
|
71
|
+
seed=cfg.calibration_seed,
|
|
72
|
+
batch_size=cfg.calibration_batch_size,
|
|
73
|
+
),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
runner.run()
|
|
77
|
+
|
|
78
|
+
for quantizer in quantizers:
|
|
79
|
+
runner.save_quantization_statistics(
|
|
80
|
+
f"quantization_statistics_{quantizer.name}.json",
|
|
81
|
+
quantizer=quantizer,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
if cfg.calc_ppl:
|
|
85
|
+
runner.benchmark_perplexity(
|
|
86
|
+
original_model=cfg.calc_original_ppl,
|
|
87
|
+
dequantized_model=True,
|
|
88
|
+
quantized_model=False,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
if cfg.calc_acc:
|
|
92
|
+
runner.benchmark_accuracy(
|
|
93
|
+
tasks=ACCURACY_TASKS,
|
|
94
|
+
original_model=cfg.calc_original_acc,
|
|
95
|
+
dequantized_model=True,
|
|
96
|
+
quantized_model=False,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
main()
|