qwen14b 转int8量化

转化成int8的模型

AutoGPTQ的方式量化:
https://github.com/QwenLM/Qwen/issues/464
https://github.com/AutoGPTQ/AutoGPTQ/issues/133
搞半天，最后能work的代码：

from transformers import AutoTokenizer, TextGenerationPipeline
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig


model_path="Qwen-14B-datayes"
tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True,trust_remote_code=True)
quantize_config = BaseQuantizeConfig(
        bits=8,  # quantize model to 4-bit
        group_size=128,  # it is recommended to set the value to 128
        desc_act=False,  # set to False can significantly speed up inference but the perplexity may slightly bad
    )

model = AutoGPTQForCausalLM.from_pretrained(model_path, quantize_config,trust_remote_code=True)
examples = [
            tokenizer(
                        "auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."
                            )
            ]

quantized_model_dir="Qwen-14B-datayes-int8"
model.quantize(examples)


# save quantized model
model.save_quantized(quantized_model_dir, use_safetensors=True)
tokenizer.save_pretrained(quantized_model_dir)
~                                                                         

测试完之后，还要能推理：
需要把Qwen下面的py 全部cp过来，然后还需要根据提示的问题各种测试：
目前只能用AutoGPTQForCausalLM.from_quantized来加载。
另外注意model_basename 为量化保存的模型名字，不要带后缀，不然找不到，坑得要死。

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM  # 注意：这里假设存在类似的接口  
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig

torch.cuda.set_device(1)

quantized_model_dir="Qwen-14B-datayes-int8"
model_basename="gptq_model-8bit-128g"

tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir,use_fast=True, trust_remote_code=True)
model = AutoGPTQForCausalLM.from_quantized(quantized_model_dir,model_basename=model_basename,
        device_map="auto",use_safetensors=True,trust_remote_code=True)

# 准备输入  
input_text = "分析一下宁德时代的投资价值"
input_ids = tokenizer(input_text, return_tensors='pt').input_ids.cuda()
output = model.generate(inputs=input_ids, temperature=0.7, max_new_tokens=256, min_new_tokens=100)
print(tokenizer.decode(output[0]))