Qwen3.5 0.8B/2B/4B/9B 小模型本地部署指南，微调教程

# 克隆最新代码
git clone https://github.com/ggml-org/llama.cpp.git
cd llama.cpp

# macOS / CPU 编译
cmake -B build -DGGML_CUDA=OFF
cmake --build build --config Release -j

# 如果有 NVIDIA GPU，改成：
# cmake -B build -DGGML_CUDA=ON
# cmake --build build --config Release -j

pip install huggingface_hub hf_transfer

# 下载 9B 的 Q4_K_M 量化版本
huggingface-cli download unsloth/Qwen3.5-9B-GGUF \
  --include "Qwen3.5-9B-Q4_K_M.gguf" \
  --local-dir ./models

./build/bin/llama-cli \
  -m ./models/Qwen3.5-9B-Q4_K_M.gguf \
  --ctx-size 16384 \
  -cnv

./build/bin/llama-server \
  -m ./models/Qwen3.5-9B-Q4_K_M.gguf \
  --ctx-size 16384 \
  --chat-template-kwargs '{"enable_thinking":true}'

# Non-Thinking 模式（默认，推荐日常使用）
./build/bin/llama-server \
  -m ./models/Qwen3.5-9B-Q4_K_M.gguf \
  --ctx-size 16384 \
  --port 8080 \
  --n-gpu-layers 35

# Thinking 模式
./build/bin/llama-server \
  -m ./models/Qwen3.5-9B-Q4_K_M.gguf \
  --ctx-size 16384 \
  --port 8080 \
  --n-gpu-layers 35 \
  --chat-template-kwargs '{"enable_thinking":true}'

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="EMPTY"
)

response = client.chat.completions.create(
    model="Qwen3.5-9B",
    messages=[
        {"role": "user", "content": "用 Python 写一个快速排序"}
    ],
    temperature=0.7,
    top_p=0.8,
    max_tokens=4096
)

print(response.choices[0].message.content)

# vLLM 部署
vllm serve Qwen/Qwen3.5-9B \
  --port 8000 \
  --tensor-parallel-size 1 \
  --max-model-len 32768 \
  --reasoning-parser qwen3

# SGLang 部署
python -m sglang.launch_server \
  --model-path Qwen/Qwen3.5-9B \
  --port 8000 \
  --tp-size 1 \
  --mem-fraction-static 0.8 \
  --context-length 32768 \
  --reasoning-parser qwen3

pip install --upgrade --force-reinstall --no-cache-dir unsloth unsloth_zoo

from unsloth import FastLanguageModel
import torch
from datasets import load_dataset
from trl import SFTTrainer, SFTConfig

max_seq_length = 2048# 先从小的开始，跑通再加大

# 加载示例数据集（替换成你自己的）
url = "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl"
dataset = load_dataset("json", data_files={"train": url}, split="train")

# 加载 Qwen3.5-9B（可以换成 0.8B/2B/4B）
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "Qwen/Qwen3.5-9B",
    max_seq_length = max_seq_length,
    load_in_4bit = True,     # 4-bit QLoRA，省显存
    full_finetuning = False,
)

# 挂上 LoRA 适配器
model = FastLanguageModel.get_peft_model(
    model,
    r = 16,
    target_modules = [
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj",
    ],
    lora_alpha = 16,
    lora_dropout = 0,
    bias = "none",
    use_gradient_checkpointing = "unsloth",  # 降低显存 + 支持更长上下文
    random_state = 3407,
    max_seq_length = max_seq_length,
)

# 开始训练
trainer = SFTTrainer(
    model = model,
    train_dataset = dataset,
    tokenizer = tokenizer,
    args = SFTConfig(
        max_seq_length = max_seq_length,
        per_device_train_batch_size = 1,
        gradient_accumulation_steps = 4,
        warmup_steps = 10,
        max_steps = 100,        # 先跑 100 步看看效果
        logging_steps = 1,
        output_dir = "outputs_qwen35",
        optim = "adamw_8bit",
        seed = 3407,
    ),
)
trainer.train()

from unsloth import FastVisionModel

model = FastVisionModel.get_peft_model(
    model,
    finetune_vision_layers  = True,   # 微调视觉层
    finetune_language_layers = True,  # 微调语言层
    finetune_attention_modules = True, # 微调注意力层
    finetune_mlp_modules = True,       # 微调 MLP 层
    r = 16,
    lora_alpha = 16,
    lora_dropout = 0,
    bias = "none",
    random_state = 3407,
    target_modules = "all-linear",
)

# 导出为 Q4_K_M 量化的 GGUF
model.save_pretrained_gguf("my_model", tokenizer, quantization_method="q4_k_m")

# 或者导出为 Q8 量化
model.save_pretrained_gguf("my_model", tokenizer, quantization_method="q8_0")

# 想上传到 HuggingFace？
model.push_to_hub_gguf("你的用户名/my_model", tokenizer, quantization_method="q4_k_m")

model.save_pretrained_merged("finetuned_model", tokenizer, save_method="merged_16bit")

# 或者上传到 HuggingFace
model.push_to_hub_merged("你的用户名/model", tokenizer, save_method="merged_16bit", token="")

model.save_pretrained("finetuned_lora")
tokenizer.save_pretrained("finetuned_lora")

VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 vllm serve Qwen/Qwen3.5-9B \
  --hf-overrides '{"text_config": {"rope_parameters": {"mrope_interleaved": true, "mrope_section": [11, 11, 10], "rope_type": "yarn", "rope_theta": 10000000, "partial_rotary_factor": 0.25, "factor": 4.0, "original_max_position_embeddings": 262144}}}' \
  --max-model-len 1010000