大模型应用：大模型的token频率偏见：高频词与低频词的嵌入表示差异分析.46

from transformers import AutoTokenizer, AutoModel
import torch
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
from modelscope.hub.snapshot_download import snapshot_download

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'Arial']
plt.rcParams['axes.unicode_minus'] = False

cache_dir = "D:\\modelscope\\hub"
model_name = "google-bert/bert-base-chinese"
# 下载模型到本地
local_bert_path = snapshot_download(
    model_name,
    cache_dir=cache_dir,
    revision="master"
)

# 2. 加载模型和分词器（选用轻量级的bert-base-chinese，CPU可运行）
tokenizer = AutoTokenizer.from_pretrained(local_bert_path)
model = AutoModel.from_pretrained(local_bert_path)

# 3. 选择测试词：高频词列表 + 低频词列表
high_freq_words = ["苹果", "手机", "人工智能", "吃饭", "睡觉"]  # 日常高频
low_freq_words = ["氍毹", "䴙䴘", "横向联邦", "量子纠缠", "熵增定律"]  # 低频/专业

# 4. 提取Token和嵌入向量
def get_word_embedding(word):
    # 分词：获取word的Token ID
    inputs = tokenizer(word, return_tensors="pt", padding=True, truncation=True)
    # 前向传播：获取模型输出（包含嵌入向量）
    with torch.no_grad():  # 关闭梯度计算，节省CPU资源
        outputs = model(**inputs)
    # 取[CLS] token的向量作为单词的整体嵌入（简化处理）
    embedding = outputs.last_hidden_state[:, 0, :].squeeze().numpy()
    return embedding

# 5. 批量获取向量
high_freq_embeddings = [get_word_embedding(word) for word in high_freq_words]
low_freq_embeddings = [get_word_embedding(word) for word in low_freq_words]

# 6. t-SNE降维：将高维向量降到2维，方便可视化
import numpy as np
all_embeddings = high_freq_embeddings + low_freq_embeddings
all_embeddings = np.array(all_embeddings)  # 转换为numpy数组
tsne = TSNE(n_components=2, random_state=42, perplexity=3)  # perplexity适配小样本
tsne_results = tsne.fit_transform(all_embeddings)

# 7. 绘图：对比高频词和低频词的向量分布
plt.figure(figsize=(10, 6))
plt.subplots_adjust(left=0.08, right=0.95, top=0.95, bottom=0.08)  # 缩小边距
# 绘制高频词：红色，圆形
plt.scatter(tsne_results[:5, 0], tsne_results[:5, 1], c="red", label="高频词", marker="o", s=100)
# 绘制低频词：蓝色，三角形
plt.scatter(tsne_results[5:, 0], tsne_results[5:, 1], c="blue", label="低频词", marker="^", s=100)

# 添加词标签
for i, word in enumerate(high_freq_words + low_freq_words):
    plt.annotate(word, (tsne_results[i, 0], tsne_results[i, 1]), fontsize=12)

plt.title("Token频率偏见：高频词 vs 低频词 嵌入向量分布")
plt.legend()
plt.grid(True)
plt.show()

# 1. 安装依赖
# pip install transformers torch scikit-learn matplotlib pandas

# 2. 导入库
from transformers import AutoTokenizer, AutoModel
import torch
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from modelscope.hub.snapshot_download import snapshot_download

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'Arial']
plt.rcParams['axes.unicode_minus'] = False

cache_dir = "D:\\modelscope\\hub"
model_name = "google-bert/bert-base-chinese"
# 下载模型到本地
local_bert_path = snapshot_download(
    model_name,
    cache_dir=cache_dir,
    revision="master"
)

# 3. 加载基础模型（以中文医疗场景为例）
tokenizer = AutoTokenizer.from_pretrained(local_bert_path)
model = AutoModel.from_pretrained(local_bert_path)
model.eval()  # 评估模式，关闭梯度

# 4. 定义高频/低频Token（医疗场景示例）
high_freq_tokens = ["发热", "咳嗽", "头痛", "乏力", "恶心"]  # 常见症状高频
low_freq_tokens = ["亨廷顿舞蹈症", "卟啉病", "氍毹样皮疹", "横向联邦学习", "票据转贴现"]  # 专业低频

# 5. 提取Token嵌入向量
def get_embedding(token):
    """获取单个Token的嵌入向量"""
    inputs = tokenizer(token, return_tensors="pt", padding=True, truncation=True)
    with torch.no_grad():
        outputs = model(**inputs)
    # 取[CLS]位置的向量作为Token的整体嵌入（简化处理）
    embedding = outputs.last_hidden_state[:, 0, :].squeeze().numpy()
    return embedding

# 6. 批量提取向量
high_embeddings = [get_embedding(t) for t in high_freq_tokens]
low_embeddings = [get_embedding(t) for t in low_freq_tokens]
all_embeddings = high_embeddings + low_embeddings
all_labels = high_freq_tokens + low_freq_tokens

# 7. TSNE降维（高维向量→2维，方便可视化）
all_embeddings = np.array(all_embeddings)  # 转换为numpy数组

# 中心化处理：将所有向量减去均值，使高频词更容易聚集在中心
all_embeddings_centered = all_embeddings - np.mean(all_embeddings, axis=0)

# TSNE参数微调：降低perplexity以增强局部聚集，early_exaggeration增强分离
tsne = TSNE(n_components=2, random_state=123, perplexity=2, early_exaggeration=20, max_iter=2000)
tsne_results = tsne.fit_transform(all_embeddings_centered)

# 8. 可视化原始嵌入分布（偏见诊断图）
plt.figure(figsize=(12, 8))
plt.subplots_adjust(left=0.08, right=0.95, top=0.92, bottom=0.08)  # 缩小边距

# 绘制高频Token：红色圆形，标注清晰，增加边缘宽度
plt.scatter(tsne_results[:5, 0], tsne_results[:5, 1], c="red", label="高频Token", marker="o", s=200, edgecolors='darkred', linewidth=2)
# 绘制低频Token：蓝色三角形
plt.scatter(tsne_results[5:, 0], tsne_results[5:, 1], c="blue", label="低频Token", marker="^", s=150, edgecolors='darkblue', linewidth=1)

# 添加中心标记（高频词期望聚集区域）
plt.axhline(y=0, color='gray', linestyle='--', alpha=0.3)
plt.axvline(x=0, color='gray', linestyle='--', alpha=0.3)

# 添加Token标签，偏移避免重叠
for i, label in enumerate(all_labels):
    offset_x = -0.5 if i < 5 else 0.5
    offset_y = 0.3 if i < 5 else -0.3
    plt.annotate(label, (tsne_results[i, 0] + offset_x, tsne_results[i, 1] + offset_y),
                 fontsize=11, ha="center", va="center",
                 bbox=dict(boxstyle='round,pad=0.3', facecolor='white', alpha=0.7))

plt.title("Token频率偏见诊断：原始嵌入向量分布（医疗场景）\n高频词应聚集在中心区域", fontsize=14)
plt.xlabel("TSNE维度1", fontsize=12)
plt.ylabel("TSNE维度2", fontsize=12)
plt.legend(fontsize=12, loc='upper right')
plt.grid(alpha=0.3)
plt.savefig("原始嵌入分布.png", dpi=300, bbox_inches="tight")
plt.show()

from modelscope.hub.snapshot_download import snapshot_download

cache_dir = "D:\\modelscope\\hub"
model_name = "google-bert/bert-base-chinese"
# 下载模型到本地
local_bert_path = snapshot_download(
    model_name,
    cache_dir=cache_dir,
    revision="master"
)

# 1. 自定义分词器（扩展bert-base-chinese的词表）
from transformers import BertTokenizer,AutoModel

# 2. 加载原始分词器
tokenizer = BertTokenizer.from_pretrained(local_bert_path)
model = AutoModel.from_pretrained(local_bert_path)
# 3. 定义需要加入的低频专业术语（医疗场景）
new_special_tokens = ["亨廷顿舞蹈症", "卟啉病", "氍毹样皮疹"]

# 4. 添加新Token到分词器词表
tokenizer.add_tokens(new_special_tokens)
# 同步更新模型的嵌入层（适配新Token）
model.resize_token_embeddings(len(tokenizer))

# 5. 验证分词效果（对比优化前后）
test_sentence = "患者出现氍毹样皮疹，疑似亨廷顿舞蹈症"
# 优化前分词（拆分低频Token）
old_tokenizer = BertTokenizer.from_pretrained(local_bert_path)
old_tokens = old_tokenizer.tokenize(test_sentence)
# 优化后分词（完整保留低频Token）
new_tokens = tokenizer.tokenize(test_sentence)

print("优化前分词结果：", old_tokens)
print("优化后分词结果：", new_tokens)

# 1. 安装LoRA依赖
# pip install peft datasets accelerate

# 2. 导入库
from peft import LoraConfig, get_peft_model, TaskType
from datasets import Dataset
import torch
from transformers import TrainingArguments, Trainer, AutoModelForSequenceClassification
from modelscope.hub.snapshot_download import snapshot_download
import matplotlib.pyplot as plt

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'Arial']
plt.rcParams['axes.unicode_minus'] = False

cache_dir = "D:\\modelscope\\hub"
model_name = "google-bert/bert-base-chinese"
# 下载模型到本地
local_bert_path = snapshot_download(
    model_name,
    cache_dir=cache_dir,
    revision="master"
)
# 3. 准备低频Token扩充语料（医疗场景示例）
train_data = [
    {"text": "亨廷顿舞蹈症是一种常染色体显性遗传的神经退行性疾病", "label": 1},
    {"text": "卟啉病主要表现为皮肤光敏、腹痛等症状", "label": 1},
    {"text": "氍毹样皮疹是罕见病的典型皮肤表现", "label": 1},
    {"text": "普通皮疹与氍毹样皮疹的鉴别要点是形态和分布", "label": 0},
    {"text": "亨廷顿舞蹈症的诊断需结合基因检测", "label": 1}
]
# 转为Dataset格式
dataset = Dataset.from_list(train_data)

# 4. 配置LoRA（轻量化微调，避免全量训练）
lora_config = LoraConfig(
    task_type=TaskType.SEQ_CLS,  # 序列分类任务（可根据场景调整）
    r=8,  # 低秩矩阵维度
    lora_alpha=32,
    lora_dropout=0.1,
    target_modules=["bert.encoder.layer.11.attention.self.query", "bert.encoder.layer.11.attention.self.key"]  # 针对BERT最后一层
)

# 5. 加载模型并绑定LoRA
model = AutoModelForSequenceClassification.from_pretrained(local_bert_path, num_labels=2)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # 查看可训练参数（仅占0.1%左右）

# 6. 训练配置
training_args = TrainingArguments(
    output_dir="./lora_finetune",
    per_device_train_batch_size=2,
    num_train_epochs=5,
    logging_steps=10,
    learning_rate=1e-4,
    weight_decay=0.01,
    fp16=False  # CPU环境关闭混合精度
)

# 7. 定义Trainer并训练
def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=128)

tokenized_dataset = dataset.map(tokenize_function, batched=True)
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_dataset
)
trainer.train()

# 8. 保存微调后的模型
model.save_pretrained("./medical_lora_model")
tokenizer.save_pretrained("./medical_lora_tokenizer")

# 1. 加载微调后的模型
from transformers import AutoTokenizer, AutoModel

finetuned_tokenizer = AutoTokenizer.from_pretrained("./medical_lora_tokenizer")
finetuned_model = AutoModel.from_pretrained("./medical_lora_model")
finetuned_model.eval()

# 2. 重新提取优化后的低频Token嵌入
def get_finetuned_embedding(token):
    inputs = finetuned_tokenizer(token, return_tensors="pt", padding=True, truncation=True)
    with torch.no_grad():
        outputs = finetuned_model(**inputs)
    embedding = outputs.last_hidden_state[:, 0, :].squeeze().numpy()
    return embedding

# 3. 提取优化后向量
optimized_low_embeddings = [get_finetuned_embedding(t) for t in low_freq_tokens]
optimized_all_embeddings = high_embeddings + optimized_low_embeddings

# 4. TSNE降维
tsne_optimized = TSNE(n_components=2, random_state=42, perplexity=3)
tsne_optimized_results = tsne_optimized.fit_transform(optimized_all_embeddings)

# 5. 可视化优化后分布
plt.figure(figsize=(12, 8))
# 高频Token：红色圆形
plt.scatter(tsne_optimized_results[:5, 0], tsne_optimized_results[:5, 1], c="red", label="高频Token", marker="o", s=150)
# 优化后低频Token：绿色正方形（对比之前的蓝色三角形）
plt.scatter(tsne_optimized_results[5:, 0], tsne_optimized_results[5:, 1], c="green", label="优化后低频Token", marker="s", s=150)

# 添加标签
for i, label in enumerate(all_labels):
    plt.annotate(label, (tsne_optimized_results[i, 0], tsne_optimized_results[i, 1]), fontsize=10, ha="center")

plt.title("Token频率偏见优化后：嵌入向量分布（医疗场景）", fontsize=14)
plt.xlabel("TSNE维度1", fontsize=12)
plt.ylabel("TSNE维度2", fontsize=12)
plt.legend(fontsize=12)
plt.grid(alpha=0.3)
plt.savefig("优化后嵌入分布.png", dpi=300, bbox_inches="tight")
plt.show()

# 1. 加载模型和分词器
from transformers import AutoTokenizer, AutoModelForCausalLM
from modelscope import snapshot_download
import torch
# 下载模型到./model文件夹
model_name = "qwen/Qwen1.5-1.8B-Chat"
cache_dir = "D:\\modelscope\\hub"
print("正在下载/校验模型缓存...")
local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
tokenizer = AutoTokenizer.from_pretrained(local_model_path)
model = AutoModelForCausalLM.from_pretrained(local_model_path)

# 2. 定义针对低频Token的Prompt模板
def build_prompt(question):
    prompt = f"""
    你是专业的医疗顾问，需遵循以下规则回答问题：
    1. 优先检索《罕见病诊疗指南》和专科病历库；
    2. 对于"亨廷顿舞蹈症""卟啉病""氍毹样皮疹"等术语，必须给出核心症状、诊断标准和治疗方案；
    3. 禁止仅输出通用套话，需包含具体医学数据。
    
    问题：{question}
    回答：
    """
    return prompt

# 3. 测试低频Token问答
question = "什么是亨廷顿舞蹈症？需要做哪些检查？"
prompt = build_prompt(question)

# 4. 生成回答
inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True)
with torch.no_grad():
    outputs = model.generate(**inputs, max_new_tokens=200, temperature=0.7)
answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
print("优化后回答：", answer)