大模型应用：多尺度注意力解析：大模型特征提取的基础概念与应用价值.43

import torch
import torch.nn as nn
import torch.nn.functional as F

# 极简多尺度注意力模块（文本处理）
# 功能：通过不同尺度的卷积核提取文本特征，并动态融合多尺度特征
# 结构说明：
# 1. 包含两个卷积层（small_conv和large_conv）分别提取细粒度和粗粒度特征
# 2. 通过attention_weight层动态计算两个尺度特征的权重
# 3. 最终输出加权融合后的特征
class SimpleMultiScaleAttention(nn.Module):
    def __init__(self, embed_dim=128):
        super().__init__()
        self.embed_dim = embed_dim  # 词向量维度
        # 定义两个不同尺度的窗口卷积（用于提取不同粒度特征）
        self.small_conv = nn.Conv1d(embed_dim, embed_dim, kernel_size=2, padding=1)  # 小窗口（细粒度）：关注相邻2个词
        self.large_conv = nn.Conv1d(embed_dim, embed_dim, kernel_size=4, padding=2)  # 大窗口（粗粒度）：关注相邻4个词
        # 定义权重计算层（判断两个尺度特征的重要性）
        self.attention_weight = nn.Linear(embed_dim * 2, 2)  # 输入：两个尺度特征拼接；输出：2个权重（对应两个尺度）

    def forward(self, x):
        # 输入说明：
        # x: 输入的词向量序列，shape=(batch_size, seq_len, embed_dim)
        # 例如：(2, 5, 128)表示2个样本，每个样本5个词，每个词128维向量
        batch_size, seq_len, embed_dim = x.shape
        
        # 步骤1：调整维度以适应Conv1d
        # Conv1d需要输入shape=(batch_size, embed_dim, seq_len)
        print("步骤1：调整输入维度以适应Conv1d")
        print(f"原始输入维度: {x.shape}")
        x_trans = x.transpose(1, 2)  # 转置后shape=(2, 128, 5)
        print(f"调整后维度: {x_trans.shape}")
        
        # 步骤2：多尺度特征提取
        # 细粒度特征：关注相邻2个词（小窗口）
        fine_feature = self.small_conv(x_trans)  # shape=(2, 128, 5)
        print("\n步骤2：多尺度特征提取")
        print("细粒度特征提取完成，shape:", fine_feature.shape)
        # 粗粒度特征：关注相邻4个词（大窗口）
        coarse_feature = self.large_conv(x_trans)  # shape=(2, 128, 5)
        print("粗粒度特征提取完成，shape:", coarse_feature.shape)
        
        # 步骤3：调整维度回原格式
        # 恢复为(batch_size, seq_len, embed_dim)以计算权重
        print("\n步骤3：调整维度回原格式")
        fine_feature = fine_feature.transpose(1, 2)  # (2, 5, 128)
        print("细粒度特征调整维度完成，shape:", fine_feature.shape)
        coarse_feature = coarse_feature.transpose(1, 2)  # (2, 5, 128)
        print("粗粒度特征调整维度完成，shape:", coarse_feature.shape)
        
        # 步骤4：动态权重计算
        # 拼接两个尺度的特征：(batch_size, seq_len, embed_dim*2)
        print("\n步骤4：动态权重计算")
        combined_feature = torch.cat([fine_feature, coarse_feature], dim=-1)
        print("特征拼接完成，shape:", combined_feature.shape)
        # 计算权重（softmax归一化）
        weights = F.softmax(self.attention_weight(combined_feature), dim=-1)  # (2, 5, 2)
        print("权重计算完成，shape:", weights.shape)
        weight_fine, weight_coarse = weights[..., 0:1], weights[..., 1:2]  # 拆分权重
        print("权重拆分完成，细粒度权重shape:", weight_fine.shape, "粗粒度权重shape:", weight_coarse.shape)
        # 步骤5：特征融合（加权求和）
        print("\n步骤5：特征融合")
        fused_feature = weight_fine * fine_feature + weight_coarse * coarse_feature  # (2, 5, 128)
        print("特征融合完成，shape:", fused_feature.shape)
        
        return fused_feature

# 测试代码
if __name__ == "__main__":
    # 构造模拟的词向量序列（2个样本，每个样本5个词，每个词128维）
    x = torch.randn(2, 5, 128)
    # 初始化多尺度注意力模块
    multi_scale_attn = SimpleMultiScaleAttention(embed_dim=128)
    # 执行前向传播，得到融合后的特征
    fused_output = multi_scale_attn(x)
    print("融合后特征的shape：", fused_output.shape)  

import torch
import torch.nn as nn
import torch.nn.functional as F

# 1. 定义简易多尺度注意力模块（适配案例文本）
class CaseMultiScaleAttention(nn.Module):
    def __init__(self, vocab_size=100, embed_dim=16):
        super().__init__()
        self.embed_dim = embed_dim
        # 词嵌入层：将词语转为向量（适配案例文本的简易词汇表）
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        # 细粒度提取：小窗口卷积（窗口大小2，捕捉相邻词搭配）
        self.fine_conv = nn.Conv1d(embed_dim, embed_dim, kernel_size=2, padding=1)
        # 粗粒度提取：全局平均池化（捕捉句子主旨）
        self.coarse_pool = nn.AdaptiveAvgPool1d(1)
        # 权重计算层：动态判断细/粗粒度重要性
        self.weight_layer = nn.Linear(embed_dim * 2, 2)
        # 地点分类头
        self.location_head = nn.Linear(embed_dim, 2)  # 假设地点类别数为2（如“图书馆”和其他）
        # 行为识别头
        self.action_head = nn.Linear(embed_dim, 2)  # 假设行为类别数为2（如“看书”和其他）

    def forward(self, x):
        # x: 输入的词语索引序列，shape=(batch_size, seq_len)
        batch_size, seq_len = x.shape
        
        # 步骤2：数据预处理（词嵌入，将词语转为向量）
        embedded = self.embedding(x)  # shape=(batch_size, seq_len, embed_dim)
        print(f"步骤2-词嵌入后特征shape: {embedded.shape}")
        
        # 步骤3：多尺度特征提取
        # 3.1 细粒度特征提取（小窗口卷积）
        embedded_trans = embedded.transpose(1, 2)  # 调整维度适配Conv1d，shape=(batch_size, embed_dim, seq_len)
        fine_feature = self.fine_conv(embedded_trans)  # shape=(batch_size, embed_dim, seq_len)
        fine_feature = fine_feature.transpose(1, 2)  # 恢复原维度，shape=(batch_size, seq_len, embed_dim)
        print(f"步骤3-细粒度特征shape: {fine_feature.shape}")
        
        # 3.2 粗粒度特征提取（全局平均池化）
        coarse_feature = self.coarse_pool(embedded_trans)  # shape=(batch_size, embed_dim, 1)
        coarse_feature = coarse_feature.squeeze(-1)  # 压缩维度，shape=(batch_size, embed_dim)
        # 扩展维度，与细粒度特征对齐（便于后续融合）
        coarse_feature = coarse_feature.unsqueeze(1).repeat(1, fine_feature.size(1), 1)  # shape=(batch_size, seq_len, embed_dim)
        print(f"步骤3-粗粒度特征shape: {coarse_feature.shape}")
        
        # 步骤4：动态权重计算
        combined = torch.cat([fine_feature, coarse_feature], dim=-1)  # 拼接两尺度特征，shape=(batch_size, seq_len, 2*embed_dim)
        weights = F.softmax(self.weight_layer(combined), dim=-1)  # 计算权重，shape=(batch_size, seq_len, 2)
        weight_fine, weight_coarse = weights[..., 0:1], weights[..., 1:2]  # 拆分细/粗粒度权重
        print(f"步骤4-细粒度权重示例: {weight_fine[0][:3].detach().numpy()}")  # 打印前3个词的细粒度权重
        print(f"步骤4-粗粒度权重示例: {weight_coarse[0][:3].detach().numpy()}")  # 打印前3个词的粗粒度权重
        
        # 步骤5：特征融合与输出
        fused_feature = weight_fine * fine_feature + weight_coarse * coarse_feature  # 加权融合，shape=(batch_size, seq_len, embed_dim)
        print(f"步骤5-融合后特征shape: {fused_feature.shape}")
        # 地点分类和行为识别
        location_logits = self.location_head(fused_feature.max(dim=1).values)  # 最大池化后分类
        action_logits = self.action_head(fused_feature[:, -1, :])  # 使用序列最后一个时间步的特征作为行为分类输入
        return fused_feature, location_logits, action_logits

# 测试代码：复现“小明在图书馆认真看书”的处理流程
if __name__ == "__main__":
    # 步骤1：原始数据输入与词汇映射（模拟简易词汇表）
    sentence = "小明在图书馆认真看书"
    words = ['小明', '在', '图书馆', '认真', '看书']  # 手动分词
    # 构建简易词汇表（将每个词映射为唯一索引）
    vocab = {"小明": 0, "在": 1, "图书馆": 2, "认真": 3, "看书": 4, "<PAD>": 5}
    # 转换为词语索引序列（模型输入格式）
    word_indices = torch.tensor([[vocab[word] for word in words]])  # shape=(1, 5)，batch_size=1，seq_len=5
    print(f"步骤1-原始文本: {sentence}")
    print(f"步骤1-词语索引序列: {word_indices}")
    
    # 初始化模型并执行流程
    model = CaseMultiScaleAttention(vocab_size=len(vocab), embed_dim=16)
    model.eval()  # 推理模式，关闭梯度计算
    with torch.no_grad():
        fused_feature, location_logits, action_logits = model(word_indices)
        # 打印分类结果
        location_pred = torch.argmax(location_logits, dim=1).item()
        action_pred = torch.argmax(action_logits, dim=1).item()
        print(f"地点分类结果: {'图书馆' if location_pred == 0 else '其他'}")
        print(f"行为识别结果: {'看书' if action_pred == 0 else '其他'}")
        print(f"地点分类概率: {F.softmax(location_logits, dim=1)[0].detach().numpy()}")
        print(f"行为识别概率: {F.softmax(action_logits, dim=1)[0].detach().numpy()}")