从单模态到多模态AI Agent在情感计算中的深度学习方法演进

import torch
import torch.nn as nn
from transformers import BertTokenizer, BertModel
from torchvision import models, transforms
from PIL import Image

# 1. 文本编码 (BERT)
class TextEncoder(nn.Module):
    def __init__(self):
        super(TextEncoder, self).__init__()
        self.bert = BertModel.from_pretrained("bert-base-uncased")
        
    def forward(self, input_ids, attention_mask):
        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
        return outputs.pooler_output  # [batch, hidden_size]

# 2. 图像编码 (ResNet50)
class ImageEncoder(nn.Module):
    def __init__(self):
        super(ImageEncoder, self).__init__()
        resnet = models.resnet50(pretrained=True)
        self.feature_extractor = nn.Sequential(*list(resnet.children())[:-1])  
        
    def forward(self, x):
        x = self.feature_extractor(x)
        return x.view(x.size(0), -1)  # [batch, feature_dim]

# 3. 多模态融合 + 分类
class MultiModalSentiment(nn.Module):
    def __init__(self, text_dim=768, img_dim=2048, hidden_dim=512, num_classes=3):
        super(MultiModalSentiment, self).__init__()
        self.fc = nn.Sequential(
            nn.Linear(text_dim + img_dim, hidden_dim),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(hidden_dim, num_classes)
        )
    
    def forward(self, text_feat, img_feat):
        fusion = torch.cat([text_feat, img_feat], dim=1)
        return self.fc(fusion)

# 4. 模拟推理
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
text = "I am very happy today!"
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)

# 文本编码
text_encoder = TextEncoder()
text_feat = text_encoder(inputs["input_ids"], inputs["attention_mask"])

# 图像编码
img = Image.open("happy_face.jpg").convert("RGB")
transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor()
])
img_tensor = transform(img).unsqueeze(0)
img_encoder = ImageEncoder()
img_feat = img_encoder(img_tensor)

# 融合模型
model = MultiModalSentiment()
logits = model(text_feat, img_feat)
pred = torch.argmax(logits, dim=1)
print("Predicted sentiment:", pred.item())  # 输出 0=负面,1=中性,2=正面