【大模型学习 | CLIP 原理&实现】

# 线性映射文本、图像# 线性映射文本、图像# 线性映射文本、图像# 线性映射文本、图像te'z
I_e = l2_normalize(np.dot(I_f, W_i), axis=1)
T_e = l2_normalize(np.dot(T_f, W_t), axis=1)

# 图文相似矩阵向量
logits = np.dot(I_e, T_e.T) * np.exp(t)师弟

# symeertric entropy loss
loss_i = cross_entropy_loss(logits, labels, axis=0)  # 图像 → 文本，即图像作为query
loss_t = cross_entropy_loss(logits, labels, axis=1)  # 文本 → 图像，即文本作为query
loss = (loss_i + loss_t) / 2

from PIL import Image
import requests

from transformers import CLIPProcessor, CLIPModel

model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True)

outputs = model(**inputs)
logits_per_image = outputs.logits_per_image # this is the image-text similarity score
probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities

from transformers import CLIPModel
import torch
import torch.quantization

model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")

# 对 Linear 层做动态量化
quantized_model = torch.quantization.quantize_dynamic(
    model,
    {torch.nn.Linear},  # 可以加上 torch.nn.LayerNorm 等
    dtype=torch.qint8
)

# 保存模型
torch.save(quantized_model.state_dict(), "clip_quantized.pth")

import torch
from torch import nn
from torch.utils.data import DataLoader, Dataset
from transformers import CLIPProcessor, CLIPModel
from PIL import Image

# 1. 加载模型和 processor
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

# 2. 冻结图像和文本编码器（只训练投影头）
for param in model.vision_model.parameters():
    param.requires_grad = False
for param in model.text_model.parameters():
    param.requires_grad = False

# 3. 自定义小数据集（图像路径 + 对应文本）
class MyDataset(Dataset):
    def __init__(self, image_paths, texts):
        self.image_paths = image_paths
        self.texts = texts

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        image = Image.open(self.image_paths[idx]).convert("RGB")
        text = self.texts[idx]
        return image, text

# 替换为你自己的图像路径和描述文本
image_paths = ["cat.jpg", "dog.jpg", "apple.jpg"]
texts = ["a photo of a cat", "a photo of a dog", "a photo of an apple"]
dataset = MyDataset(image_paths, texts)
dataloader = DataLoader(dataset, batch_size=2, shuffle=True)

# 4. 定义优化器（只优化 projection 层）
optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=5e-5)
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# 5. 训练 loop
for epoch in range(10):
    model.train()
    total_loss = 0
    for images, texts in dataloader:
        inputs = processor(text=texts, images=images, return_tensors="pt", padding=True, truncation=True).to(device)
        outputs = model(**inputs)

        logits_per_image = outputs.logits_per_image  # [batch, batch]
        logits_per_text = outputs.logits_per_text    # [batch, batch]
        ground_truth = torch.arange(len(images)).to(device)

        # CLIP 的双向对比损失
        loss_i = nn.CrossEntropyLoss()(logits_per_image, ground_truth)
        loss_t = nn.CrossEntropyLoss()(logits_per_text, ground_truth)
        loss = (loss_i + loss_t) / 2

        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        total_loss += loss.item()

    print(f"Epoch {epoch+1}, Loss: {total_loss:.4f}")