构建AI智能体：大模型性能评估技巧：本地模型的完整测试框架与可视化分析

import time
import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

def test_inference_speed(model, tokenizer):
    """测试推理速度"""
    print("=== 推理速度测试 ===")
    
    test_texts = [
        "介绍一下人工智能",
        "写一个简短的故事",
        "解释机器学习的基本概念"
    ]
    
    speeds = []
    for text in test_texts:
        start_time = time.time()
        
        inputs = tokenizer(text, return_tensors="pt", max_length=512, truncation=True)
        with torch.no_grad():
            outputs = model.generate(
                **inputs,
                max_new_tokens=100,
                do_sample=True,
                temperature=0.7
            )
        
        end_time = time.time()
        time_taken = end_time - start_time
        token_count = len(outputs[0]) - len(inputs['input_ids'][0])
        speed = token_count / time_taken
        
        speeds.append(speed)
        print(f"文本: {text[:30]}...")
        print(f"生成 {token_count} tokens, 耗时 {time_taken:.2f}s, 速度: {speed:.2f} tokens/s")
    
    avg_speed = sum(speeds) / len(speeds)
    print(f"\n平均生成速度: {avg_speed:.2f} tokens/秒")
    return avg_speed

# 主函数：串联所有评估方法
def main():

    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    results = test_inference_speed(model, tokenizer)
    return results

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM
import psutil

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 2. 内存占用分析
def analyze_memory_usage(model, tokenizer):
    """分析内存使用情况"""
    print("=== 内存使用分析 ===")
    
    # 模型参数内存
    param_memory = sum(p.numel() * p.element_size() for p in model.parameters())
    
    # 测试推理时的峰值内存
    process = psutil.Process()
    initial_memory = process.memory_info().rss
    
    # 执行推理测试
    test_text = "请介绍一下人工智能"
    inputs = tokenizer(test_text, return_tensors="pt")
    with torch.no_grad():
        outputs = model.generate(**inputs, max_new_tokens=50)
    
    peak_memory = process.memory_info().rss
    inference_memory = peak_memory - initial_memory
    
    print(f"模型参数内存: {param_memory / 1024 / 1024:.2f} MB")
    print(f"推理峰值内存: {inference_memory / 1024 / 1024:.2f} MB")
    print(f"总内存占用: {(param_memory + inference_memory) / 1024 / 1024:.2f} MB")
    
    return param_memory / 1024 / 1024  # 返回MB

# 主函数：
def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 执行内存使用分析
    print("\n步骤 2/2: 执行内存使用分析...")
    results = analyze_memory_usage(model, tokenizer)
    return results

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 辅助函数：生成回复
def generate_response(model, tokenizer, prompt, max_length=200):
    """通用的回复生成函数"""
    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # 移除输入部分，只保留生成的回复
    if prompt in response:
        response = response.replace(prompt, "").strip()
    return response

# 3. 中文理解能力测试
def test_chinese_understanding(model, tokenizer):
    """测试中文理解能力"""
    print("=== 中文理解能力测试 ===")
    
    test_cases = [
        {
            "category": "语义理解",
            "prompt": "这句话是什么意思：'他这是醉翁之意不在酒'",
            "expected_keywords": ["真实意图", "表面", "实际目的", "另有目的"]
        },
        {
            "category": "逻辑推理", 
            "prompt": "如果所有猫都喜欢鱼，而咪咪是一只猫，那么咪咪喜欢什么？",
            "expected_keywords": ["鱼", "喜欢"]
        },
        {
            "category": "上下文理解",
            "prompt": "小明说：'我昨天去了北京。' 小红问：'你去哪里了？' 小明回答：",
            "expected_keywords": ["北京"]
        }
    ]
    
    scores = []
    for case in test_cases:
        response = generate_response(model, tokenizer, case["prompt"])
        print(f"\n[{case['category']}]")
        print(f"问题: {case['prompt']}")
        print(f"回答: {response}")
        
        # 简单关键词匹配评分
        keyword_score = sum(1 for keyword in case["expected_keywords"] if keyword in response)
        score = keyword_score / len(case["expected_keywords"])
        scores.append(score)
        print(f"匹配度: {score:.2f}")
    
    avg_score = sum(scores) / len(scores)
    print(f"\n中文理解平均得分: {avg_score:.2f}/1.0")
    return avg_score

def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 执行中文理解能力测试
    print("\n步骤 2/2: 执行中文理解能力测试...")
    results = test_chinese_understanding(model, tokenizer)

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 辅助函数：生成回复
def generate_response(model, tokenizer, prompt, max_length=200):
    """通用的回复生成函数"""
    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # 移除输入部分，只保留生成的回复
    if prompt in response:
        response = response.replace(prompt, "").strip()
    return response

# 4. 知识准确性评估
def test_knowledge_accuracy(model, tokenizer):
    """测试知识准确性"""
    print("=== 知识准确性测试 ===")
    
    knowledge_questions = [
        {
            "question": "中国的首都是哪里？",
            "correct_answer": "北京",
            "category": "地理"
        },
        {
            "question": "Python是什么类型的编程语言？", 
            "correct_answer": "解释型",
            "category": "计算机"
        },
        {
            "question": "《红楼梦》的作者是谁？",
            "correct_answer": "曹雪芹",
            "category": "文学"
        }
    ]
    
    correct_count = 0
    for q in knowledge_questions:
        response = generate_response(model, tokenizer, q["question"])
        print(f"\n[{q['category']}] {q['question']}")
        print(f"模型回答: {response}")
        print(f"正确答案: {q['correct_answer']}")
        
        if q["correct_answer"] in response:
            correct_count += 1
            print("✅ 正确")
        else:
            print("❌ 错误")
    
    accuracy = correct_count / len(knowledge_questions)
    print(f"\n知识准确率: {accuracy:.2f}")
    return accuracy

# 主函数：
def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 执行执行知识准确性评估
    print("\n步骤 2/2: 执行知识准确性评估...")
    results = test_knowledge_accuracy(model, tokenizer)

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 辅助函数：生成回复
def generate_response(model, tokenizer, prompt, max_length=200):
    """通用的回复生成函数"""
    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # 移除输入部分，只保留生成的回复
    if prompt in response:
        response = response.replace(prompt, "").strip()
    return response

# 5. 对话连贯性测试
def test_conversation_coherence(model, tokenizer):
    """测试多轮对话连贯性"""
    print("=== 多轮对话连贯性测试 ===")
    
    conversation = [
        "你好，我叫小明",
        "你记得我的名字吗？",
        "我今天想去北京旅游，你有什么建议？",
        "我刚才说想去哪里旅游？"
    ]
    
    conversation_history = ""
    coherence_score = 0
    
    for i, user_input in enumerate(conversation):
        # 构建对话历史
        if i > 0:
            prompt = f"{conversation_history}用户: {user_input}\n助手:"
        else:
            prompt = f"用户: {user_input}\n助手:"
        
        response = generate_response(model, tokenizer, prompt)
        print(f"\n第{i+1}轮:")
        print(f"用户: {user_input}")
        print(f"助手: {response}")
        
        # 检查对话连贯性
        if i == 1 and "小明" in response:  # 应该记得名字
            coherence_score += 1
        elif i == 3 and "北京" in response:  # 应该记得旅游地点
            coherence_score += 1
            
        # 更新对话历史
        conversation_history += f"用户: {user_input}\n助手: {response}\n"
    
    final_score = coherence_score / 2  # 两个检查点
    print(f"\n对话连贯性得分: {final_score:.2f}/1.0")
    return final_score

# 6. 创造性测试
def test_creativity(model, tokenizer):
    """测试创造性思维能力"""
    print("=== 创造性思维测试 ===")
    
    creative_tasks = [
        "写一首关于秋天的四行诗",
        "为一个智能水杯想三个创新的功能",
        "用'月亮、猫咪、键盘'编一个简短的故事"
    ]
    
    creativity_scores = []
    
    for task in creative_tasks:
        response = generate_response(model, tokenizer, task)
        print(f"\n创意任务: {task}")
        print(f"生成内容:\n{response}")
        
        # 简单评估创造性
        score = min(len(response) / 100, 1.0)  # 简单长度评分
        creativity_scores.append(score)
        print(f"创造性评分: {score:.2f}")
    
    avg_creativity = sum(creativity_scores) / len(creativity_scores)
    print(f"\n平均创造性得分: {avg_creativity:.2f}/1.0")
    return avg_creativity

# 7. 代码生成能力
def test_code_generation(model, tokenizer):
    """测试代码生成能力"""
    print("=== 代码生成能力测试 ===")
    
    coding_tasks = [
        "写一个Python函数计算斐波那契数列",
        "用JavaScript实现数组去重",
        "写一个SQL查询，找出成绩大于90分的学生"
    ]
    
    code_scores = []
    
    for task in coding_tasks:
        response = generate_response(model, tokenizer, task)
        print(f"\n编程任务: {task}")
        print(f"生成代码:\n{response}")
        
        # 简单代码质量评估
        score = 0
        if "def " in response or "function" in response or "SELECT" in response:
            score += 0.5
        if "return" in response or "console.log" in response:
            score += 0.3
        if len(response) > 50:  # 有一定长度
            score += 0.2
            
        code_scores.append(score)
        print(f"代码质量评分: {score:.2f}")
    
    avg_code_score = sum(code_scores) / len(code_scores)
    print(f"\n平均代码生成得分: {avg_code_score:.2f}/1.0")
    return avg_code_score

# 主函数
def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 执行对话连贯性测试
    print("\n步骤 2/2: 执行对话连贯性测试...")
    results = test_conversation_coherence(model, tokenizer)
    
    return results

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")    
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 辅助函数：生成回复
def generate_response(model, tokenizer, prompt, max_length=200):
    """通用的回复生成函数"""
    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # 移除输入部分，只保留生成的回复
    if prompt in response:
        response = response.replace(prompt, "").strip()
    return response

# 6. 创造性测试
def test_creativity(model, tokenizer):
    """测试创造性思维能力"""
    print("=== 创造性思维测试 ===")
    
    creative_tasks = [
        "写一首关于秋天的四行诗",
        "为一个智能水杯想三个创新的功能",
        "用'月亮、猫咪、键盘'编一个简短的故事"
    ]
    
    creativity_scores = []
    
    for task in creative_tasks:
        response = generate_response(model, tokenizer, task)
        print(f"\n创意任务: {task}")
        print(f"生成内容:\n{response}")
        
        # 简单评估创造性
        score = min(len(response) / 100, 1.0)  # 简单长度评分
        creativity_scores.append(score)
        print(f"创造性评分: {score:.2f}")
    
    avg_creativity = sum(creativity_scores) / len(creativity_scores)
    print(f"\n平均创造性得分: {avg_creativity:.2f}/1.0")
    return avg_creativity

# 主函数
def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 执行创造性思维能力测试
    print("\n步骤 2/2: 执行创造性思维能力测试...")
    results = test_creativity(model, tokenizer)
    
    return results

# 执行评估
if __name__ == "__main__":
    main()

import torch
from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM

# 首先加载模型
def load_model_and_tokenizer():
    """加载模型和分词器"""
    cache_dir = "D:\\modelscope\\hub"
    model_name = "qwen/Qwen1.5-1.8B-Chat"
    
    print("正在加载模型...")
    local_model_path = snapshot_download(model_name, cache_dir=cache_dir)
    
    tokenizer = AutoTokenizer.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        use_fast=True,
        padding_side='left',
        truncation_side='right'
    )
    
    model = AutoModelForCausalLM.from_pretrained(
        local_model_path,
        trust_remote_code=True,
        torch_dtype=torch.float32,
        device_map="cpu",
        low_cpu_mem_usage=True
    )
    
    # 设置padding token
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    return model, tokenizer

# 辅助函数：生成回复
def generate_response(model, tokenizer, prompt, max_length=200):
    """通用的回复生成函数"""
    inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_length,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            pad_token_id=tokenizer.eos_token_id
        )
    
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # 移除输入部分，只保留生成的回复
    if prompt in response:
        response = response.replace(prompt, "").strip()
    return response

# 7. 代码生成能力
def test_code_generation(model, tokenizer):
    """测试代码生成能力"""
    print("=== 代码生成能力测试 ===")
    
    coding_tasks = [
        "写一个Python函数计算斐波那契数列",
        "用JavaScript实现数组去重",
        "写一个SQL查询，找出成绩大于90分的学生"
    ]
    
    code_scores = []
    
    for task in coding_tasks:
        response = generate_response(model, tokenizer, task)
        print(f"\n编程任务: {task}")
        print(f"生成代码:\n{response}")
        
        # 简单代码质量评估
        score = 0
        if "def " in response or "function" in response or "SELECT" in response:
            score += 0.5
        if "return" in response or "console.log" in response:
            score += 0.3
        if len(response) > 50:  # 有一定长度
            score += 0.2
            
        code_scores.append(score)
        print(f"代码质量评分: {score:.2f}")
    
    avg_code_score = sum(code_scores) / len(code_scores)
    print(f"\n平均代码生成得分: {avg_code_score:.2f}/1.0")
    return avg_code_score

# 主函数
def main():
    # 1. 加载模型
    print("步骤 1/2: 加载模型和分词器...")
    model, tokenizer = load_model_and_tokenizer()
    print("✅ 模型加载完成")
    
    # 2. 代码生成能力测试
    print("\n步骤 2/2: 代码生成能力测试...")
    results = test_code_generation(model, tokenizer)
    
    return results

# 执行评估
if __name__ == "__main__":
    main()