ROC / AUC 在什么场景下才有意义：安全攻防中的曲线解读艺术

真阳性率（TPR） = TP / (TP + FN)  # 召回率
假阳性率（FPR） = FP / (FP + TN)

精确率（Precision） = TP / (TP + FP)
召回率（Recall） = TP / (TP + FN)  # 真阳性率

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import (
    roc_curve, auc, precision_recall_curve, average_precision_score,
    f1_score, precision_score, recall_score
)

# 生成不同不平衡程度的安全数据集
def generate_imbalanced_data(imbalance_ratio):
    return make_classification(
        n_samples=10000,
        n_classes=2,
        weights=[imbalance_ratio, 1 - imbalance_ratio],
        random_state=42
    )

# 测试不同不平衡程度下的ROC-AUC和PR-AUC表现
imbalance_ratios = [0.5, 0.9, 0.95, 0.99]  # 从平衡到极端不平衡
results = []

for ratio in imbalance_ratios:
    # 生成数据
    X, y = generate_imbalanced_data(ratio)
    
    # 划分训练集和测试集
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42, stratify=y
    )
    
    # 训练随机森林模型
    model = RandomForestClassifier(
        n_estimators=100,
        max_depth=10,
        class_weight="balanced",
        random_state=42
    )
    model.fit(X_train, y_train)
    
    # 预测
    y_pred_proba = model.predict_proba(X_test)[:, 1]
    y_pred = model.predict(X_test)
    
    # 计算ROC曲线和AUC
    fpr, tpr, roc_thresholds = roc_curve(y_test, y_pred_proba)
    roc_auc = auc(fpr, tpr)
    
    # 计算PR曲线和PR-AUC
    precision, recall, pr_thresholds = precision_recall_curve(y_test, y_pred_proba)
    pr_auc = average_precision_score(y_test, y_pred_proba)
    
    # 计算其他指标
    f1 = f1_score(y_test, y_pred)
    precision_score_val = precision_score(y_test, y_pred)
    recall_score_val = recall_score(y_test, y_pred)
    
    # 记录结果
    results.append({
        "不平衡比例": ratio,
        "攻击样本比例": 1 - ratio,
        "ROC-AUC": roc_auc,
        "PR-AUC": pr_auc,
        "F1分数": f1,
        "精确率": precision_score_val,
        "召回率": recall_score_val
    })

# 转换为DataFrame进行分析
import pandas as pd
results_df = pd.DataFrame(results)
print("=== 不同不平衡程度下的指标表现 ===")
print(results_df.round(4))

# 可视化ROC-AUC和PR-AUC随不平衡程度的变化
plt.figure(figsize=(12, 6))

# ROC-AUC vs PR-AUC
plt.subplot(1, 2, 1)
plt.plot(results_df["攻击样本比例"], results_df["ROC-AUC"], label='ROC-AUC', marker='o')
plt.plot(results_df["攻击样本比例"], results_df["PR-AUC"], label='PR-AUC', marker='s')
plt.xlabel('攻击样本比例')
plt.ylabel('分数')
plt.title('ROC-AUC vs PR-AUC 随不平衡程度变化')
plt.legend()
plt.grid(True)

# F1分数变化
plt.subplot(1, 2, 2)
plt.plot(results_df["攻击样本比例"], results_df["F1分数"], label='F1分数', marker='o')
plt.plot(results_df["攻击样本比例"], results_df["精确率"], label='精确率', marker='s')
plt.plot(results_df["攻击样本比例"], results_df["召回率"], label='召回率', marker='^')
plt.xlabel('攻击样本比例')
plt.ylabel('分数')
plt.title('F1分数、精确率、召回率随不平衡程度变化')
plt.legend()
plt.grid(True)

plt.tight_layout()
plt.savefig('roc_vs_pr_auc.png')
print("\nROC-AUC与PR-AUC对比可视化完成，保存为roc_vs_pr_auc.png")

# 分析ROC-AUC的误导性
print("\n=== ROC-AUC误导性分析 ===")
print("当攻击样本比例为1%时:")
print(f"- ROC-AUC: {results_df.loc[3, 'ROC-AUC']:.4f} (看起来很好)")
print(f"- PR-AUC: {results_df.loc[3, 'PR-AUC']:.4f} (更真实地反映模型性能)")
print(f"- F1分数: {results_df.loc[3, 'F1分数']:.4f} (实际效果)")
print(f"- 精确率: {results_df.loc[3, '精确率']:.4f}")
print(f"- 召回率: {results_df.loc[3, '召回率']:.4f}")

print("\n结论: 随着数据集不平衡程度增加，ROC-AUC变得越来越具有误导性，而PR-AUC能更真实地反映模型在少数类上的性能。")

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score, average_precision_score, fbeta_score

# 生成安全数据集
X, y = make_classification(
    n_samples=10000,
    n_classes=2,
    weights=[0.95, 0.05],
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# 训练模型
model = RandomForestClassifier(
    n_estimators=100,
    max_depth=10,
    class_weight="balanced",
    random_state=42
)
model.fit(X_train, y_train)

y_pred_proba = model.predict_proba(X_test)[:, 1]

# 定义不同安全场景
scenarios = {
    "入侵检测": {
        "描述": "关注攻击检测，漏报成本高",
        "beta": 2.0,  # 更看重召回率
        "正样本重要性": "高",
        "误判成本不对称": "是"
    },
    "反垃圾邮件": {
        "描述": "关注正常邮件不被误判，误报成本高",
        "beta": 0.5,  # 更看重精确率
        "正样本重要性": "中",
        "误判成本不对称": "是"
    },
    "欺诈检测": {
        "描述": "平衡漏报和误报，两者成本都较高",
        "beta": 1.0,  # 平衡精确率和召回率
        "正样本重要性": "高",
        "误判成本不对称": "是"
    },
    "用户行为分类": {
        "描述": "用户行为分类，类别分布相对平衡",
        "beta": 1.0,
        "正样本重要性": "中",
        "误判成本不对称": "否"
    }
}

# 分析不同场景下的指标适用性
def analyze_scenario_applicability(y_true, y_pred_proba, scenario):
    """分析不同场景下ROC-AUC的适用性"""
    # 计算指标
    roc_auc = roc_auc_score(y_true, y_pred_proba)
    pr_auc = average_precision_score(y_true, y_pred_proba)
    
    # 基于阈值的指标（使用默认阈值0.5）
    y_pred = y_pred_proba >= 0.5
    f1 = fbeta_score(y_true, y_pred, beta=scenario["beta"])
    
    # 评估ROC-AUC的适用性
    applicability = "中"
    if scenario["正样本重要性"] == "高" and scenario["误判成本不对称"] == "是":
        applicability = "低"
    elif scenario["误判成本不对称"] == "否":
        applicability = "高"
    
    return {
        "ROC-AUC": roc_auc,
        "PR-AUC": pr_auc,
        "F-beta分数": f1,
        "ROC-AUC适用性": applicability
    }

# 分析不同场景
for scenario_name, scenario in scenarios.items():
    print(f"\n=== {scenario_name} 场景分析 ===")
    print(f"场景描述: {scenario['描述']}")
    print(f"F-beta的beta值: {scenario['beta']}")
    
    metrics = analyze_scenario_applicability(y_test, y_pred_proba, scenario)
    
    print(f"ROC-AUC: {metrics['ROC-AUC']:.4f}")
    print(f"PR-AUC: {metrics['PR-AUC']:.4f}")
    print(f"F-beta分数: {metrics['F-beta分数']:.4f}")
    print(f"ROC-AUC适用性: {metrics['ROC-AUC适用性']}")
    
    if metrics['ROC-AUC适用性'] == "低":
        print("建议: 使用PR-AUC替代ROC-AUC，或结合多个指标进行评估")
    elif metrics['ROC-AUC适用性'] == "中":
        print("建议: 结合ROC-AUC和PR-AUC进行评估")
    else:
        print("建议: ROC-AUC适用，可以作为主要评估指标")

# 可视化不同场景下的指标对比
scenario_names = list(scenarios.keys())
roc_aucs = []
pr_aucs = []
f_betas = []

for scenario_name, scenario in scenarios.items():
    metrics = analyze_scenario_applicability(y_test, y_pred_proba, scenario)
    roc_aucs.append(metrics['ROC-AUC'])
    pr_aucs.append(metrics['PR-AUC'])
    f_betas.append(metrics['F-beta分数'])

plt.figure(figsize=(12, 6))

# 不同场景下的指标对比
x = np.arange(len(scenario_names))
width = 0.25

plt.bar(x - width, roc_aucs, width, label='ROC-AUC')
plt.bar(x, pr_aucs, width, label='PR-AUC')
plt.bar(x + width, f_betas, width, label='F-beta分数')

plt.xlabel('安全场景')
plt.ylabel('分数')
plt.title('不同安全场景下的指标对比')
plt.xticks(x, scenario_names)
plt.legend()
plt.grid(True, axis='y')

plt.tight_layout()
plt.savefig('scenario_applicability.png')
print("\n不同场景下的指标对比可视化完成，保存为scenario_applicability.png")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score

# 生成极端不平衡的安全数据集
X, y = make_classification(
    n_samples=10000,
    n_classes=2,
    weights=[0.99, 0.01],
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

print(f"训练集样本数: {len(X_train)}, 攻击样本数: {np.sum(y_train)}")
print(f"测试集样本数: {len(X_test)}, 攻击样本数: {np.sum(y_test)}")
print(f"测试集攻击样本比例: {np.sum(y_test)/len(y_test):.2%}")

# 1. 基准模型1：随机猜测
dummy_random = DummyClassifier(strategy="stratified", random_state=42)
dummy_random.fit(X_train, y_train)
dummy_random_proba = dummy_random.predict_proba(X_test)[:, 1]

# 2. 基准模型2：总是预测为多数类
dummy_majority = DummyClassifier(strategy="most_frequent", random_state=42)
dummy_majority.fit(X_train, y_train)
dummy_majority_proba = dummy_majority.predict_proba(X_test)[:, 1]

# 3. 智能模型：随机森林
rf_model = RandomForestClassifier(
    n_estimators=100,
    max_depth=10,
    class_weight="balanced",
    random_state=42
)
rf_model.fit(X_train, y_train)
rf_proba = rf_model.predict_proba(X_test)[:, 1]

# 计算所有模型的ROC曲线和AUC
def calculate_roc(y_true, y_pred_proba, model_name):
    fpr, tpr, thresholds = roc_curve(y_true, y_pred_proba)
    roc_auc = auc(fpr, tpr)
    return {
        "fpr": fpr,
        "tpr": tpr,
        "auc": roc_auc,
        "name": model_name
    }

# 计算所有模型的PR曲线和PR-AUC
def calculate_pr(y_true, y_pred_proba, model_name):
    precision, recall, thresholds = precision_recall_curve(y_true, y_pred_proba)
    pr_auc = average_precision_score(y_true, y_pred_proba)
    return {
        "precision": precision,
        "recall": recall,
        "auc": pr_auc,
        "name": model_name
    }

# 计算所有模型的指标
models = {
    "随机猜测": dummy_random_proba,
    "总是预测为正常": dummy_majority_proba,
    "随机森林": rf_proba
}

roc_results = []
pr_results = []

for name, proba in models.items():
    roc_results.append(calculate_roc(y_test, proba, name))
    pr_results.append(calculate_pr(y_test, proba, name))

# 可视化ROC曲线和PR曲线
plt.figure(figsize=(12, 6))

# ROC曲线
plt.subplot(1, 2, 1)
for result in roc_results:
    plt.plot(result["fpr"], result["tpr"], label=f'{result["name"]} (AUC = {result["auc"]:.4f})')
plt.plot([0, 1], [0, 1], 'k--', label='随机猜测基准')
plt.xlabel('假阳性率 (FPR)')
plt.ylabel('真阳性率 (TPR)')
plt.title('ROC曲线对比')
plt.legend()
plt.grid(True)

# PR曲线
plt.subplot(1, 2, 2)
for result in pr_results:
    plt.plot(result["recall"], result["precision"], label=f'{result["name"]} (PR-AUC = {result["auc"]:.4f})')
plt.xlabel('召回率 (Recall)')
plt.ylabel('精确率 (Precision)')
plt.title('PR曲线对比')
plt.legend()
plt.grid(True)

plt.tight_layout()
plt.savefig('roc_pr_comparison.png')
print("\nROC曲线和PR曲线对比可视化完成，保存为roc_pr_comparison.png")

# 分析ROC-AUC的局限性
print("\n=== ROC-AUC局限性分析 ===")
print("模型性能对比:")
for result in roc_results:
    print(f"- {result['name']} ROC-AUC: {result['auc']:.4f}")

print("\nPR-AUC对比:")
for result in pr_results:
    print(f"- {result['name']} PR-AUC: {result['auc']:.4f}")

print("\n结论:")
print("1. ROC-AUC可能高估模型在不平衡数据上的性能，即使是随机猜测模型也能获得高于0.5的ROC-AUC")
print("2. PR-AUC更能真实反映模型在少数类上的性能，随机猜测模型的PR-AUC接近正样本比例")
print("3. 在极端不平衡数据上，PR-AUC比ROC-AUC更能区分不同模型的性能")

import numpy as np
from sklearn.metrics import (
    roc_curve, auc, precision_recall_curve, average_precision_score,
    fbeta_score, confusion_matrix
)

def compute_roc_auc(y_true, y_pred_proba):
    """计算ROC曲线和AUC"""
    fpr, tpr, thresholds = roc_curve(y_true, y_pred_proba)
    roc_auc = auc(fpr, tpr)
    return {
        "fpr": fpr,
        "tpr": tpr,
        "thresholds": thresholds,
        "auc": roc_auc
    }

def compute_pr_auc(y_true, y_pred_proba):
    """计算PR曲线和PR-AUC"""
    precision, recall, thresholds = precision_recall_curve(y_true, y_pred_proba)
    pr_auc = average_precision_score(y_true, y_pred_proba)
    return {
        "precision": precision,
        "recall": recall,
        "thresholds": thresholds,
        "auc": pr_auc
    }

def evaluate_model(y_true, y_pred_proba, beta=1.0, threshold=0.5):
    """全面评估模型性能"""
    # 计算ROC-AUC
    roc_results = compute_roc_auc(y_true, y_pred_proba)
    
    # 计算PR-AUC
    pr_results = compute_pr_auc(y_true, y_pred_proba)
    
    # 基于阈值的指标
    y_pred = y_pred_proba >= threshold
    tp, fp, fn, tn = confusion_matrix(y_true, y_pred).ravel()
    
    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
    f1 = fbeta_score(y_true, y_pred, beta=beta)
    
    return {
        "roc_auc": roc_results["auc"],
        "pr_auc": pr_results["auc"],
        "precision": precision,
        "recall": recall,
        "f1_score": f1,
        "confusion_matrix": {
            "tp": tp,
            "fp": fp,
            "fn": fn,
            "tn": tn
        },
        "threshold_used": threshold
    }

def compare_models(models, y_true, X_test, beta=1.0):
    """比较多个模型的性能"""
    results = []
    
    for model_name, model in models.items():
        y_pred_proba = model.predict_proba(X_test)[:, 1]
        metrics = evaluate_model(y_true, y_pred_proba, beta=beta)
        metrics["model_name"] = model_name
        results.append(metrics)
    
    return results