过拟合与正则化

过拟合是深度学习中最常见也最关键的问题——模型在训练集上表现极好但在测试集上性能急剧下降，本质上是模型“记住了噪声而非规律”。应对手段形成了一整套正则化工具箱：L2 正则化（Weight Decay）在损失函数中加入 λ‖w‖² 使权重趋近于零，等价于对参数施加高斯先验，是 Adam 优化器的标配（AdamW 专门解耦了 weight decay）；L1 正则化产生稀疏权重，可用于特征选择；Dropout 随机关闭一部分神经元（p=0.1~0.5），强迫网络学习冗余表示——BERT、GPT 中广泛使用；Early Stopping 监控验证集 loss，在开始上升时停止训练，是最简单有效的防过拟合策略；数据增强（Mixup、CutMix、RandAugment）通过合成新样本扩大训练集；Label Smoothing 将 hard label 软化为 (1-ε)·one_hot + ε/K，防止模型对输出过度自信——GPT、ViT 等模型均使用 ε=0.1。本节逐一讲解每种正则化技术的数学原理，并用代码验证其效果。

一、偏差-方差权衡

二、过拟合的判断方法

三、数据集划分

四、L2 正则化（权重衰减）

五、L1 正则化（稀疏化）

六、Dropout

七、Early Stopping（早停）

八、数据增强

九、Batch Normalization 的正则化效果

十、正则化强度的选择

十一、正则化方法总结

十二、代码验证（C# / Rust）

C#（.NET 10）

// dotnet run 即可执行
// ===== L2 / L1 正则化效果 =====
double[] w = { 2.5, -1.8, 0.3, 3.1, -0.1 };
double[] grad = { 0.5, -0.3, 0.1, 0.8, -0.05 };
double eta = 0.01, lam = 0.1;

Console.Write("无正则化: ");
for (int i = 0; i < 5; i++) Console.Write($"{w[i] - eta * grad[i],8:F4}");
Console.WriteLine();

Console.Write("L2 正则化:");
for (int i = 0; i < 5; i++) Console.Write($"{w[i] - eta * (grad[i] + lam * w[i]),8:F4}");
Console.WriteLine();

Console.Write("L1 正则化:");
for (int i = 0; i < 5; i++) Console.Write($"{w[i] - eta * (grad[i] + lam * Math.Sign(w[i])),8:F4}");
Console.WriteLine();

// ===== Dropout 前向传播 =====
double[] h = { 0.8, -0.5, 1.2, 0.3, -0.9, 0.6 };
double pDrop = 0.5;
int[] mask = { 0, 1, 1, 1, 0, 0 }; // 模拟随机掩码 (seed=42)
Console.WriteLine($"\nmask = [{string.Join(", ", mask)}]");
Console.Write("Dropout 输出 = [");
for (int i = 0; i < 6; i++)
{
    if (i > 0) Console.Write(", ");
    Console.Write($"{h[i] * mask[i] / (1 - pDrop):F2}");
}
Console.WriteLine($"]\n期望输出均值 = {h.Average():F3}");

// ===== Early Stopping 模拟 =====
int patience = 5, wait = 0, bestEpoch = 0;
double bestVal = double.MaxValue;
for (int epoch = 0; epoch < 50; epoch++)
{
    double val = 2.5 * Math.Exp(-0.06 * epoch) + 0.1 + 0.005 * Math.Max(0, epoch - 15);
    if (val < bestVal) { bestVal = val; bestEpoch = epoch; wait = 0; }
    else if (++wait >= patience)
    {
        Console.WriteLine($"\nEarly Stopping at epoch {epoch}");
        Console.WriteLine($"Best epoch: {bestEpoch}, Best val loss: {bestVal:F4}");
        break;
    }
}

// ===== Mixup 数据增强 =====
double[] imgA = { 0.8, 0.5, 0.3 }, imgB = { 0.2, 0.7, 0.6 };
double[] labelA = { 1, 0 }, labelB = { 0, 1 };
double lamMix = 0.7;
Console.Write("\nMixup 图片: [");
for (int i = 0; i < 3; i++)
{
    if (i > 0) Console.Write(", ");
    Console.Write($"{lamMix * imgA[i] + (1 - lamMix) * imgB[i]:F2}");
}
Console.Write("]\nMixup 标签: [");
for (int i = 0; i < 2; i++)
{
    if (i > 0) Console.Write(", ");
    Console.Write($"{lamMix * labelA[i] + (1 - lamMix) * labelB[i]:F1}");
}
Console.WriteLine("]");

// ===== Label Smoothing =====
int K = 10; double eps = 0.1;
Console.Write("\n硬标签: [");
for (int i = 0; i < K; i++) { if (i > 0) Console.Write(", "); Console.Write(i == 3 ? "1" : "0"); }
Console.Write("]\n软标签: [");
for (int i = 0; i < K; i++)
{
    if (i > 0) Console.Write(", ");
    double hard = i == 3 ? 1.0 : 0.0;
    Console.Write($"{(1 - eps) * hard + eps / K:F2}");
}
Console.WriteLine("]");

Rust

fn main() {
    // ===== L2 / L1 正则化效果 =====
    let w = [2.5_f64, -1.8, 0.3, 3.1, -0.1];
    let grad = [0.5_f64, -0.3, 0.1, 0.8, -0.05];
    let (eta, lam) = (0.01_f64, 0.1);

    print!("无正则化: ");
    for i in 0..5 { print!("{:8.4}", w[i] - eta * grad[i]); }
    println!();
    print!("L2 正则化:");
    for i in 0..5 { print!("{:8.4}", w[i] - eta * lam.mul_add(w[i], grad[i])); }
    println!();
    print!("L1 正则化:");
    for i in 0..5 { print!("{:8.4}", w[i] - eta * lam.mul_add(w[i].signum(), grad[i])); }
    println!();

    // ===== Dropout 前向传播 =====
    let h = [0.8_f64, -0.5, 1.2, 0.3, -0.9, 0.6];
    let p_drop = 0.5_f64;
    let mask = [0.0_f64, 1.0, 1.0, 1.0, 0.0, 0.0]; // 模拟随机掩码
    println!("\nmask = {:?}", mask.map(|v| v as i32));
    print!("Dropout 输出 = [");
    for i in 0..6 {
        if i > 0 { print!(", "); }
        print!("{:.2}", h[i] * mask[i] / (1.0 - p_drop));
    }
    let h_mean: f64 = h.iter().sum::<f64>() / h.len() as f64;
    println!("]\n期望输出均值 = {h_mean:.3}");

    // ===== Early Stopping 模拟 =====
    let (patience, mut wait, mut best_epoch) = (5_i32, 0_i32, 0_i32);
    let mut best_val = f64::MAX;
    for epoch in 0..50 {
        let val = 2.5 * (-0.06 * epoch as f64).exp()
                  + 0.1 + 0.005 * (epoch - 15).max(0) as f64;
        if val < best_val { best_val = val; best_epoch = epoch; wait = 0; }
        else { wait += 1; if wait >= patience {
            println!("\nEarly Stopping at epoch {epoch} (patience={patience})");
            println!("Best epoch: {best_epoch}, Best val loss: {best_val:.4}");
            break;
        }}
    }

    // ===== Mixup 数据增强 =====
    let img_a = [0.8, 0.5, 0.3_f64];
    let img_b = [0.2, 0.7, 0.6_f64];
    let label_a = [1.0, 0.0_f64];
    let label_b = [0.0, 1.0_f64];
    let lam_mix = 0.7_f64;
    print!("\nMixup 图片: [");
    for i in 0..3 {
        if i > 0 { print!(", "); }
        print!("{:.2}", lam_mix.mul_add(img_a[i], (1.0 - lam_mix) * img_b[i]));
    }
    print!("]\nMixup 标签: [");
    for i in 0..2 {
        if i > 0 { print!(", "); }
        print!("{:.1}", lam_mix.mul_add(label_a[i], (1.0 - lam_mix) * label_b[i]));
    }
    println!("]");

    // ===== Label Smoothing =====
    let (k, eps) = (10, 0.1_f64);
    print!("\n硬标签: [");
    for i in 0..k { if i > 0 { print!(", "); } print!("{}", if i == 3 { 1 } else { 0 }); }
    print!("]\n软标签: [");
    for i in 0..k {
        if i > 0 { print!(", "); }
        let hard = if i == 3 { 1.0 } else { 0.0 };
        print!("{:.2}", (1.0 - eps).mul_add(hard, eps / k as f64));
    }
    println!("]");
}