优化方法

优化器是深度学习训练的引擎——给定损失函数的梯度，优化器决定如何更新参数来最小化损失。SGD 是最朴素的 w←w-η·g，但在鞍点和狭长山谷中收敛极慢；Momentum 引入动量 v=βv+g，像小球滚下山坡一样积累速度，加速收敛并平滑震荡；AdaGrad 为每个参数维护历史梯度平方和，让高频更新的参数自动减速、低频参数加速——适合 NLP 中稀疏特征（如罕见词的 embedding）；Adam 结合了 Momentum 的一阶矩估计和 RMSProp 的二阶矩估计，是 GPT、BERT、Stable Diffusion 等模型训练的默认选择。学习率调度同样关键：Warmup 在前几百步线性增大学习率避免初期参数剧变，Cosine Decay 使后期学习率平滑衰减到接近零以精细调优——这是 LLM 训练的标准策略。本节逐个解释每种优化器的数学原理，并用 f(w)=w² 对比它们的实际收敛行为。

一、优化问题的本质

二、批量梯度下降 vs 随机梯度下降 vs 小批量

三、梯度下降（SGD）

四、SGD + 动量（Momentum）

五、AdaGrad——自适应学习率

六、RMSProp——修复 AdaGrad

七、Adam——工业标配优化器

八、学习率调度

九、优化器选择指南

十、代码验证（C# / Rust）

C#（.NET 10）

// dotnet run 即可执行
double F(double w) => w * w;
double GradF(double w) => 2.0 * w;

// ===== SGD / Momentum / Adam 对比 =====
void Train(string name, double lr, double wInit = 5.0, int steps = 20)
{
    double w = wInit, m = 0, v = 0, vel = 0;
    double beta1 = 0.9, beta2 = 0.999, eps = 1e-8;
    for (int t = 1; t <= steps; t++)
    {
        double g = GradF(w);
        switch (name)
        {
            case "SGD":      w -= lr * g; break;
            case "Momentum": vel = 0.9 * vel + g; w -= lr * vel; break;
            case "Adam":
                m = beta1 * m + (1 - beta1) * g;
                v = beta2 * v + (1 - beta2) * g * g;
                double mH = m / (1 - Math.Pow(beta1, t));
                double vH = v / (1 - Math.Pow(beta2, t));
                w -= lr * mH / (Math.Sqrt(vH) + eps);
                break;
        }
    }
    Console.WriteLine($"{name,-10}: 最终 w={w:F6}, f(w)={F(w):F8}");
}

Train("SGD", 0.1);
Train("Momentum", 0.01);
Train("Adam", 0.5);

// ===== AdaGrad 演示 =====
Console.WriteLine("\nAdaGrad (高频 vs 低频参数):");
double w1 = 5.0, w2 = 5.0, s1 = 0.0, s2 = 0.0;
double lrAdagrad = 0.5;
double[] gW1 = { 2.0, 1.5, 1.8, 2.1, 1.3 };
double[] gW2 = { 0.1, 0.0, 0.2, 0.0, 0.3 };
for (int t = 0; t < 5; t++)
{
    s1 += gW1[t] * gW1[t]; s2 += gW2[t] * gW2[t];
    double eff1 = lrAdagrad / (Math.Sqrt(s1) + 1e-8);
    double eff2 = lrAdagrad / (Math.Sqrt(s2) + 1e-8);
    w1 -= eff1 * gW1[t]; w2 -= eff2 * gW2[t];
    Console.WriteLine($"  t={t + 1}: 高频lr={eff1:F4}, 低频lr={eff2:F4}");
}

// ===== Warmup + Cosine Decay =====
Console.WriteLine("\nWarmup + Cosine Decay 调度:");
int totalSteps = 100, warmup = 10;
double lrMax = 0.001;
foreach (int step in new[] { 1, 5, 10, 25, 50, 75, 100 })
{
    double lrS = step <= warmup
        ? lrMax * step / warmup
        : lrMax * 0.5 * (1 + Math.Cos(Math.PI * (step - warmup) / (double)(totalSteps - warmup)));
    Console.WriteLine($"  Step {step,3}: lr = {lrS:F6}");
}

Rust

fn main() {
    let f = |w: f64| w * w;
    let grad_f = |w: f64| 2.0 * w;

    // ===== SGD / Momentum / Adam 对比 =====
    for &(name, lr) in &[("SGD", 0.1), ("Momentum", 0.01), ("Adam", 0.5)] {
        let (mut w, mut m, mut v, mut vel) = (5.0_f64, 0.0, 0.0, 0.0);
        let (beta1, beta2, eps) = (0.9_f64, 0.999, 1e-8);
        for t in 1..=20_i32 {
            let g = grad_f(w);
            match name {
                "SGD"      => w -= lr * g,
                "Momentum" => { vel = 0.9_f64.mul_add(vel, g); w -= lr * vel; }
                "Adam"     => {
                    m = beta1.mul_add(m, (1.0 - beta1) * g);
                    v = beta2.mul_add(v, (1.0 - beta2) * g * g);
                    let m_hat = m / (1.0 - beta1.powi(t));
                    let v_hat = v / (1.0 - beta2.powi(t));
                    w -= lr * m_hat / (v_hat.sqrt() + eps);
                }
                _ => {}
            }
        }
        println!("{name:<10}: 最终 w={w:.6}, f(w)={:.8}", f(w));
    }

    // ===== AdaGrad 演示 =====
    println!("\nAdaGrad (高频 vs 低频参数):");
    let (mut w1, mut w2, mut s1, mut s2) = (5.0_f64, 5.0, 0.0, 0.0);
    let lr = 0.5_f64;
    let gw1 = [2.0, 1.5, 1.8, 2.1, 1.3];
    let gw2 = [0.1, 0.0, 0.2, 0.0, 0.3];
    for t in 0..5 {
        s1 += gw1[t] * gw1[t]; s2 += gw2[t] * gw2[t];
        let eff1 = lr / (s1.sqrt() + 1e-8);
        let eff2 = lr / (s2.sqrt() + 1e-8);
        w1 -= eff1 * gw1[t]; w2 -= eff2 * gw2[t];
        println!("  t={}: 高频lr={eff1:.4}, 低频lr={eff2:.4}", t + 1);
    }

    // ===== Warmup + Cosine Decay =====
    println!("\nWarmup + Cosine Decay 调度:");
    let (total, warmup, lr_max) = (100_i32, 10_i32, 0.001_f64);
    for step in [1, 5, 10, 25, 50, 75, 100] {
        let lr_s = if step <= warmup {
            lr_max * step as f64 / warmup as f64
        } else {
            let prog = (step - warmup) as f64 / (total - warmup) as f64;
            lr_max * (std::f64::consts::PI * prog).cos().mul_add(0.5, 0.5)
        };
        println!("  Step {step:3}: lr = {lr_s:.6}");
    }
}