函数与导数

导数描述了函数在某一点变化的速率，是梯度下降的数学源头。深度学习的训练循环就是：前向传播算 loss → 反向传播算每个参数的导数 → 沿导数反方向更新参数。Sigmoid σ'(x)=σ(x)(1-σ(x)) 用于二分类输出层与 LSTM 门控；ReLU 分段导数解决了 Sigmoid 的梯度消失问题，成为 CNN/MLP 默认激活函数；Softmax + 交叉熵的复合导数简化为 ŷ-y，是 GPT、BERT 等分类头的核心公式；Tanh 导数用于 RNN/LSTM 内部状态更新。数值梯度检查 (f(x+ε)-f(x-ε))/2ε 是调试自定义层的标准手段。本节从高中导数公式出发，逐一讲解 AI 中高频出现的每个激活函数及损失函数的导数推导。

一、导数的精确定义

二、基本求导公式大全

三、AI 中最重要的函数：激活函数

3.1 Sigmoid 函数

逐步数值计算：

3.2 ReLU 函数

3.3 ReLU 的改进变体

3.4 Softmax 函数

四、损失函数的导数

4.1 均方误差（MSE）

4.2 交叉熵（Cross Entropy）

五、导数的数值验证方法

六、代码验证（C# / Rust）

C#（.NET 10）

// dotnet run 即可执行
double Sigmoid(double x) => 1.0 / (1.0 + Math.Exp(-x));
double SigmoidDeriv(double x) { double s = Sigmoid(x); return s * (1 - s); }

// ===== Sigmoid 及其导数 =====
double[] xs = { -5, -2, -1, 0, 1, 2, 5 };
Console.Write("x:        ");
foreach (var v in xs) Console.Write($"{v,8}");
Console.Write("\nσ(x):     ");
foreach (var v in xs) Console.Write($"{Sigmoid(v),8:F4}");
Console.Write("\nσ'(x):    ");
foreach (var v in xs) Console.Write($"{SigmoidDeriv(v),8:F4}");
Console.WriteLine();

// ===== ReLU =====
double[] z = { -1.2, 0.8, -0.3, 2.1, 0.0, -0.5, 1.5, 0.3 };
Console.Write("ReLU:     ");
foreach (var v in z) Console.Write($"{Math.Max(0, v),6:F1}");
int zeros = z.Count(v => Math.Max(0, v) == 0);
Console.WriteLine($"\n稀疏率:    {(double)zeros / z.Length:F2}");

// ===== Softmax =====
double[] logits = { 2.1, -0.3, 0.8, 5.2, -1.0, 0.5, -0.2, 1.1, 0.3, -0.5 };
double logMax = logits.Max();
double[] exps = logits.Select(v => Math.Exp(v - logMax)).ToArray();
double sumExp = exps.Sum();
double[] probs = exps.Select(e => e / sumExp).ToArray();

Console.Write("Softmax概率: ");
foreach (var p in probs) Console.Write($"{p:F4} ");
Console.WriteLine($"\n概率总和:  {probs.Sum():F4}");
Console.WriteLine($"预测类别:  {Array.IndexOf(probs, probs.Max())}");

// ===== 交叉熵 + Softmax 梯度 =====
double[] yTrue = { 1, 0, 0 };
double[] yPred = { 0.7, 0.2, 0.1 };
double loss = 0;
for (int i = 0; i < 3; i++) loss -= yTrue[i] * Math.Log(yPred[i]);
Console.Write($"交叉熵损失: {loss:F4}\n梯度: ");
for (int i = 0; i < 3; i++) Console.Write($"{yPred[i] - yTrue[i]:F1} ");
Console.WriteLine();

// ===== 数值梯度检查 =====
double F(double x) => x * x * x;
double FGrad(double x) => 3 * x * x;
double x0 = 2.0, eps = 1e-7;
double numGrad = (F(x0 + eps) - F(x0 - eps)) / (2 * eps);
double anaGrad = FGrad(x0);
double relErr = Math.Abs(numGrad - anaGrad) / Math.Max(Math.Abs(numGrad), Math.Abs(anaGrad));
Console.WriteLine($"解析梯度: {anaGrad}, 数值梯度: {numGrad:F10}");
Console.WriteLine($"相对误差: {relErr:E2}");

Rust

fn sigmoid(x: f64) -> f64 { 1.0 / (1.0 + (-x).exp()) }
fn sigmoid_deriv(x: f64) -> f64 { let s = sigmoid(x); s * (1.0 - s) }

fn main() {
    // ===== Sigmoid 及其导数 =====
    let xs = [-5.0, -2.0, -1.0, 0.0, 1.0, 2.0, 5.0_f64];
    print!("x:        ");
    for i in 0..xs.len() { print!("{:8.0}", xs[i]); }
    print!("\nσ(x):     ");
    for i in 0..xs.len() { print!("{:8.4}", sigmoid(xs[i])); }
    print!("\nσ'(x):    ");
    for i in 0..xs.len() { print!("{:8.4}", sigmoid_deriv(xs[i])); }
    println!();

    // ===== ReLU =====
    let z = [-1.2, 0.8, -0.3, 2.1, 0.0, -0.5, 1.5, 0.3_f64];
    print!("ReLU:     ");
    for i in 0..z.len() { print!("{:6.1}", z[i].max(0.0)); }
    let zeros = z.iter().filter(|v| **v <= 0.0).count();
    println!("\n稀疏率:    {:.2}", zeros as f64 / z.len() as f64);

    // ===== Softmax =====
    let logits = [2.1, -0.3, 0.8, 5.2, -1.0, 0.5, -0.2, 1.1, 0.3, -0.5_f64];
    let max_l = logits.iter().copied().reduce(f64::max).unwrap();
    let exps: [f64; 10] = core::array::from_fn(|i| (logits[i] - max_l).exp());
    let sum_e: f64 = exps.iter().sum();
    let probs: [f64; 10] = core::array::from_fn(|i| exps[i] / sum_e);

    print!("Softmax概率: ");
    for i in 0..10 { print!("{:.4} ", probs[i]); }
    let sum_p: f64 = probs.iter().sum();
    println!("\n概率总和:  {sum_p:.4}");
    let argmax = probs.iter().enumerate()
        .max_by(|a, b| a.1.total_cmp(b.1)).unwrap().0;
    println!("预测类别:  {argmax}");

    // ===== 交叉熵 + Softmax 梯度 =====
    let y_true = [1.0, 0.0, 0.0_f64];
    let y_pred = [0.7, 0.2, 0.1_f64];
    let mut loss = 0.0_f64;
    for i in 0..3 { loss -= y_true[i] * y_pred[i].ln(); }
    print!("交叉熵损失: {loss:.4}\n梯度: ");
    for i in 0..3 { print!("{:.1} ", y_pred[i] - y_true[i]); }
    println!();

    // ===== 数值梯度检查 =====
    let f = |x: f64| x.powi(3);
    let f_grad = |x: f64| 3.0 * x.powi(2);
    let (x0, eps) = (2.0_f64, 1e-7);
    let num_g = (f(x0 + eps) - f(x0 - eps)) / (2.0 * eps);
    let ana_g = f_grad(x0);
    let rel = (num_g - ana_g).abs() / num_g.abs().max(ana_g.abs());
    println!("解析梯度: {ana_g}, 数值梯度: {num_g:.10}");
    println!("相对误差: {rel:.2e}");
}