期望、方差与协方差

期望和方差是描述分布最基本的两个统计量，贯穿深度学习的各个环节。BatchNorm 在每一层对 mini-batch 计算均值 μ 和方差 σ²，将激活值标准化为零均值单位方差，加速收敛并稳定训练——ResNet、EfficientNet 等主流 CNN 几乎每个卷积后都接 BatchNorm。Layer Norm 沿特征维度做同样的均值/方差标准化，是 Transformer/GPT/BERT 的默认选择。损失函数的期望 E[L(θ)] 就是经验风险，整个训练过程就是最小化它。协方差矩阵刻画特征之间的相关性：PCA 对协方差矩阵做特征值分解实现降维；金融风控中协方差矩阵评估组合风险；多维高斯分布完全由均值向量和协方差矩阵决定。方差还出现在 Xavier/He 初始化公式中——权重初始值的方差必须匹配层宽度，否则梯度会消失或爆炸。本节用真实数据从概念到计算进行完整推导。

一、期望（均值）E[X]

二、方差 Var(X) 与标准差 σ

三、协方差 Cov(X, Y)

四、协方差矩阵

五、期望和方差在损失函数中的角色

六、代码验证（C# / Rust）

C#（.NET 10）

// dotnet run 即可执行
// ===== 学生成绩方差 =====
double[] scores = { 72, 85, 90, 68, 95 };
double mean = scores.Average();
double varPop = scores.Select(s => (s-mean)*(s-mean)).Average();
double varSample = scores.Select(s => (s-mean)*(s-mean)).Sum() / (scores.Length-1);
Console.WriteLine($"均值={mean}, 总体方差={varPop}, 样本方差={varSample}, 标准差={Math.Sqrt(varPop):F2}");

// ===== 协方差 =====
double[] X = { 5, 8, 10, 12, 15 };
double[] Y = { 60, 72, 80, 85, 93 };
double mx = X.Average(), my = Y.Average();
double covXY = 0, sx2 = 0, sy2 = 0;
for (int i = 0; i < 5; i++) {
    covXY += (X[i]-mx)*(Y[i]-my);
    sx2 += (X[i]-mx)*(X[i]-mx); sy2 += (Y[i]-my)*(Y[i]-my);
}
covXY /= 5;
double rho = covXY / (Math.Sqrt(sx2/5) * Math.Sqrt(sy2/5));
Console.WriteLine($"\nCov(X,Y) = {covXY:F2}");
Console.WriteLine($"ρ = {rho:F4}");

// 总体协方差矩阵
Console.WriteLine($"\n协方差矩阵(ddof=0):");
Console.WriteLine($"  [{sx2/5,8:F2} {covXY,8:F2}]");
Console.WriteLine($"  [{covXY,8:F2} {sy2/5,8:F2}]");

// ===== 5名学生、3项指标的协方差矩阵 =====
double[,] data3 = {{85,90,88},{70,75,72},{92,88,95},{65,70,68},{78,82,80}};
int n = 5, p = 3;
double[] mu3 = new double[p];
for (int j = 0; j < p; j++) { for (int i = 0; i < n; i++) mu3[j] += data3[i,j]; mu3[j] /= n; }

Console.WriteLine("\n3×3 样本协方差矩阵 (替代 Iris):");
for (int j1 = 0; j1 < p; j1++) {
    for (int j2 = 0; j2 < p; j2++) {
        double c = 0;
        for (int i = 0; i < n; i++) c += (data3[i,j1]-mu3[j1])*(data3[i,j2]-mu3[j2]);
        Console.Write($"{c/(n-1),8:F2}");
    }
    Console.WriteLine();
}

// ===== BatchNorm =====
double[] act = { 2.0, 4.0, 6.0, 8.0 };
double muBn = act.Average();
double vBn = act.Select(v => (v-muBn)*(v-muBn)).Average();
double gamma = 2.0, beta = 1.0;
Console.Write($"\nBatchNorm: μ={muBn}, σ²={vBn}\n标准化: ");
foreach (var v in act) Console.Write($"{(v-muBn)/Math.Sqrt(vBn+1e-5):F3} ");
Console.Write($"\n缩放平移(γ=2,β=1): ");
foreach (var v in act) Console.Write($"{gamma*((v-muBn)/Math.Sqrt(vBn+1e-5))+beta:F3} ");
Console.WriteLine();

// ===== MSE =====
double[] yT = { 300, 450, 200, 550 };
double[] yP = { 320, 430, 210, 500 };
double mse = 0;
for (int i = 0; i < 4; i++) mse += Math.Pow(yT[i]-yP[i], 2);
mse /= 4;
Console.WriteLine($"\nMSE = {mse}, RMSE = {Math.Sqrt(mse):F2}");

Rust

fn main() {
    // ===== 学生成绩方差 =====
    let scores = [72.0, 85.0, 90.0, 68.0, 95.0_f64];
    let n = 5.0_f64;
    let mut mean = 0.0; for i in 0..5 { mean += scores[i]; } mean /= n;
    let mut var_pop = 0.0; for i in 0..5 { var_pop += (scores[i]-mean).powi(2); } var_pop /= n;
    let var_sample = var_pop * n / (n - 1.0);
    println!("均值={mean}, 总体方差={var_pop}, 样本方差={var_sample}, 标准差={:.2}", var_pop.sqrt());

    // ===== 协方差 =====
    let x = [5.0, 8.0, 10.0, 12.0, 15.0_f64];
    let y = [60.0, 72.0, 80.0, 85.0, 93.0_f64];
    let mut mx = 0.0; for i in 0..5 { mx += x[i]; } mx /= 5.0;
    let mut my = 0.0; for i in 0..5 { my += y[i]; } my /= 5.0;
    let (mut cov_xy, mut sx2, mut sy2) = (0.0, 0.0, 0.0_f64);
    for i in 0..5 {
        cov_xy += (x[i]-mx)*(y[i]-my);
        sx2 += (x[i]-mx).powi(2); sy2 += (y[i]-my).powi(2);
    }
    cov_xy /= 5.0;
    let rho = cov_xy / ((sx2/5.0).sqrt() * (sy2/5.0).sqrt());
    println!("\nCov(X,Y) = {cov_xy:.2}");
    println!("ρ = {rho:.4}");
    println!("\n协方差矩阵(ddof=0):");
    println!("  [{:8.2} {:8.2}]", sx2/5.0, cov_xy);
    println!("  [{:8.2} {:8.2}]", cov_xy, sy2/5.0);

    // ===== 3×3 协方差矩阵 =====
    let data3 = [[85.0,90.0,88.0],[70.0,75.0,72.0],[92.0,88.0,95.0],
                  [65.0,70.0,68.0],[78.0,82.0,80.0_f64]];
    let mut mu3 = [0.0_f64; 3];
    for j in 0..3 { for i in 0..5 { mu3[j] += data3[i][j]; } mu3[j] /= 5.0; }
    println!("\n3×3 样本协方差矩阵:");
    for j1 in 0..3 {
        for j2 in 0..3 {
            let mut c = 0.0;
            for i in 0..5 { c += (data3[i][j1]-mu3[j1])*(data3[i][j2]-mu3[j2]); }
            print!("{:8.2}", c / 4.0);
        }
        println!();
    }

    // ===== BatchNorm =====
    let act = [2.0, 4.0, 6.0, 8.0_f64];
    let mut mu_bn = 0.0; for i in 0..4 { mu_bn += act[i]; } mu_bn /= 4.0;
    let mut v_bn = 0.0; for i in 0..4 { v_bn += (act[i]-mu_bn).powi(2); } v_bn /= 4.0;
    let (gamma, beta) = (2.0_f64, 1.0);
    print!("\nBatchNorm: μ={mu_bn}, σ²={v_bn}\n标准化: ");
    for i in 0..4 { print!("{:.3} ", (act[i]-mu_bn)/(v_bn+1e-5).sqrt()); }
    print!("\n缩放平移(γ=2,β=1): ");
    for i in 0..4 { print!("{:.3} ", gamma.mul_add((act[i]-mu_bn)/(v_bn+1e-5).sqrt(), beta)); }
    println!();

    // ===== MSE =====
    let yt = [300.0, 450.0, 200.0, 550.0_f64];
    let yp = [320.0, 430.0, 210.0, 500.0_f64];
    let mut mse = 0.0; for i in 0..4 { mse += (yt[i]-yp[i]).powi(2); } mse /= 4.0;
    println!("\nMSE = {mse}, RMSE = {:.2}", mse.sqrt());
}