GBDT算法之推导过程

import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_regression

# 1. GBDT推导验证
print("=== GBDT推导验证 ===")
np.random.seed(42)

# 生成数据
X, y = make_regression(
    n_samples=100,
    n_features=5,
    noise=0.1,
    random_state=42
)

# 初始化预测值（真实值的均值）
f0 = np.mean(y)
print(f"初始化预测值 f0 = {f0:.4f}")
print(f"真实值均值 = {np.mean(y):.4f}")

# 计算第一轮残差
residuals = y - f0
print(f"\n第一轮残差（前5个）: {residuals[:5]}")

# 2. 手动实现GBDT
print("\n=== 手动实现GBDT ===")
class SimpleGBDT:
    """简单的GBDT实现（回归）"""
    
    def __init__(self, n_estimators=10, learning_rate=0.1, max_depth=3):
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.max_depth = max_depth
        self.trees = []
        self.f0 = 0
    
    def fit(self, X, y):
        # 初始化：f0 = 均值
        self.f0 = np.mean(y)
        print(f"初始化 f0 = {self.f0:.4f}")
        
        # 当前预测值
        current_pred = np.full(len(y), self.f0)
        
        for i in range(self.n_estimators):
            # 计算残差（负梯度）
            residuals = y - current_pred
            
            # 训练回归树
            tree = DecisionTreeRegressor(
                max_depth=self.max_depth,
                random_state=i
            )
            tree.fit(X, residuals)
            
            # 更新预测值
            current_pred += self.learning_rate * tree.predict(X)
            self.trees.append(tree)
            
            # 计算损失
            loss = 0.5 * np.sum((y - current_pred) ** 2)
            
            if (i + 1) % 5 == 0:
                print(f"第{i+1}轮: 损失={loss:.4f}")
    
    def predict(self, X):
        # 初始化预测
        pred = np.full(len(X), self.f0)
        
        # 累加每棵树的预测
        for tree in self.trees:
            pred += self.learning_rate * tree.predict(X)
        
        return pred

# 生成数据
X, y = make_regression(
    n_samples=500,
    n_features=10,
    noise=0.1,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 训练
gbdt = SimpleGBDT(n_estimators=20, learning_rate=0.1, max_depth=3)
gbdt.fit(X_train, y_train)

# 预测
y_pred = gbdt.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print(f"\n测试集MSE: {mse:.4f}")

# 3. 对比不同初始化
print("\n=== 对比不同初始化 ===")
# 使用均值初始化
f0_mean = np.mean(y_train)
loss_mean = 0.5 * np.sum((y_train - f0_mean) ** 2)

# 使用0初始化
f0_zero = 0
loss_zero = 0.5 * np.sum((y_train - f0_zero) ** 2)

# 使用中位数初始化
f0_median = np.median(y_train)
loss_median = 0.5 * np.sum((y_train - f0_median) ** 2)

print(f"均值初始化 (f0={f0_mean:.4f}): 损失={loss_mean:.4f}")
print(f"中位数初始化 (f0={f0_median:.4f}): 损失={loss_median:.4f}")
print(f"零初始化 (f0={f0_zero:.4f}): 损失={loss_zero:.4f}")

# 4. 验证推导
print("\n=== 验证推导 ===")
# 生成数据
np.random.seed(42)
y = np.random.randn(100) * 10 + 50  # 均值约50，标准差10

# 计算不同预测值的损失
predictions = np.linspace(40, 60, 100)
losses = [0.5 * np.sum((y - p) ** 2) for p in predictions]

# 找到最小损失对应的预测值
min_idx = np.argmin(losses)
min_pred = predictions[min_idx]
min_loss = losses[min_idx]

print(f"真实值均值: {np.mean(y):.4f}")
print(f"最小损失对应预测值: {min_pred:.4f}")
print(f"最小损失: {min_loss:.4f}")

# 5. 可视化损失函数
print("\n=== 损失函数分析 ===")
print("预测值 -> 损失:")
for p in [45, 48, 50, 52, 55]:
    loss = 0.5 * np.sum((y - p) ** 2)
    print(f"  f(x)={p}: 损失={loss:.4f}")

# 6. GBDT分类（使用对数损失）
print("\n=== GBDT分类推导 ===")
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_classification

X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# GBDT分类
gbc = GradientBoostingClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)
gbc.fit(X_train, y_train)

# 初始化值（对数损失）
# 对于二分类，初始化值为 log(p/(1-p))，其中p是正样本比例
p = np.mean(y_train)
f0 = np.log(p / (1 - p))
print(f"正样本比例: {p:.4f}")
print(f"初始化值（对数损失）: {f0:.4f}")

# 预测
y_pred = gbc.predict(X_test)
acc = accuracy_score(y_test, y_pred)
print(f"准确率: {acc:.4f}")