GBDT算法之梯度提升树介绍

import numpy as np
from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, accuracy_score
from sklearn.datasets import make_regression, make_classification

# 1. 模拟残差提升过程
print("=== 模拟残差提升过程 ===")
np.random.seed(42)

# 真实值
y_true = 100

# 多轮预测
predictions = []
residuals = []
current_target = y_true

for i in range(5):
    # 模拟预测（逐步逼近）
    pred = current_target * 0.8  # 预测80%
    residual = current_target - pred
    
    predictions.append(pred)
    residuals.append(residual)
    
    print(f"第{i+1}轮: 目标={current_target:.2f}, 预测={pred:.2f}, 残差={residual:.2f}")
    
    # 残差作为下一轮的目标
    current_target = residual

print(f"\n最终预测: {sum(predictions):.2f}")
print(f"真实值: {y_true}")

# 2. GBDT回归
print("\n=== GBDT回归 ===")
X, y = make_regression(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    noise=0.1,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# GBDT回归
gbr = GradientBoostingRegressor(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)
gbr.fit(X_train, y_train)

y_pred = gbr.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print(f"MSE: {mse:.4f}")

# 3. GBDT分类
print("\n=== GBDT分类 ===")
X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# GBDT分类
gbc = GradientBoostingClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3,
    random_state=42
)
gbc.fit(X_train, y_train)

y_pred = gbc.predict(X_test)
acc = accuracy_score(y_test, y_pred)
print(f"准确率: {acc:.4f}")

# 4. 手动实现GBDT回归
print("\n=== 手动实现GBDT回归 ===")
class SimpleGBDT:
    """简单的GBDT回归实现"""
    
    def __init__(self, n_estimators=10, learning_rate=0.1, max_depth=3):
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.max_depth = max_depth
        self.trees = []
        self.initial_pred = 0
    
    def fit(self, X, y):
        # 初始化预测值（均值）
        self.initial_pred = np.mean(y)
        current_pred = np.full(len(y), self.initial_pred)
        
        for i in range(self.n_estimators):
            # 计算残差（负梯度）
            residual = y - current_pred
            
            # 训练回归树
            tree = DecisionTreeRegressor(max_depth=self.max_depth, random_state=i)
            tree.fit(X, residual)
            
            # 更新预测值
            current_pred += self.learning_rate * tree.predict(X)
            self.trees.append(tree)
            
            if (i + 1) % 5 == 0:
                mse = mean_squared_error(y, current_pred)
                print(f"第{i+1}轮: MSE={mse:.4f}")
    
    def predict(self, X):
        pred = np.full(len(X), self.initial_pred)
        for tree in self.trees:
            pred += self.learning_rate * tree.predict(X)
        return pred

# 生成数据
X, y = make_regression(
    n_samples=500,
    n_features=10,
    noise=0.1,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 训练
gbdt = SimpleGBDT(n_estimators=20, learning_rate=0.1, max_depth=3)
gbdt.fit(X_train, y_train)

# 预测
y_pred = gbdt.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
print(f"\n测试集MSE: {mse:.4f}")

# 5. 对比不同算法
print("\n=== 对比不同算法 ===")
from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier

X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 随机森林
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
rf_acc = accuracy_score(y_test, rf.predict(X_test))

# AdaBoost
ab = AdaBoostClassifier(n_estimators=100, random_state=42)
ab.fit(X_train, y_train)
ab_acc = accuracy_score(y_test, ab.predict(X_test))

# GBDT
gbc = GradientBoostingClassifier(n_estimators=100, random_state=42)
gbc.fit(X_train, y_train)
gbc_acc = accuracy_score(y_test, gbc.predict(X_test))

print(f"随机森林(Bagging): {rf_acc:.4f}")
print(f"AdaBoost(Boosting): {ab_acc:.4f}")
print(f"GBDT(Boosting): {gbc_acc:.4f}")