XGBoost极限梯度提升树之简介

import numpy as np
from sklearn.datasets import make_classification, make_regression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, mean_squared_error

# 1. 模拟打分函数
print("=== 模拟打分函数 ===")
def calculate_score(G, H, lambda_=1.0):
    """计算打分"""
    return -0.5 * (G ** 2) / (H + lambda_)

# 模拟分裂前后的打分
G_total = 10  # 总一阶导数
H_total = 5   # 总二阶导数
lambda_ = 1.0

# 分裂前
score_before = calculate_score(G_total, H_total, lambda_)
print(f"分裂前打分: {score_before:.4f}")

# 分裂后
G_L, H_L = 6, 3  # 左子树
G_R, H_R = 4, 2  # 右子树

score_L = calculate_score(G_L, H_L, lambda_)
score_R = calculate_score(G_R, H_R, lambda_)
score_after = score_L + score_R
print(f"分裂后打分: {score_after:.4f} (左: {score_L:.4f}, 右: {score_R:.4f})")

# 判断是否分裂
gamma = 0.5  # 正则化参数
gain = score_after - score_before - gamma
print(f"分裂增益: {gain:.4f}")
print(f"决策: {'分裂' if gain > 0 else '不分裂'}")

# 2. XGBoost分类
print("\n=== XGBoost分类 ===")
try:
    from xgboost import XGBClassifier
    
    X, y = make_classification(
        n_samples=1000,
        n_features=20,
        n_informative=10,
        n_classes=2,
        random_state=42
    )
    
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    # XGBoost分类
    xgb = XGBClassifier(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        random_state=42
    )
    xgb.fit(X_train, y_train)
    
    y_pred = xgb.predict(X_test)
    acc = accuracy_score(y_test, y_pred)
    print(f"准确率: {acc:.4f}")
    
except ImportError:
    print("请安装xgboost: pip install xgboost")
    
    # 使用GBDT作为替代
    from sklearn.ensemble import GradientBoostingClassifier
    
    X, y = make_classification(
        n_samples=1000,
        n_features=20,
        n_informative=10,
        n_classes=2,
        random_state=42
    )
    
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    gbc = GradientBoostingClassifier(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        random_state=42
    )
    gbc.fit(X_train, y_train)
    
    y_pred = gbc.predict(X_test)
    acc = accuracy_score(y_test, y_pred)
    print(f"GBDT准确率(替代): {acc:.4f}")

# 3. XGBoost回归
print("\n=== XGBoost回归 ===")
try:
    from xgboost import XGBRegressor
    
    X, y = make_regression(
        n_samples=1000,
        n_features=20,
        n_informative=10,
        noise=0.1,
        random_state=42
    )
    
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    xgb_reg = XGBRegressor(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        random_state=42
    )
    xgb_reg.fit(X_train, y_train)
    
    y_pred = xgb_reg.predict(X_test)
    mse = mean_squared_error(y_test, y_pred)
    print(f"MSE: {mse:.4f}")
    
except ImportError:
    print("请安装xgboost: pip install xgboost")

# 4. 对比不同算法
print("\n=== 对比不同算法 ===")
from sklearn.ensemble import (
    RandomForestClassifier,
    AdaBoostClassifier,
    GradientBoostingClassifier
)
from sklearn.tree import DecisionTreeClassifier

X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 单一决策树
dt = DecisionTreeClassifier(random_state=42)
dt.fit(X_train, y_train)
dt_acc = accuracy_score(y_test, dt.predict(X_test))

# 随机森林
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
rf_acc = accuracy_score(y_test, rf.predict(X_test))

# AdaBoost
ab = AdaBoostClassifier(n_estimators=100, random_state=42)
ab.fit(X_train, y_train)
ab_acc = accuracy_score(y_test, ab.predict(X_test))

# GBDT
gbc = GradientBoostingClassifier(n_estimators=100, random_state=42)
gbc.fit(X_train, y_train)
gbc_acc = accuracy_score(y_test, gbc.predict(X_test))

print(f"单一决策树: {dt_acc:.4f}")
print(f"随机森林(Bagging): {rf_acc:.4f}")
print(f"AdaBoost(Boosting): {ab_acc:.4f}")
print(f"GBDT(Boosting): {gbc_acc:.4f}")

try:
    from xgboost import XGBClassifier
    xgb = XGBClassifier(n_estimators=100, random_state=42)
    xgb.fit(X_train, y_train)
    xgb_acc = accuracy_score(y_test, xgb.predict(X_test))
    print(f"XGBoost(Boosting): {xgb_acc:.4f}")
except ImportError:
    print("XGBoost: 需要安装xgboost库")

# 5. XGBoost参数说明
print("\n=== XGBoost参数说明 ===")
print("常用参数:")
print("  n_estimators: 树的数量")
print("  learning_rate: 学习率")
print("  max_depth: 树的最大深度")
print("  subsample: 样本采样比例")
print("  colsample_bytree: 特征采样比例")
print("  gamma: 分裂最小增益")
print("  lambda: L2正则化系数")
print("  alpha: L1正则化系数")