Boosting思想之AdaBoost自适应提升树介绍

import numpy as np
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_classification

# 1. 生成数据
X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 2. AdaBoost基本使用
print("=== AdaBoost基本使用 ===")
ab = AdaBoostClassifier(
    n_estimators=50,
    random_state=42
)
ab.fit(X_train, y_train)
ab_acc = accuracy_score(y_test, ab.predict(X_test))
print(f"准确率: {ab_acc:.4f}")

# 3. 查看弱学习器权重
print("\n=== 弱学习器权重 ===")
for i, weight in enumerate(ab.estimator_weights_[:10]):
    print(f"弱学习器{i+1}: 权重={weight:.4f}")

# 4. 模拟AdaBoost过程
print("\n=== 模拟AdaBoost过程 ===")
np.random.seed(42)
n_samples = len(X_train)

# 初始化样本权重（均匀分布）
sample_weights = np.ones(n_samples) / n_samples

# 训练多个弱学习器
weak_learners = []
learner_weights = []

for t in range(5):
    # 训练弱学习器
    dt = DecisionTreeClassifier(max_depth=1, random_state=t)
    dt.fit(X_train, y_train, sample_weight=sample_weights)
    
    # 预测
    y_pred = dt.predict(X_train)
    
    # 计算错误率
    incorrect = (y_pred != y_train)
    error_rate = np.sum(sample_weights * incorrect)
    
    # 计算弱学习器权重
    alpha = 0.5 * np.log((1 - error_rate) / error_rate)
    
    # 更新样本权重
    sample_weights = sample_weights * np.exp(-alpha * y_train * (2 * y_pred - 1))
    
    # 归一化
    sample_weights = sample_weights / np.sum(sample_weights)
    
    weak_learners.append(dt)
    learner_weights.append(alpha)
    
    print(f"第{t+1}轮: 错误率={error_rate:.4f}, 权重={alpha:.4f}")

# 5. 对比不同算法
print("\n=== 对比不同算法 ===")
from sklearn.ensemble import RandomForestClassifier

# AdaBoost
ab = AdaBoostClassifier(n_estimators=100, random_state=42)
ab.fit(X_train, y_train)
ab_acc = accuracy_score(y_test, ab.predict(X_test))

# 随机森林
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
rf_acc = accuracy_score(y_test, rf.predict(X_test))

# 单一决策树
dt = DecisionTreeClassifier(random_state=42)
dt.fit(X_train, y_train)
dt_acc = accuracy_score(y_test, dt.predict(X_test))

print(f"单一决策树: {dt_acc:.4f}")
print(f"随机森林(Bagging): {rf_acc:.4f}")
print(f"AdaBoost(Boosting): {ab_acc:.4f}")

# 6. 可视化权重变化
print("\n=== 权重变化示例 ===")
# 模拟10个样本的权重变化
np.random.seed(42)
n = 10
weights = np.ones(n) / n  # 初始权重

print(f"初始权重: {weights}")

# 模拟5轮
for t in range(5):
    # 模拟预测（随机）
    y_true = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
    y_pred = np.random.randint(0, 2, n)
    
    # 计算错误率
    incorrect = (y_pred != y_true)
    error_rate = np.sum(weights * incorrect)
    
    if error_rate == 0 or error_rate >= 0.5:
        continue
    
    # 计算权重
    alpha = 0.5 * np.log((1 - error_rate) / error_rate)
    
    # 更新权重
    weights = weights * np.exp(alpha * incorrect * 2 - alpha)
    weights = weights / np.sum(weights)
    
    print(f"第{t+1}轮: 错误率={error_rate:.4f}, alpha={alpha:.4f}")
    print(f"  权重: {weights}")