集成学习之Bagging和Boosting思想介绍

import numpy as np
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.datasets import make_classification

# 1. 生成数据
X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_classes=2,
    random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 2. Bagging思想 - 随机森林
print("=== Bagging - 随机森林 ===")
rf = RandomForestClassifier(
    n_estimators=100,  # 100个弱学习器
    random_state=42
)
rf.fit(X_train, y_train)
rf_acc = accuracy_score(y_test, rf.predict(X_test))
print(f"准确率: {rf_acc:.4f}")

# 3. Boosting思想 - AdaBoost
print("\n=== Boosting - AdaBoost ===")
ab = AdaBoostClassifier(
    n_estimators=100,
    random_state=42
)
ab.fit(X_train, y_train)
ab_acc = accuracy_score(y_test, ab.predict(X_test))
print(f"准确率: {ab_acc:.4f}")

# 4. 模拟Bagging有放回抽样
print("\n=== 模拟Bagging有放回抽样 ===")
np.random.seed(42)
data_size = 1000
sample_size = 700

# 3个弱学习器各自抽样
samples = []
for i in range(3):
    sample = np.random.choice(data_size, sample_size, replace=True)
    samples.append(sample)
    print(f"弱学习器{i+1}: 抽取{sample_size}条，唯一值{len(set(sample))}个")

# 计算交集和差异
intersection_12 = len(set(samples[0]) & set(samples[1]))
intersection_13 = len(set(samples[0]) & set(samples[2]))
intersection_23 = len(set(samples[1]) & set(samples[2]))

print(f"\n弱学习器1和2的交集: {intersection_12}")
print(f"弱学习器1和3的交集: {intersection_13}")
print(f"弱学习器2和3的交集: {intersection_23}")
print("既有交集又有差异，投票才有意义")

# 5. 模拟Bagging投票决策
print("\n=== 模拟Bagging投票决策 ===")
# 训练3个弱学习器
weak_learners = []
for i in range(3):
    dt = DecisionTreeClassifier(max_depth=3, random_state=i)
    # 有放回抽样
    sample_idx = np.random.choice(len(X_train), len(X_train), replace=True)
    dt.fit(X_train[sample_idx], y_train[sample_idx])
    weak_learners.append(dt)

# 各弱学习器预测
predictions = np.array([learner.predict(X_test) for learner in weak_learners])
print(f"3个弱学习器的预测形状: {predictions.shape}")

# 投票（多数表决）
from scipy.stats import mode
final_pred = mode(predictions, axis=0)[0].ravel()
vote_acc = accuracy_score(y_test, final_pred)
print(f"投票决策准确率: {vote_acc:.4f}")

# 6. 模拟Boosting串行训练
print("\n=== 模拟Boosting串行训练 ===")
# AdaBoost的串行过程
ab = AdaBoostClassifier(n_estimators=5, random_state=42)
ab.fit(X_train, y_train)

# 查看各弱学习器的权重
print("各弱学习器权重:")
for i, weight in enumerate(ab.estimator_weights_):
    print(f"  弱学习器{i+1}: 权重={weight:.4f}")

# 7. 可视化Bagging vs Boosting
fig, axes = plt.subplots(1, 2, figsize=(15, 6))

# Bagging流程图
axes[0].set_title('Bagging思想（并行）', fontsize=14)
axes[0].axis('off')
bagging_text = """
数据集（1000条）
    ↓ 有放回抽样
┌───┬───┬───┐
↓   ↓   ↓
弱学习器1  弱学习器2  弱学习器3
（并行训练）
    ↓   ↓   ↓
预测1  预测2  预测3
    ↓   ↓   ↓
    投票评选
        ↓
    最终结果
"""
axes[0].text(0.1, 0.5, bagging_text, fontsize=12, verticalalignment='center')

# Boosting流程图
axes[1].set_title('Boosting思想（串行）', fontsize=14)
axes[1].axis('off')
boosting_text = """
数据集（全量）
    ↓
弱学习器1 → 不足
    ↓
弱学习器2（针对不足）
    ↓ 不足
弱学习器3（针对不足）
    ↓
加权投票
    ↓
最终结果
"""
axes[1].text(0.1, 0.5, boosting_text, fontsize=12, verticalalignment='center')

plt.tight_layout()
plt.show()

# 8. 对比总结
print("\n=== 对比总结 ===")
print(f"Bagging（随机森林）: {rf_acc:.4f}")
print(f"Boosting（AdaBoost）: {ab_acc:.4f}")
print("\nBagging: 有放回抽样，并行训练，投票评选")
print("Boosting: 全量数据，串行训练，加权投票")