商品评论情感分析之数据预处理

data = pd.read_csv('data.csv', encoding='gbk')

data['labels'] = np.where(data['sentiment'] == '好评', 1, 0)

# 读取停用词文件
with open('stopwords.txt', 'r', encoding='utf-8') as f:
    stop_words = f.readlines()

# 移除换行符
stop_words = [word.strip() for word in stop_words]

# 去重
stop_words = list(set(stop_words))

import jieba

# 对文档进行分词
data['cut_words'] = data['content'].apply(lambda x: ' '.join(jieba.lcut(x)))

# 列表套列表 → 列表套字符串
data['cut_words'] = data['content'].apply(lambda x: ' '.join(jieba.lcut(x)))

from sklearn.feature_extraction.text import CountVectorizer

# 创建向量化对象
vectorizer = CountVectorizer()

# 转换
X = vectorizer.fit_transform(data['cut_words'])

feature_names = vectorizer.get_feature_names_out()
print(f"特征数量: {len(feature_names)}")

import pandas as pd
import numpy as np
import jieba
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score

# 1. 模拟商品评论数据
print("=== 1. 读数据 ===")
data = pd.DataFrame({
    'content': [
        '从编程小白的角度看，入门极佳，推荐购买',
        '质量很好，用起来很舒服，好评',
        '性价比高，值得入手，非常满意',
        '物流快，包装好，质量不错',
        '差评，质量太差了，浪费钱',
        '不好用，太失望了，不推荐',
        '太差了，退货了，质量堪忧',
        '失望，与描述不符，差评',
        '这本书写得很好，通俗易懂，推荐',
        '内容丰富，案例多，值得一看',
        '纸张质量差，印刷模糊，差评',
        '内容空洞，没有实质内容，不推荐'
    ],
    'sentiment': ['好评', '好评', '好评', '好评', '差评', '差评', 
                  '差评', '差评', '好评', '好评', '差评', '差评']
})
print(f"数据量: {len(data)}")
print(data.head())

# 2. 新增标签列
print("\n=== 2. 新增标签列 ===")
data['labels'] = np.where(data['sentiment'] == '好评', 1, 0)
print(data[['content', 'sentiment', 'labels']].head())

# 3. 加载停用词
print("\n=== 3. 加载停用词 ===")
# 模拟停用词
stop_words_list = [
    '，', '。', '、', '了', '的', '从', '看', '很', '太', '是',
    '在', '和', '与', '都', '也', '就', '只', '还', '又', '但'
]

# 去重
stop_words = list(set(stop_words_list))
print(f"停用词数量: {len(stop_words)}")
print(f"停用词: {stop_words}")

# 4. 分词
print("\n=== 4. 分词 ===")

def cut_words(text):
    """分词并删除停用词"""
    words = jieba.lcut(text)
    # 删除停用词
    valid_words = [word for word in words if word not in stop_words and word.strip()]
    return ' '.join(valid_words)

data['cut_content'] = data['content'].apply(cut_words)

for i in range(3):
    print(f"原文: {data.iloc[i]['content']}")
    print(f"分词: {data.iloc[i]['cut_content']}")
    print()

# 5. 词频矩阵
print("=== 5. 词频矩阵 ===")
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(data['cut_content'])

print(f"词频矩阵形状: {X.shape}")
print(f"特征数量: {X.shape[1]}")

# 查看特征名称
feature_names = vectorizer.get_feature_names_out()
print(f"\n前20个特征: {feature_names[:20]}")

# 查看词频矩阵（前3行）
print(f"\n词频矩阵（前3行）:")
print(X[:3].toarray())

# 6. 划分数据集
print("\n=== 6. 划分数据集 ===")
y = data['labels']

# 前10行训练，后2行测试
X_train = X[:10]
X_test = X[10:]
y_train = y[:10]
y_test = y[10:]

print(f"训练集: {X_train.shape}")
print(f"测试集: {X_test.shape}")

# 7. 训练模型
print("\n=== 7. 训练模型 ===")
nb = MultinomialNB(alpha=1.0)
nb.fit(X_train, y_train)
print("模型训练完成")

# 8. 评估模型
print("\n=== 8. 评估模型 ===")
y_pred = nb.predict(X_test)
acc = accuracy_score(y_test, y_pred)
print(f"准确率: {acc:.4f}")

# 9. 预测新评论
print("\n=== 9. 预测新评论 ===")
new_comments = [
    '质量很好，推荐购买',
    '太差了，浪费钱，不推荐'
]

for comment in new_comments:
    # 分词
    cut_comment = cut_words(comment)
    # 向量化
    X_new = vectorizer.transform([cut_comment])
    # 预测
    pred = nb.predict(X_new)[0]
    proba = nb.predict_proba(X_new)[0]
    
    label = '好评' if pred == 1 else '差评'
    print(f"评论: {comment}")
    print(f"分词: {cut_comment}")
    print(f"预测: {label} (好评概率: {proba[1]:.4f})")
    print()

# 10. 完整流程函数
def sentiment_analysis_preprocess():
    """情感分析数据预处理完整流程"""
    print("=" * 50)
    print("商品评论情感分析 - 数据预处理")
    print("=" * 50)
    
    # 1. 读数据
    print("\n1. 读数据")
    data = pd.DataFrame({
        'content': [
            '从编程小白的角度看，入门极佳，推荐购买',
            '质量很好，用起来很舒服，好评',
            '性价比高，值得入手，非常满意',
            '物流快，包装好，质量不错',
            '差评，质量太差了，浪费钱',
            '不好用，太失望了，不推荐',
            '太差了，退货了，质量堪忧',
            '失望，与描述不符，差评'
        ],
        'sentiment': ['好评', '好评', '好评', '好评', '差评', '差评', '差评', '差评']
    })
    print(f"   数据量: {len(data)}")
    
    # 2. 新增标签列
    print("\n2. 新增标签列")
    data['labels'] = np.where(data['sentiment'] == '好评', 1, 0)
    
    # 3. 加载停用词
    print("\n3. 加载停用词")
    stop_words = ['，', '。', '、', '了', '的', '从', '看', '很', '太', '是']
    stop_words = list(set(stop_words))
    print(f"   停用词数量: {len(stop_words)}")
    
    # 4. 分词
    print("\n4. 分词")
    def cut_words(text):
        words = jieba.lcut(text)
        valid_words = [w for w in words if w not in stop_words and w.strip()]
        return ' '.join(valid_words)
    
    data['cut_content'] = data['content'].apply(cut_words)
    
    # 5. 词频矩阵
    print("\n5. 词频矩阵")
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(data['cut_content'])
    print(f"   矩阵形状: {X.shape}")
    print(f"   特征数量: {X.shape[1]}")
    
    # 6. 划分数据集
    print("\n6. 划分数据集")
    y = data['labels']
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.25, random_state=42
    )
    print(f"   训练集: {X_train.shape}")
    print(f"   测试集: {X_test.shape}")
    
    # 7. 训练模型
    print("\n7. 训练模型")
    nb = MultinomialNB(alpha=1.0)
    nb.fit(X_train, y_train)
    
    # 8. 评估
    print("\n8. 评估模型")
    y_pred = nb.predict(X_test)
    acc = accuracy_score(y_test, y_pred)
    print(f"   准确率: {acc:.4f}")
    
    print("\n" + "=" * 50)
    print("预处理完成!")
    print("=" * 50)
    
    return nb, vectorizer

from sklearn.model_selection import train_test_split

# 运行
nb, vectorizer = sentiment_analysis_preprocess()