词袋模型文本分类

# -*- coding: utf-8 -*-

import numpyas np

import pandasas pd

import jieba

from sklearn.feature_extraction.textimport TfidfVectorizer

from sklearn.preprocessingimport LabelEncoder

from sklearn.model_selectionimport train_test_split

from sklearn.linear_modelimport LogisticRegression

from sklearn.metricsimport confusion_matrix, precision_recall_fscore_support

import dill

import pickle

加载停用词

def stop_words():
     stopwords = set()
     with open('../data/stopwords.txt', 'r',encoding='utf8') as infile:
     for line in infile:
             line = line.strip('\n') if line:
             stopwords.add(line.lower())
     return stopwords

训练模型

def get_tf_idf():

    # 常用参数说明

    # penalty: 正则项类型，l1还是l2

    # C: 正则项惩罚系数的倒数，越大则惩罚越小

    # fit_intercept: 是否拟合常数项

    # max_iter: 最大迭代次数

    # multi_class: 以何种方式训练多分类模型

    #    ovr = 对每个标签训练二分类模型

    #    multinomial = 直接训练多分类模型，仅当solver={newton-cg, sag, lbfgs}时支持

    # solver: 用哪种方法求解，可选有{liblinear, newton-cg, sag, lbfgs}

    #    小数据liblinear比较好，大数据量sag更快

    #    多分类问题，liblinear只支持ovr模式，其他支持ovr和multinomial

    #    liblinear支持l1正则，其他只支持l2正则

    stopwords=stop_words()

    train_data = pd.read_csv('../data/sohu_train.txt',sep='\t',header=None,dtype=np.str_,encoding='utf8',

names=['频道','文章'])

    tfidf = TfidfVectorizer(tokenizer=jieba.lcut,stop_words=stopwords,min_df=50,max_df=0.3)

    x = tfidf.fit_transform(train_data['文章'])

    y_encoder = LabelEncoder()

    y = y_encoder.fit_transform(train_data['频道'])

    train_idx, test_idx = train_test_split(range(len(y)),test_size=0.2,stratify=y)

    train_x = x[train_idx, :]

    train_y = y[train_idx]

    test_x = x[test_idx, :]

    test_y = y[test_idx]

    model = LogisticRegression(multi_class='multinomial',solver='lbfgs')

    model.fit(train_x, train_y)

    test_y_pred = model.predict(test_x)

# 保存模型

    with open('../model_path/tf_idf_model.pkl','wb')as outfile:

    pickle.dump({

    'y_encoder': y_encoder,

    'tfidf': tfidf,

    'lr': model

    },outfile)

    eval_model(test_y, test_y_pred, y_encoder.classes_)

计算各项评价指标

def eval_model(y_true, y_pred, labels):

# 计算每个分类的Precision, Recall, f1, support

    p, r, f1, s = precision_recall_fscore_support(y_true, y_pred)

# 计算总体的平均Precision, Recall, f1, support

    tot_p = np.average(p,weights=s)

    tot_r = np.average(r,weights=s)

    tot_f1 = np.average(f1,weights=s)

    tot_s = np.sum(s)

    res1 = pd.DataFrame({

    u'Label': labels,

    u'Precision': p,

    u'Recall': r,

    u'F1': f1,

    u'Support': s

    })

    res2 = pd.DataFrame({

    u'Label': [u'总体'],

    u'Precision': [tot_p],

    u'Recall': [tot_r],

    u'F1': [tot_f1],

    u'Support': [tot_s]

    })

    res2.index = [999]

    res = pd.concat([res1, res2])

    return res[[u'Label',u'Precision',u'Recall',u'F1',u'Support']]

对新数据进行预测

def prdition(model_file,new_data):

    import pickle

    with open(model_file,'rb')as infile:

        model = pickle.load(infile)

    new_x = model['tfidf'].transform(new_data[u'文章'][:10])

    new_y_pred = model['lr'].predict(new_x)

    return new_y_pred

if __name__ =='__main__':

    get_tf_idf()