步骤：

数据与输出
重排打散

import pyprind
import pandas as pd
import os
basepath = ''
#将原始的txt文件都组合起来到一个csv中去
labels = {'pos':1,'neg':0}
pbar = pyprind.ProgBar(50000)
df = pd.DataFrame()
for s in ('test','train'):
    for l in ('pos','neg'):
        path = os.path.join(basepath,s,l)
        for file in os.listdir(path):
            with open(os.path.join(path,file),'r',encoding='utf-8') as infile:
                txt = infile.read()
                df = df.append([[txt,labels[l]]],ignore_index=True)
                pbar.update()
df.columns = ['review','sentiment']

import numpy as np
np.random.seed(0)
df = df.reindex(np.random.permutation(df.index))
df.to_csv(movie_data.csv,index=False,encoding='utf-8')
#看一下数据
df = pd.read_csv('movie_data.csv',encoding='utf-8')
df.head(3)

bag-of-words词袋模型
词袋模型

n-gram模型
将文本里面的内容按照字节进行大小为N的滑动窗口操作，形成了长度是N的字节片段序列。
n-gram模型
使用sklearn中的CountVectorizer进行特征提取

tf-idf
获取词频.tf(t,d)是前一个部分的词频，idf(t,d)是逆文档频率

link
使用CountVectorizer将文本中的词语转换为词频矩阵

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
count = CountVectorizer()#将文本中词语转换为 词频矩阵（类
docs = np.array(['today is raining','the air is wet','today is raining and the air is wet'])
bag = count.fit_transform(docs)
print(count.vocabulary_)
print(bag.toarray())

from sklearn.feature_extraction.text import TfidfTransformer
tfidf = TfidfTransformer(use_idf=True,norm='l2',smooth_idf=True)#统计每个词语的tf-idf权值（类
np.set_printoptions(precision=2)
print(tfidf.fit_transform(count.fit_transform(docs)).toarray())#计算tf-idf&将文本转为词频矩阵

NLTK
基于python的自然语言处理工具集

建立特征向量
去停词

df=pd.read_csv(homedir+'/movie_data.csv')  

def preprocessor(text):  

    text=re.sub('<[^>]*>','',text)#移除HTML标记，#把<>里面的东西删掉包括内容  

    emotions=re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)',text)  

    text=re.sub('[\W]+',' ',text.lower())+''.join(emotions).replace('-','')  

    return text  

#print (preprocessor(df.loc[0,'review'][-50:]))#数据集第一行review字段的最后50个字符  

#print (preprocessor("This :) is :( a test :-)!"))  

df['review']=df['review'].apply(preprocessor)  

def tokenizer(text):#提取词汇  

    return text.split()  

porter=PorterStemmer()  

def tokenizer_porter(text):#文本分词并提取词干  

    return [porter.stem(word) for word in text.split()]  

nltk.download('stopwords')#停用词移除(stop-word removal)，停用词是文本中常见单不能有效判别信息的词汇  

stop = stopwords.words('english')#获得英文停用词集  

#print ([w for w in tokenizer_porter('a runner likes running and runs a lot') if w not in stop])

训练一个模型来分类影评 logistic regression model

#50000个数据，对半分
X_train=df.loc[:25000,'review'].values  

y_train=df.loc[:25000,'sentiment'].values  

X_test=df.loc[25000:,'review'].values  

y_test=df.loc[25000:,'sentiment'].values  

tfidf=TfidfVectorizer(strip_accents=None,lowercase=False,preprocessor=None)  

param_grid = [{'vect__ngram_range':[(1,1)],'vect__stop_words':[stop,None],'vect__tokenizer':[tokenizer,tokenizer_porter],'clf__penalty':['l1','l2'],'clf__C':[1.0,10.1,100.0]},\  

{'vect__ngram_range':[(1,1)],'vect__stop_words':[stop,None],'vect__tokenizer':[tokenizer,tokenizer_porter],'vect__use_idf':[False],'vect__norm':[None],'clf__penalty':['l1','l2'],'clf__C':[1.0,10.1,100.0]} ]  

lr_tfidf =Pipeline([('vect',tfidf),('clf',LogisticRegression(random_state=0))])  

gs_lr_tfidf=GridSearchCV(lr_tfidf,param_grid,scoring='accuracy',cv=5,verbose=1,n_jobs=-1)  

gs_lr_tfidf.fit(X_train,y_train)  

print ('Best parameter set :%s' % gs_lr_tfidf.best_params_)  

print ('CV Accuracy:%.3f'%gs_lr_tfidf.best_score_)  

clf=gs_lr_tfidf.best_estimator_  

print ('Test Accuracy:%.3f'%clf.score(X_test,y_test))  

end = time.clock()      

print('finish all in %s' % str(end - start))

使用out-of-core来处理大数据

去除停词

import numpy as np 
import re
from nltk.corpus import stopwords
stop = stopwords.words('english')

def tokenizer(text):
    text = re.sub('<[^>]*>','',text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)',text)
    text = (re.sub('[\W]+',' ',text.lower()))+' '.join(emoticons).replace('-','')
    tokenized = [w for w in text.split() if w not in stop]
    return tokenized
#一次返回一个文件
def stream_docs(path):
    with open(path,'r',encoding='utf-8') as csv:
        next(csv)
        for line in csv:
            text,label = line[:-3],int(line[-2])
            yield text,label
next(stream_docs(path = 'movie_data.csv'))

由于CountVectorizer，TfidfVectorizer需要在内存里掌握完整词汇和词频，因此在out-of-core学习中不能使用。
可以用来向量化：HashingVectorizer
link
聚类以判断分类结果

#得到一个文件流，size参数决定文件个数
def get_minibatch(doc_stream,size):
    docs,y=[],[]
    try:
        for _ in range(size):
            text,label = next(doc_stream)
            docs.append(text)
            y.append(label)
    except StopInteration:
        return None,None
    return docs,y

from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.linear_model import SGDClassifier
vect = HashingVectorizer(decode_error='ignore',n_features=2**21,preprocessor=None,tokenizer=tokenizer)
clf = SGDClassifier(loss='log',random_state=1,n_iter=1)
doc_stream = stream_docs(path='movie_data.csv')

训练

import pyprind
pbar = pyprind.ProgBar(45)
classes = np.array([0,1])
for _ in range(45):
    X_train,y_train = get_minibatch(doc_stream,size=1000)
    if not X_train:
        break
    X_train = vect.transform(X_train)
    clf.partial_fit(X_traina,y_train,classes=classes)
    pbar.update()

得到的结果是：Accuracy: 0.878。稍微低于之前的结果Accuracy: 0.899，但是省内存而且速度快。

主题建模with LDA（隐含狄利克雷分布）

LDA是一个生成概率模型，试图找出在不同文档中频繁出现的词组。这些频繁出现的词代表了主题。假设每个文档是不同单词的混合，LDA的输入是前面讨论的词袋模型，给定词袋矩阵作为输入，LDA分解为两个新矩阵：文档主题矩阵、词对主题矩阵。

LDA分解字袋矩阵的方式，如果把两个矩阵乘起来，可以以低误差复制输入，词袋矩阵。唯一的缺点是必须预先定义主题的数量，主题的数量是必须手动指定的LDA的超参数。

加载数据集

LatentDirichletAllocation类：in scikit-learn）分解数据库，按主题分类。

import pandas as pd 
df = pd.read_csv('movie_data.csv',encoding='utf-8')
from sklearn.feature_extraction.text import CountVectorizer
count = CountVectorizer(stop_words='english',max_df=.1,max_features=5000)
#熟悉的countvectorizer创建词袋矩阵
#5000限制了最常出现的5000词
#1限制了最多出现的词频为十分之一，去掉非常常见的词语
X = count.fit_transform(df['review'].values)

fit a LDA estimator，找到10个主题

from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_topics=10,random_state=123,learning_method='batch')
#batch：让lda每次预测都使用所有training data（词袋矩阵）默认是online，比较快
X_topics=lda.fit_transform(X)

lda使用的em算法更新模型参数
link

#输出每个主题的前几个词语
n_top_words = 10
feature_names = count.get_feature_names()
for topic_idx,topic in enumerate(lda.components_):
    print("topic %d:"%(topic_idx+1))
    print(" ".join([feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]))

利用pyLDAvis 可视化结果

import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
pyLDAvis.sklearn.prepare(lda, X, count)