一、文本预处理阶段###
1.1 设定训练集和测试集
训练集每一类的数量为500个文档，测试集每一类的数量也为500个文档。
1.2 计算每个文本的DF
为每一个文本计算TF，return格式为：'word', 'file_name', term-frequency
先算出每个文档中的'word', term-frequency, 在结束改文本的循环后将该文本中出现的词以 'word', 'file_name', term-frequency的形式加入 word_docid_tf
def compute_tf_by_file(self):
word_docid_tf = []
for name in self.filenames:
with open(join(name), 'r') as f:
tf_dict = dict()
for line in f:
line = self.process_line(line)
words = jieba.cut(line.strip(), cut_all=False)
for word in words:
tf_dict[word] = tf_dict.get(word, 0) + 1
tf_list = tf_dict.items()
word_docid_tf += [[item[0], name, item[1]] for item in tf_list]
return word_docid_tf
1.3 计算每个词项的TF、DF
为每一个词项计算TF，return的term_freq格式为：'word', dict ( 'file_name ', tf )
为每一个词项计算DF，return的doc_freq格式为：'word', df
def compute_tfidf(self):
word_docid_tf = self.compute_tf_by_file()
word_docid_tf.sort()
doc_freq = dict()
term_freq = dict()
for current_word, group in groupby(word_docid_tf, itemgetter(0)):
doclist = []
df = 0
for current_word, file_name, tf in group:
doclist.append((file_name, tf))
df += 1
term_freq[current_word] = dict(doclist)
doc_freq[current_word] = df
return term_freq, doc_freq
1.4 精简term_freq, doc_freq
除去只出现在一个或0个文档中的词项
除去数字词项
def reduce_tfidf(self, term_freq, doc_freq):
remove_list = []
for key in term_freq.keys():
if len(key) < 2:#该词只出现在一个或0个文档中
remove_list.append(key)
else:
try:
float(key)#该词是数字
remove_list.append(key)
except ValueError:
continue
for key in remove_list:
term_freq.pop(key)
doc_freq.pop(key)
return term_freq, doc_freq
1.5 为每个文本构建特征向量train_feature, train_target
为term_freq, doc_freq中的key，也就是词项标明index
用jieba分词，将分好的词放入一个临时的数组中。
遍历数组，由doc_freq[word]取得DF并计算iDF，由term_freq[word][name]
取得该词项在该文档中的TF，并计算每个词项的tf-idf值，并作为向量中词项对应index那一维的值。
train_feature, train_target = train_tfidf.tfidf_feature(os.path.join(input_path, 'train'),train_tf, train_df, N)
def tfidf_feature(self, dir, term_freq, doc_freq, N):
filenames = []
for (dirname, dirs, files) in os.walk(dir):
for file in files:
filenames.append(os.path.join(dirname, file))
word_list = dict()
for idx, word in enumerate(doc_freq.keys()):
word_list[word] = idx
features = []
target = []
for name in filenames:
feature = np.zeros(len(doc_freq.keys()))
words_in_this_file = set()
tags = re.split('[/\\\\]', name)
tag = tags[-2]
with open(name, 'rb') as f:
for line in f:
line = self.process_line(line)
words = jieba.cut(line.strip(), cut_all=False)
for word in words:
words_in_this_file.add(word)
for word in words_in_this_file:
try:
idf = np.log(float(N) / doc_freq[word])
tf = term_freq[word][name]
feature[word_list[word]] = tf*idf
except KeyError:
continue
features.append(feature)
target.append(tag)
return sparse.csr_matrix(np.asarray(features)), np.asarray(target)
1.6 存储&加载
为了节约之后运行的时间，可以通过如下方式把测试集tf和df的值直接存储：
Pickle.dump(train_tf, open(os.path.join(input_path, 'train_tf.pkl'), 'wb'))
print "saved train_tf.pkl"
Pickle.dump(train_df, open(os.path.join(input_path, 'train_df.pkl'), 'wb'))
print "saved train_df.pkl"
之后运行时，可以通过如下方式把测试集tf和df的值直接加载到内存，省去了重新计算的时间：
train_tf = Pickle.load(open(os.path.join(input_path, 'train_tf.pkl'), 'rb'))
print "loaded train_tf.pkl"
train_df = Pickle.load(open(os.path.join(input_path, 'train_df.pkl'), 'rb'))
train_tfidf.doc_freq=train_df
print "loaded train_df.pkl"
