1、读取并整理数据
library(pacman)
p_load(tidytext,tidyverse,stringr)
# 读取文件
txt <- readLines("./data_set/xslm/血色浪漫.txt",encoding = "unknown",n=-1L,ok=T,warn=T,skipNul=T)
血色浪漫
# 删除分节等重复信息
txt <- txt %>% gsub("出版社：长江文艺出版社作者：都梁|\\[上一篇\\]","",.) %>% str_trim()
# 按【下一篇】拆分
txt2 <- txt %>% paste(collapse = " ") %>% str_split("\\[下一篇\\]")
# 按章节整理为数据框
chapter <- vector("character",244)
content <- vector("character",244)
for (line in txt2) {
temp <- line %>% str_split("引子|\\([0-9]+\\)")
for (i in 1:length(line)) {
chapter[i] <- temp[[i]][1] %>% gsub("[《血色浪漫上》]","",.) %>% str_trim(side = "both")
content[i] <- temp[[i]][2] %>% str_trim(side = "both")
}
df <- tibble(chapter=chapter,content=content)
}
# 给第0章添加标题“引子”
df$chapter[1] <- "引子"
# 按章节（chapter）将各小节（content）合并
txt3 <- aggregate(content~chapter,df,paste)
# 去掉字符及字母
txt3$content <- txt3$content %>% gsub("[%a-z\\\\\\\"()]","",.)
# 创建有效水平章节名称列表
chapter.levels <- c("引子","第一章","第二章","第三章","第四章","第五章","第六章",
"第七章","第八章","第九章","第十章","第十一章","第十二章",
"第十三章","第十四章","第十五章","第十六章","第十七章",
"第十八章","第十九章","第二十章","第二十一章","第二十二章",
"第二十三章","第二十四章","第二十五章","尾声")
txt3$chapter <- factor(txt3$chapter,levels = chapter.levels)
# 按章节重新排序
txt3 <- txt3 %>% arrange(chapter)
# 写入文件
# write.csv(txt3,"./txt3.csv")
整理后的文档
2、中文分词
使用dplyr::count()函数计算词频一直有问题，最后选择使用text2vec包。
p_load(jiebaR,plyr,text2vec)
# txt3 <- read.csv("./txt3.csv",header = T,stringsAsFactors = F)
# 将小说主要人物姓名保存为用户字典xslm
wk <-worker(user = "./dict/characters-master/xslm")
tok.fun <- function(strings) {llply(strings,segment,wk)}
# 设置分词迭代器
it <- itoken(txt3$content,
preprocessor = identity,
tokenizer = tok.fun,
ids = txt3$chapter,
progressbar = interactive())
# 创建词矩阵
vocab <- create_vocabulary(it)
3、计算TF和TF-IDF
3.1 计算TF
book.words <- tibble(chapter=vocab$doc_count,
word=as.character(vocab$term),
n=as.integer(vocab$term_count))
# 计算每章中频次最高的词
max.word <- aggregate(n~chapter,book.words,max)
# 词频tf=某个词在文章中出现的次数/该文出现次数最多的词出现的次数
book <- book.words %>% left_join(max.word,by="chapter",copy=T) %>%
dplyr::rename(n=n.x,max=n.y) %>% mutate(tf=n/max)
print(book)
# A tibble: 24,572 x 5
chapter word n max tf
<int> <chr> <int> <int> <dbl>
1 1 得太早 1 35 0.0286
2 1 招待 1 35 0.0286
3 1 全副武装 1 35 0.0286
4 1 演义 1 35 0.0286
5 1 低么 1 35 0.0286
6 1 黄皮书 1 35 0.0286
7 1 养人 1 35 0.0286
8 1 付些 1 35 0.0286
9 1 解 1 35 0.0286
10 1 破裂 1 35 0.0286
# ... with 24,562 more rows
3.2 查看前6章节TF的分布情况
book %>% filter(chapter<7) %>%
ggplot(aes(tf, fill = chapter)) +
geom_histogram(show.legend = FALSE,bins = 40,na.rm = T,col="white") +
# xlim(NA, 1) +
labs(x=NULL,y=NULL) +
facet_wrap(~chapter, ncol=2,scales = "free_y")
词频长尾分布
3.3 齐普夫定律（Zipf’s law）
在一个自然语言的语料库中，一个词的出现频数和这个词在这个语料中的排名（这个排名是基于出现次数的）成反比。Zipf定律是文献计量学的重要定律之一，它和罗特卡定律、布拉德福定律一起被并称为文献计量学的三大定律。
book %>% select(chapter,n) %>% mutate(rank=10*(1:length(n))) %>%
ggplot(aes(rank,n)) + geom_point(col="red") + scale_x_log10()
齐普夫定律
3.4 TF-IDF矩阵
book.tfidf <- book %>% select(-max) %>% bind_tf_idf(word,chapter,n) %>%
arrange(-tf_idf)
# tf_idf表示词语在文章中的重要程度，可视化前两章最重要的13个词
book.tfidf %>% mutate(word=factor(word,levels = rev(unique(word)))) %>%
filter(chapter<3) %>% group_by(chapter) %>% top_n(13,tf_idf) %>%
ungroup() %>% ggplot(aes(word,tf_idf,fill=chapter)) +
geom_col(show.legend = F) +
labs(x=NULL,y="tf_idf") +
facet_wrap(~chapter,ncol = 2,scales = "free") +
coord_flip()
TF-IDF
