1、数据整理
library(pacman)
p_load(dplyr,stringr)
lj <- readLines("./亮剑.txt",n=-1L,ok=T,warn=F,skipNul = T) %>%
gsub("[《亮剑》 作者：都梁 | 更多精彩，更多好书，尽在www.qisuu.com]","",.) %>%
str_trim(side = "both")
lj <- lj[which(nchar(lj) > 0)]
id <- lj %>% str_match("^第.*章$|^尾声$")
chapter <- id[which(nchar(id)>0)]
txt <- lj %>% gsub("^第.*章$|^尾声$","★",.) %>%
paste0(collapse = "") %>% str_split("★") %>% unlist()
df <- tibble(chapter=chapter,content=txt[which(nchar(txt)>0)])
write.csv(df,"亮剑.csv",row.names = F)
整理前
整理后
2、读取小说文本
lj <- read.csv("亮剑.csv",header = T,stringsAsFactors = F) %>% tbl_df()
names(lj)
## [1] "chapter" "content"
xslm <- read.csv("血色浪漫-合并章节.csv",header = T,stringsAsFactors = F) %>%
select(chapter,content) %>% tbl_df()
names(xslm)
## [1] "chapter" "content"
levels <- c("引子","第一章","第二章","第三章","第四章","第五章","第六章","第七章","第八章","第九章","第十章","第十一章","第十二章","第十三章","第十四章","第十五章","第十六章","第十七章","第十八章","第十九章","第二十章","第二十一章","第二十二章","第二十三章","第二十四章","第二十五章","第二十六章","第二十七章","第二十八章","第二十九章","第三十章","第三十一章","第三十二章","第三十三章","第三十四章","第三十五章","第三十六章","第三十七章","第三十八章","第三十九章","第四十章","第四十一章","第四十二章","第四十三章","尾声")
lj$chapter <- factor(lj$chapter,levels = levels)
xslm$chapter <- factor(xslm$chapter,levels = levels)
3、各章节字数对比
p_load(ggplot2)
bind_rows(lj %>% mutate(name = "亮剑"),
xslm %>% mutate(name = "血色浪漫")) %>%
mutate(n=nchar(content)) %>%
ggplot(aes(chapter,n,fill=name)) +
geom_col(position="stack") +
labs(y=NULL,x=NULL) +
theme(axis.text.x = element_text(angle = 90,hjust = 1),
legend.position = "top",
legend.title = element_blank())
章节字数对比
4、中文分词
p_load(jiebaR,purrr)
user <- "./dict/characters-master/xslm"
stopwords <- "./dict/stopwords_wf.txt"
wk <- worker(user = user,stop_word = stopwords)
tok_fun <- function(strings) {map(strings,segment,wk)}
lj$words <- lj$content %>% tok_fun
xslm$words <- xslm$content %>% tok_fun
5、词频对比
p_load(text2vec)
# 预处理函数，英文转小写，并清除非字符串（标点符号等）
preprocessor = function(x) {
gsub("[^[:alnum:]\\s]", replacement = " ", tolower(x))
}
txt.lj <- paste0(lj$words,collapse = " ")
txt.xslm <- paste0(xslm$words,collapse = " ")
df <- tibble(name ="亮剑",words=txt.lj) %>%
bind_rows(tibble(name="血色浪漫",words=txt.xslm))
it <- itoken(df$words,
ids = df$name,
preprocessor = preprocessor,
progressbar = F)
vocab <- create_vocabulary(it)
freq <- vocab %>% mutate(name=case_when(
doc_count == 1 ~ "《血色浪漫》",
doc_count == 2 ~ "《亮剑》")) %>%
group_by(name) %>%
select(term,term_count,name) %>%
arrange(-term_count);freq
## # A tibble: 36,452 x 3
## # Groups: name [2]
## term term_count name
## <chr> <int> <chr>
## 1 钟跃民 3117 《血色浪漫》
## 2 说 2853 《亮剑》
## 3 李云龙 1936 《血色浪漫》
## 4 里 972 《亮剑》
## 5 想 955 《亮剑》
## 6 袁军 903 《血色浪漫》
## 7 郑桐 776 《血色浪漫》
## 8 周晓白 716 《血色浪漫》
## 9 宁伟 703 《血色浪漫》
## 10 道 652 《亮剑》
## # ... with 36,442 more rows
freq %>% filter(term_count>10 & term_count<1000) %>%
ggplot(aes(term_count,term,col=name)) +
geom_jitter(alpha = 0.1, size = 2.5, width = 0.25, height = 0.25,na.rm = T) +
geom_text(aes(label = term),check_overlap = TRUE, vjust = 1.5,na.rm = T) +
geom_abline(color = "red") +
labs(x=NULL,y=NULL) +
theme(axis.text.y = element_blank(),
axis.ticks.y = element_blank(),
legend.title = element_blank(),
legend.position = "top")
词频对比
6、分别使用频率最高的词
freq %>% filter(name=="《血色浪漫》") %>%
top_n(15)
## Selecting by name
## # A tibble: 26,114 x 3
## # Groups: name [1]
## term term_count name
## <chr> <int> <chr>
## 1 钟跃民 3117 《血色浪漫》
## 2 李云龙 1936 《血色浪漫》
## 3 袁军 903 《血色浪漫》
## 4 郑桐 776 《血色浪漫》
## 5 周晓白 716 《血色浪漫》
## 6 宁伟 703 《血色浪漫》
## 7 张海洋 519 《血色浪漫》
## 8 田雨 420 《血色浪漫》
## 9 跃民 412 《血色浪漫》
## 10 赵刚 388 《血色浪漫》
## # ... with 26,104 more rows
freq %>% filter(name=="《亮剑》") %>%
top_n(15)
## Selecting by name
## # A tibble: 10,338 x 3
## # Groups: name [1]
## term term_count name
## <chr> <int> <chr>
## 1 说 2853 《亮剑》
## 2 里 972 《亮剑》
## 3 想 955 《亮剑》
## 4 道 652 《亮剑》
## 5 中 643 《亮剑》
## 6 走 642 《亮剑》
## 7 事 608 《亮剑》
## 8 时 571 《亮剑》
## 9 部队 541 《亮剑》
## 10 两个 468 《亮剑》
## # ... with 10,328 more rows
freq %>% group_by(name) %>%
mutate(tf = term_count / sum(term_count)) %>%
top_n(10) %>%
ggplot(aes(reorder(term,tf), tf, fill = name)) +
geom_col(show.legend = FALSE) +
coord_flip() +
labs(x=NULL,y=NULL)
## Selecting by tf
高频词
