1、数据整理

library(pacman)
p_load(dplyr,stringr)
lj <- readLines("./亮剑.txt",n=-1L,ok=T,warn=F,skipNul = T) %>% 
  gsub("[《亮剑》 作者：都梁 | 更多精彩，更多好书，尽在www.qisuu.com]","",.) %>%
  str_trim(side = "both")

lj <- lj[which(nchar(lj) > 0)]

id <- lj %>% str_match("^第.*章$|^尾声$")
chapter <- id[which(nchar(id)>0)]

txt <- lj %>% gsub("^第.*章$|^尾声$","★",.) %>% 
  paste0(collapse = "") %>% str_split("★") %>% unlist()

df <- tibble(chapter=chapter,content=txt[which(nchar(txt)>0)])

write.csv(df,"亮剑.csv",row.names = F)

整理前

整理后

2、读取小说文本

lj <- read.csv("亮剑.csv",header = T,stringsAsFactors = F) %>% tbl_df()
names(lj)

## [1] "chapter" "content"

xslm <- read.csv("血色浪漫-合并章节.csv",header = T,stringsAsFactors = F) %>% 
  select(chapter,content) %>% tbl_df()
names(xslm)

## [1] "chapter" "content"

levels <- c("引子","第一章","第二章","第三章","第四章","第五章","第六章","第七章","第八章","第九章","第十章","第十一章","第十二章","第十三章","第十四章","第十五章","第十六章","第十七章","第十八章","第十九章","第二十章","第二十一章","第二十二章","第二十三章","第二十四章","第二十五章","第二十六章","第二十七章","第二十八章","第二十九章","第三十章","第三十一章","第三十二章","第三十三章","第三十四章","第三十五章","第三十六章","第三十七章","第三十八章","第三十九章","第四十章","第四十一章","第四十二章","第四十三章","尾声")

lj$chapter <- factor(lj$chapter,levels = levels)
xslm$chapter <- factor(xslm$chapter,levels = levels)

3、各章节字数对比

p_load(ggplot2)

bind_rows(lj %>% mutate(name = "亮剑"),
          xslm %>% mutate(name = "血色浪漫")) %>% 
 mutate(n=nchar(content)) %>%
 ggplot(aes(chapter,n,fill=name)) +
   geom_col(position="stack") +
   labs(y=NULL,x=NULL) +
   theme(axis.text.x = element_text(angle = 90,hjust = 1),
        legend.position = "top",
        legend.title = element_blank())

章节字数对比

4、中文分词

p_load(jiebaR,purrr)
user <- "./dict/characters-master/xslm"
stopwords <- "./dict/stopwords_wf.txt"
wk <- worker(user = user,stop_word = stopwords)
tok_fun <- function(strings) {map(strings,segment,wk)}

lj$words <- lj$content %>% tok_fun
xslm$words <- xslm$content %>% tok_fun

5、词频对比

p_load(text2vec)

# 预处理函数，英文转小写，并清除非字符串（标点符号等）
preprocessor = function(x) {
  gsub("[^[:alnum:]\\s]", replacement = " ", tolower(x))
  }

txt.lj <- paste0(lj$words,collapse = " ")
txt.xslm <- paste0(xslm$words,collapse = " ")
df <- tibble(name ="亮剑",words=txt.lj) %>%
  bind_rows(tibble(name="血色浪漫",words=txt.xslm))

  
it <- itoken(df$words,
             ids = df$name,
             preprocessor = preprocessor,
             progressbar = F)

vocab <- create_vocabulary(it)

freq <- vocab %>% mutate(name=case_when(
  doc_count == 1 ~ "《血色浪漫》",
  doc_count == 2 ~ "《亮剑》")) %>%
  group_by(name) %>%
  select(term,term_count,name) %>%
  arrange(-term_count);freq

## # A tibble: 36,452 x 3
## # Groups:   name [2]
##    term   term_count name        
##    <chr>       <int> <chr>       
##  1 钟跃民       3117 《血色浪漫》
##  2 说           2853 《亮剑》    
##  3 李云龙       1936 《血色浪漫》
##  4 里            972 《亮剑》    
##  5 想            955 《亮剑》    
##  6 袁军          903 《血色浪漫》
##  7 郑桐          776 《血色浪漫》
##  8 周晓白        716 《血色浪漫》
##  9 宁伟          703 《血色浪漫》
## 10 道            652 《亮剑》    
## # ... with 36,442 more rows

freq %>% filter(term_count>10 & term_count<1000) %>%
  ggplot(aes(term_count,term,col=name)) +
  geom_jitter(alpha = 0.1, size = 2.5, width = 0.25, height = 0.25,na.rm = T) +
  geom_text(aes(label = term),check_overlap = TRUE, vjust = 1.5,na.rm = T) +
  geom_abline(color = "red") +
  labs(x=NULL,y=NULL) +
  theme(axis.text.y = element_blank(),
        axis.ticks.y = element_blank(),
        legend.title = element_blank(),
        legend.position = "top")

词频对比

6、分别使用频率最高的词

freq %>% filter(name=="《血色浪漫》") %>%
  top_n(15)

## Selecting by name

## # A tibble: 26,114 x 3
## # Groups:   name [1]
##    term   term_count name        
##    <chr>       <int> <chr>       
##  1 钟跃民       3117 《血色浪漫》
##  2 李云龙       1936 《血色浪漫》
##  3 袁军          903 《血色浪漫》
##  4 郑桐          776 《血色浪漫》
##  5 周晓白        716 《血色浪漫》
##  6 宁伟          703 《血色浪漫》
##  7 张海洋        519 《血色浪漫》
##  8 田雨          420 《血色浪漫》
##  9 跃民          412 《血色浪漫》
## 10 赵刚          388 《血色浪漫》
## # ... with 26,104 more rows

freq %>% filter(name=="《亮剑》") %>%
  top_n(15)

## Selecting by name

## # A tibble: 10,338 x 3
## # Groups:   name [1]
##    term  term_count name    
##    <chr>      <int> <chr>   
##  1 说          2853 《亮剑》
##  2 里           972 《亮剑》
##  3 想           955 《亮剑》
##  4 道           652 《亮剑》
##  5 中           643 《亮剑》
##  6 走           642 《亮剑》
##  7 事           608 《亮剑》
##  8 时           571 《亮剑》
##  9 部队         541 《亮剑》
## 10 两个         468 《亮剑》
## # ... with 10,328 more rows

freq %>% group_by(name) %>%
  mutate(tf = term_count / sum(term_count)) %>%
  top_n(10) %>%
  ggplot(aes(reorder(term,tf), tf, fill = name)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  labs(x=NULL,y=NULL)

## Selecting by tf

高频词