######################################################## 
#-------------------------------------------------------
# Topic:从reactome数据库中爬取通道蛋白的描述信息
# Author:Wang Haiquan
# Date:Tue Jul 21 08:38:41 2020
# Mail:mg1835020@smail.nju.edu.cn
#-------------------------------------------------------
########################################################

#1、通道蛋白列表
#2、reactome的url规律
# 3、建立url集合
# 4、爬取信息
library(rvest)
transporter_gene_list
reactome_result<-list()
for (j in 1:length(transporter_gene_list$gene_sym)) {
  gene=as.character(transporter_gene_list$gene_sym[j])
  reactome_result[[j]]<-list()
  names(reactome_result)[j]<-gene
  url<-paste('https://reactome.org/content/query?q=',gene,'&species=Homo+sapiens&species=Entries+without+species&cluster=true',sep = "")
  #报错控制，返回404表示数据库中没有这个基因，检测到该报错后自动跳过
  fit=try({reactome_session<-html_session(url)
  reactome_html<-read_html(reactome_session)})
  if("try-error"%in%class(fit)){next}
  reactome_nodes<-list(reaction='//a[text()="Reaction"]',
                       reaction_url='//a[text()="Reaction"]/parent::h3/parent::div/following-sibling::div/div/h4/a',
                       reaction_url_summary='//div[@class="details-summation"]',
                       reaction_input='//div[contains(text(),"Input")]',
                       reaction_input_detail='//div[contains(text(),"Input")]/following-sibling::div//li/a',
                       reaction_output='//div[contains(text(),"Output")]',
                       reaction_output_detail='//div[contains(text(),"Output")]/following-sibling::div//li/a'
  )
  
  
  #如果没有reaction，返回NA，自动检测NA 决定是否跳过
  #如果有reaction，则获得进一步的url,取得href属性
  #获得summary所有的text
  #判断是否有input和output，没有，则下一个(没有的话会返回NA)
  #如果有，则保存detail
  first_detect=html_node(reactome_html,xpath = reactome_nodes[[1]])%>%html_text()
  if(!is.na(first_detect)){
    reactome_html_url2=html_nodes(reactome_html,xpath = reactome_nodes[[2]])%>%html_attr(name = "href")
    reactome_name<-html_nodes(reactome_html,xpath =reactome_nodes[[2]] )%>%html_text()
    for (i in 1:length(reactome_html_url2)) {
      reactome_name_sub<-reactome_name[i]
      reactome_html_2=paste('https://reactome.org/content',str_split(reactome_html_url2[i],"\\.",simplify = T)[,2],sep = "")
      reactome_html_2=read_html(reactome_html_2)
      reactome_summary=html_node(reactome_html_2,xpath = reactome_nodes[[3]])%>%html_text()
      reactome_input=html_node(reactome_html_2,xpath = reactome_nodes[[5]])%>%html_text()
      reactome_output=html_node(reactome_html_2,xpath = reactome_nodes[[7]])%>%html_text()
      reactome_result[[j]][[i]]<-list(summary=reactome_summary,
                                      input=reactome_input,
                                      output=reactome_output)
      names(reactome_result[[j]])[i]<-reactome_name_sub
    }
    
    
  }
  
}