自然语言处理之词性标注和命名实体识别

编程环境：

anaconda + python3.7
完整代码及数据已经更新至GitHub,欢迎fork~GitHub链接

声明：创作不易，未经授权不得复制转载
statement:No reprinting without authorization

相关工具包介绍安装请查看该篇文章内链接

内容：

利用 Chinese.txt 和 English.txt 的中英文句子，在分词的基础上，继续利用以下给定的中英文工具进行词性标注和命名实体识别。并对不同工具产生的结果进行简要对比分析。

1、英文工具：

Nltk
Spacy
Stanfordnlp

2、中文工具：（部分工具命名实体识别没有直接调用的函数，可以根据词性

标注的结果自己实现）
Jieba
StanfordCoreNLP
SnowNLP
THULAC
NLPIR
HanLP（需要 Microsoft Visual C++ 14.0）或安装教程

一、python工具包下载安装准备：

（1）nltk的命名实体功能模块安装：

img

（2）pyhanlp的安装：
教程示例：https://blog.csdn.net/huangjiajia123/article/details/84144583
第一步：下载 jpype：https://www.lfd.uci.edu/~gohlke/pythonlibs/#jpype下载对应版本已经编译好的whl文件。将 .whl 文件保存到python 所在的script 文件夹下，然后安装： pip install 【下载的文件名】
第二步：安装 pyhanlp: pip install pyhanlp
第三步：安装完成后并不能使用，需要下载一个jar包、data文件和properties文件，因为hanlp是java开发的虽然有python的API但是还是需要java环境，所以需要安装JDK，并配置Java 环境变量（即添加一个JAVA_HOME变量，变量值为java的bin目录的绝对路径）。
（1）打开 python IDE，输入 import pyhanlp，会自动下载 HanLP jar 和properties 文件，默认放在 python ->Lib->site packages 文件夹下的 pyhanlp ->static 文件夹下
（2）可以将 hanlp-1.7.0.jar 和hanlp.properties 移动到一个新的文件夹下，比如： D\HanLp
（3）去https://github.com/hankcs/HanLP/releases下载 hanlp 的data。

二、进行测试观察结果：

1、中文文本：

675754439270425c58c5cbdbb5c054e7.png

fe9a54049ba9a56fadad962e4c826386.png

f5f4a0b7e4d9a49325e0ad81b8b66fb9.png

ce1f474b6f6c3351c3ed9cd2c0a07c0e.png

2、英语文本：

NLTK标注树状图：

638c0d989f6ccb3ee560fefad29ed3e7.png

2bf6f485ff079726301948a70bc159a3.png

# -*- coding: utf-8 -*-
"""
Created on Tue Mar 26 18:31:10 2019
实验3 NLP
@author:Mr.relu
"""

import time
import jieba
import jieba.posseg as pseg  
from snownlp import SnowNLP
import thulac   
import pynlpir
from stanfordcorenlp import StanfordCoreNLP
import nltk
import spacy 
from nltk.corpus import treebank
from pyhanlp import *

spacy_nlp = spacy.load('en_core_web_sm')

def chinese_Txt_test(document):
    """
    hanlp中文测试
    """
    print(">>>>>hanlp tagging start...")
    start = time.process_time()
    
    result0 = HanLP.segment(document)
    
    elapsed = (time.process_time() - start)  
    print("hanlp Time used:",elapsed)
    print("《《hanlp》》: " )  
    
    for term in result0:
        print('{}--{}\t'.format(term.word, term.nature),end=' ') # 获取单词与词性
    print("\n")
    
    """
    测试工具包jieba词性标注
    """
    print(">>>>>jieba tagging start...")
    start = time.process_time()
    
    result1 = pseg.lcut(document)
    
    elapsed = (time.process_time() - start)  
    print("jieba Time used:",elapsed)
    print("《《jieba》》: " )  
    for word,tag in result1:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    
    """
    测试工具包SnowNLP
    """
    print(">>>>>SnowNLP tagging start...")
    start = time.process_time()
    s = SnowNLP(document)
    result2 = s.tags                
    elapsed = (time.process_time() - start)
    print("SnowNLP Time used:",elapsed)
    print("《《SnowNLP》》: " )  
    for word,tag in result2:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    
    """
    测试StanfordCoreNLP工具包
    """
    print(">>>>>StanfordCoreNLP tagging start...")
    start = time.process_time()
    nlp = StanfordCoreNLP(r'D:\anaconda\Lib\stanford-corenlp-full-2018-02-27',lang = 'zh')
    result3 = nlp.pos_tag(document)
    elapsed = (time.process_time() - start)
    print("StanfordCoreNLP Time used:",elapsed)
    print("《《StanfordCoreNLP》》: " )
    for word,tag in result3:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    #StanfordCoreNLP 'Named Entities:', nlp.ner(sentence)
    print(">>>>>StanfordCoreNLP Named Entities start...")
    start = time.process_time()
    result4 = nlp.ner(document)
    elapsed = (time.process_time() - start)
    print("StanfordCoreNLP Named Entities Time used:",elapsed)
    print("《《StanfordCoreNLP》》: " )
    nlp.close() 
    for word,tag in result4:
        if tag != 'O':
            print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    
    """
    测试thulac工具包
    """
    print(">>>>>thulac tagging start...")
    start = time.process_time()
    
    thu1 = thulac.thulac()  #默认模式
    text2d = thu1.cut(document, text=False)  #进行一句话分词
    
    elapsed = (time.process_time() - start)
    print("thulac Time used:",elapsed)
    print("《《thulac》》: ")
    for tu in text2d:
        print(str(tu[0])+"--"+str(tu[1])+"\t", end=' ')
    print('\n')
    
    
    """
    测试pynlpir工具包
    """
    print(">>>>>pynlpir tagging start...")
    start = time.process_time()
    pynlpir.open()
    result5 = pynlpir.segment(document)
    elapsed = (time.process_time() - start)
    print("pynlpir Time used:",elapsed)
    print("《《pynlpir》》:") 
    for word,tag in result5:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")

def english_Txt_test(doc):
    print(">>>>>NLTK tokenization start...")
    start = time.process_time()
    
    tokens = nltk.word_tokenize(doc)
    tagged = nltk.pos_tag(tokens)
    entities = nltk.chunk.ne_chunk(tagged)
    t = treebank.parsed_sents('wsj_0001.mrg')[0]
    t.draw()  
    elapsed = (time.process_time() - start)    
    print("NLTK Time used:",elapsed)
    print(entities)
    print("《《NLTK》》: " )
    for word,tag in tagged:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    """
    英文分词spacy
    """
    print(">>>>>spacy tagging start...")
    start = time.process_time()
    s_doc = spacy_nlp(doc)
    
    elapsed = (time.process_time() - start)
    print("spacy Time used:",elapsed)
    print("《《Spacy》》: ")
    for token in s_doc:
        print(str(token)+ "--"+str(token.pos_)+"\t", end=' ')
    print("\n")
    for ent in s_doc.ents:
        print(ent,"----", ent.label_, ent.label)
    print("\n")
    """
    英文分词StanfordCoreNLP
    """
    print(">>>>>StanfordCoreNLP tagging start...")
    start = time.process_time()
    nlp2 = StanfordCoreNLP(r'D:\anaconda\Lib\NLP实验\stanford-corenlp-full-2018-02-27')
     
    result7 =  nlp2.pos_tag(doc)

    elapsed = (time.process_time() - start)
    print("StanfordCoreNLP Time used:",elapsed)
    print("《《StanfordCoreNLP>>: " )
   
    for word,tag in result7:   
        print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    
    #StanfordCoreNLP 'Named Entities:', nlp.ner(sentence)
    print(">>>>>StanfordCoreNLP Named Entities start...")
    start = time.process_time()
    
    result8 =  nlp2.ner(doc)

    elapsed = (time.process_time() - start)
    print("StanfordCoreNLP Named Entities Time used:",elapsed)
    print("《《StanfordCoreNLP》》: " )
    nlp2.close() 
    for word,tag in result8:
        if tag != 'O':
            print(str(word)+"--"+str(tag)+"\t", end=' ')
    print("\n")
    


def main():
    
    f = open('Chinese.txt')
    document = f.read()
    f.close()
    print(document)    
    chinese_Txt_test(document)
    
    f = open('English.txt')
    doc = f.read()
    f.close()
    print(doc)
    print("\n")  
    english_Txt_test(doc)
    
    print("test finished!")
            
if __name__ == '__main__':
    main()