NLP入门：文本预处理（二）数据清洗

#文本预处理——文本数据清洗，本文以kaggle比赛 toxic_comment为例
def clean_text(text):
    """
    Clean text
    :param text: the string of text
    :return: text string after cleaning
    """
        # acronym
    text = re.sub(r"can\'t", "can not", text)
    text = re.sub(r"cannot", "can not ", text)
    text = re.sub(r"what\'s", "what is", text)
    text = re.sub(r"What\'s", "what is", text)
    text = re.sub(r"\'ve ", " have ", text)
    text = re.sub(r"n\'t", " not ", text)
    text = re.sub(r"i\'m", "i am ", text)
    text = re.sub(r"I\'m", "i am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r"c\+\+", "cplusplus", text)
    text = re.sub(r"c \+\+", "cplusplus", text)
    text = re.sub(r"c \+ \+", "cplusplus", text)
    text = re.sub(r"c#", "csharp", text)
    text = re.sub(r"f#", "fsharp", text)
    text = re.sub(r"g#", "gsharp", text)
    text = re.sub(r" e mail ", " email ", text)
    text = re.sub(r" e \- mail ", " email ", text)
    text = re.sub(r" e\-mail ", " email ", text)
    text = re.sub(r",000", '000', text)
    text = re.sub(r"\'s", " ", text)

    # spelling correction
    text = re.sub(r"ph\.d", "phd", text)
    text = re.sub(r"PhD", "phd", text)
    text = re.sub(r"pokemons", "pokemon", text)
    text = re.sub(r"pokémon", "pokemon", text)
    text = re.sub(r"pokemon go ", "pokemon-go ", text)
    text = re.sub(r" e g ", " eg ", text)
    text = re.sub(r" b g ", " bg ", text)
    text = re.sub(r" 9 11 ", " 911 ", text)
    text = re.sub(r" j k ", " jk ", text)
    text = re.sub(r" fb ", " facebook ", text)
    text = re.sub(r"facebooks", " facebook ", text)
    text = re.sub(r"facebooking", " facebook ", text)
    text = re.sub(r"insidefacebook", "inside facebook", text)
    text = re.sub(r"donald trump", "trump", text)
    text = re.sub(r"the big bang", "big-bang", text)
    text = re.sub(r"the european union", "eu", text)
    text = re.sub(r" usa ", " america ", text)
    text = re.sub(r" us ", " america ", text)
    text = re.sub(r" u s ", " america ", text)
    text = re.sub(r" U\.S\. ", " america ", text)
    text = re.sub(r" US ", " america ", text)
    text = re.sub(r" American ", " america ", text)
    text = re.sub(r" America ", " america ", text)
    text = re.sub(r" quaro ", " quora ", text)
    text = re.sub(r" mbp ", " macbook-pro ", text)
    text = re.sub(r" mac ", " macbook ", text)
    text = re.sub(r"macbook pro", "macbook-pro", text)
    text = re.sub(r"macbook-pros", "macbook-pro", text)
    
    # special replacement
    text = re.sub(r"&", " and ", text)
    text = re.sub(r"\|", " or ", text)
    text = re.sub(r"=", " equal ", text)
    text = re.sub(r"\+", " plus ", text)
    text = re.sub(r"₹", " rs ", text)      # 测试！
    text = re.sub(r"\$", " dollar ", text)
    text = re.sub(r"\t+", " ", text)
    text = re.sub(r"\n", " ", text)
    text = re.sub(r"\s{2,}", " ", text)
#     text = re.sub(r"\w{5,}", "", text)
    text = re.sub(r"^\s+|\s+$^", "", text)
#     text = re.sub(r"^\s|\s$^", "", text)
    text = re.sub('[☎]+', "", text)
    text = re.sub('[★]+', "", text)
    text = re.sub('[❤☮☺☯⚇♔✉✔😉→™₪☓♠♥♦♣†🎄♦☻–¡¶♫😜✽△←✎¤¢]+', "", text)
    text = re.sub('[♥]+', "", text)
    text = re.sub('സംവാദം മോസ്കോ മെട്രോ', "", text)
    text = re.sub('[κρανιώνας]+', "", text)
    text = re.sub('[дреновени]+', "", text)
    text = re.sub("[><·;:\`\~\-?\s+\.\!\/_,$%^\*)+(+\"\'\[\]{+}+\\]+|[+——！•，：。？、~@#￥%……&*‘’“”（）「」]+", " ",text)
    text = re.sub(r'f+u+c+k+','fuck',text)
    text = re.sub('[働労勞連絡見学迷惑]+','', text)
    text = re.sub('¡ ¿ † ‡ ↔ ↑ ↓ ¶ ½ ⅓ ⅔ ¼ ¾ ⅛ ⅜ ⅝ ⅞ ∞ ‘ “ ’ ” «» ¤ ₳ ฿ ₵ ¢ ₡ ₢ ₫ ₯ € ₠ ₣ ƒ ₴ ₭ ₤ ℳ ₥ ₦ № ₧ ₰ £ ៛ ₨ ₪ ৳ ₮ ₩ ¥ ♠ ♣ ♥ ♦ m² m³ characters á á ć ć é é í í ĺ ĺ ń ń ó ó ŕ ŕ ś ś ú ú ý ý ź ź à à è è ì ì ò ò ù ù â â ĉ ĉ ê ê ĝ ĝ ĥ ĥ î î ĵ ĵ ô ô ŝ ŝ û û ŵ ŵ ŷ ŷ ä ä ë ë ï ï ö ö ü ü ÿ ÿ ß ã ã ẽ ẽ ĩ ĩ ñ ñ õ õ ũ ũ ỹ ỹ ç ç ģ ģ ķ ķ ļ ļ ņ ņ ŗ ŗ ş ş ţ ţ đ đ ů ů ǎ ǎ č č ď ď ě ě ǐ ǐ ľ ľ ň ň ǒ ǒ ř ř š š ť ť ǔ ǔ ž ž ā ā ē ē ī ī ō ō ū ū ȳ ȳ ǣ ǣ ǖ ǘ ǚ ǜ ă ă ĕ ĕ ğ ğ ĭ ĭ ŏ ŏ ŭ ŭ ċ ċ ė ė ġ ġ i̇ ı ż ż ą ą ę ę į į ǫ ǫ ų ų ḍ ḍ ḥ ḥ ḷ ḷ ḹ ḹ ṃ ṃ ṇ ṇ ṛ ṛ ṝ ṝ ṣ ṣ ṭ ṭ ł ł ő ő ű ű ŀ ŀ ħ ħ ð ð þ þ œ œ æ æ ø ø å å ə ə greek ά ά έ έ ή ή ί ί ό ό ύ ύ ώ ώ α α β β γ γ δ δ ε ε ζ ζ η η θ θ ι ι κ κ λ λ μ μ ν ν ξ ξ ο ο π π ρ ρ σ σ ς τ τ υ υ φ φ χ χ ψ ψ ω ω cyrillic а а б б в в г г ґ ґ ѓ ѓ д д ђ ђ е е ё ё є є ж ж з з ѕ ѕ и и і і ї ї й й ј ј к к ќ ќ л л љ љ м м н н њ њ о о п п р р с с т т ћ ћ у у ў ў ф ф х х ц ц ч ч џ џ ш ш щ щ ъ ъ ы ы ь ь э э ю ю я я ipa t̪ d̪ ʈ ɖ ɟ ɡ ɢ ʡ ʔ ɸ ʃ ʒ ɕ ʑ ʂ ʐ ʝ ɣ ʁ ʕ ʜ ʢ ɦ ɱ ɳ ɲ ŋ ɴ ʋ ɹ ɻ ɰ ʙ ʀ ɾ ɽ ɫ ɬ ɮ ɺ ɭ ʎ ʟ ɥ ʍ ɧ ɓ ɗ ʄ ɠ ʛ ʘ ǀ ǃ ǂ ǁ ɨ ʉ ɯ ɪ ʏ ʊ ɘ ɵ ɤ ə ɚ ɛ ɜ ɝ ɞ ʌ ɔ ɐ ɶ ɑ ɒ ʰ ʷ ʲ ˠ ˤ ⁿ ˡ ˈ ˌ ː ˑ ̪','',text)

    text = re.sub(r"\$(\d+)", lambda m: m.group(1) + ' dollar ', text)
    text = re.sub(r"(\d+)\$", lambda m: m.group(1) + ' dollar ', text)

#     text = re.sub(r" 1 ", " one ", text)
#     text = re.sub(r" 2 ", " two ", text)
#     text = re.sub(r" 3 ", " three ", text)
#     text = re.sub(r" 4 ", " four ", text)
#     text = re.sub(r" 5 ", " five ", text)
#     text = re.sub(r" 6 ", " six ", text)
#     text = re.sub(r" 7 ", " seven ", text)
#     text = re.sub(r" 8 ", " eight ", text)
#     text = re.sub(r" 9 ", " nine ", text)
    text = re.sub(r'\d+', '', text)
    text = re.sub(r"googling", " google ", text)
    text = re.sub(r"googled", " google ", text)
    text = re.sub(r"googleable", " google ", text)
    text = re.sub(r"googles", " google ", text)
    text = re.sub(r" rs(\d+)", lambda m: ' rs ' + m.group(1), text)
    text = re.sub(r"(\d+)rs", lambda m: ' rs ' + m.group(1), text)
    text = re.sub(r"the european union", " eu ", text)
    text = re.sub(r"dollars", " dollar ", text)

    # remove extra space
    text = ' '.join(text.split(' '))
    return text