这几天对DeepSeek研究不少，重新把之前不熟悉的知识点再check一下。参照的书籍正是《Build a Large Language Model (From Scratch)》，良心好书哟，希望快出中文版收藏哈。

一，手工实现

代码

import re

class SimpleTokenizerV2:
    def __init__(self, vocab):
        self.str_to_int = vocab
        self.int_to_str = {i: s for s, i in vocab.items()}

    def encode(self, text):
        preprocessed = re.split(r'([,.:;?_!"()\']|--|\s)', text)
        preprocessed = [item.strip() for item in preprocessed if item.strip()]
        preprocessed = [
            item if item in self.str_to_int
            else "<|unk|>" for item in preprocessed
        ]
        ids = [self.str_to_int[s] for s in preprocessed]
        return ids

    def decode(self, ids):
        text = " ".join([self.int_to_str[i] for i in ids])
        # Replace spaces before the specified punctuations
        text = re.sub(r'\s+([,.?!"()\'])', r'\1', text)
        return text


with open('the-verdict.txt', 'r', encoding='utf-8') as f:
    raw_text = f.read()

preprocessed = re.split(r'([,.?_!"()\']|--|\s)', raw_text)
preprocessed = [item.strip() for item in preprocessed if item.strip()]

all_tokens = sorted(list(set(preprocessed)))
all_tokens.extend(["<|endoftext|>", "<|unk|>"])
vocab = {token:integer for integer,token in enumerate(all_tokens)}

tokenizer = SimpleTokenizerV2(vocab)

text1 = "Hello, do you like tea?"
text2 = "In the sunlit terraces of the palace."

text = " <|endoftext|> ".join((text1, text2))

print(text)
print(tokenizer.decode(tokenizer.encode(text)))

输出

D:\Python\Python310\python.exe D:\test\Llm_Scratch\token_test.py 
Hello, do you like tea? <|endoftext|> In the sunlit terraces of the palace.
<|unk|>, do you like tea? <|endoftext|> In the sunlit terraces of the <|unk|>.

Process finished with exit code 0

二，pip库实现

代码

import tiktoken
from importlib.metadata import version

print(version("tiktoken"))

tokenizer = tiktoken.get_encoding('gpt2')
text = (
    "Hello, do you like tea? <|endoftext|> In the sunlit terraces"
     "of someunknownPlace."
)
integers = tokenizer.encode(text, allowed_special={"<|endoftext|>"})

print(integers)

strings = tokenizer.decode(integers)

print(strings)

输出

D:\Python\Python310\python.exe D:\test\Llm_Scratch\tiktoken_test.py 
0.5.1
[15496, 11, 466, 345, 588, 8887, 30, 220, 50256, 554, 262, 4252, 18250, 8812, 2114, 1659, 617, 34680, 27271, 13]
Hello, do you like tea? <|endoftext|> In the sunlit terracesof someunknownPlace.

Process finished with exit code 0

人工实现，可以知道原理，库实现，可以更快速高效

image.png