import os

imdb_dir = r"D:\dataset\aclImdb\aclImdb"
train_dir = os.path.join(imdb_dir,"train")

labels = []
texts = []

for label_type in ["neg","pos"]:
    dir_name = os.path.join(train_dir,label_type)
    for fname in os.listdir(dir_name):
        if fname[-4:] == ".txt":
            with open(os.path.join(dir_name,fname),encoding='UTF-8') as f:
                texts.append(f.read())
            if label_type == "neg":
                labels.append(0)
            else:
                labels.append(1)           

from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
import numpy as np

maxlen = 100
training_samples = 200
validation_samples = 10000
max_words = 10000

tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
print(len(sequences))
word_index = tokenizer.word_index
print(len(word_index))

data = sequence.pad_sequences(sequences,maxlen=maxlen)
labels = np.asarray(labels)
print(data.shape)
print(labels.shape)

indices = np.arange(0,data.shape[0])

np.random.shuffle(indices)   #######随机打乱数据

print(labels.shape)

data = data[indices ]
labels = labels[indices]
x_train = data[:training_samples]
y_train = labels[:training_samples]
x_val = data[training_samples:training_samples+validation_samples]
y_val = labels[training_samples:training_samples+validation_samples]
print(x_train.shape)
print(y_train.shape)
print(x_val.shape)
print(y_val.shape)

###########解析GloVe词嵌入文件
glove_dir = r"D:\dataset\glove.6B"
embeddings_index = {}
f = open(os.path.join(glove_dir,"glove.6B.100d.txt"),encoding="utf-8")
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:],dtype="float32")
    embeddings_index[word] = coefs
f.close()
print(len(embeddings_index))

embedding_dim = 100

embedding_matrix = np.zeros((max_words,embedding_dim))
for word,i in word_index.items():
    if i < max_words:
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector

from keras.models import Sequential
from keras.layers import Embedding,Flatten,Dense

model = Sequential()
model.add(Embedding(max_words,embedding_dim,input_length=maxlen))
model.add(Flatten())
model.add(Dense(32,activation="relu"))
model.add(Dense(1,activation="sigmoid"))

model.layers[0].set_weights([embedding_matrix])
model.layers[0].trainable = False

model.compile(
    optimizer = "rmsprop",
    loss = "binary_crossentropy",
    metrics = ["acc"]
)
print(model.summary())

history = model.fit(x_train,y_train,epochs=40,batch_size=32,validation_data=(x_val,y_val))
model.save_weights("glove_model.h5")